diff --git a/.env.dev.example b/.env.dev.example index ba19d905..000d4c76 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -60,6 +60,10 @@ PULLBOX_LOG_LEVEL=INFO PULLBOX_LOG_SIZE_LIMIT_MB=1 PULLBOX_LOG_BACKUP_COUNT=5 +# Manual creation of empty Story Arcs is experimental. Provider and import +# creation remain available while this is disabled. +PULLBOX_STORY_ARC_MANUAL_CREATE_ENABLED=false + # ── Embedded Comic Reader ─────────────────────────────────────────── # Default-on emergency gate. Disabling it preserves comics and resume state. PULLBOX_READER_ENABLED=true diff --git a/.github/scripts/validate-development-image.py b/.github/scripts/validate-development-image.py new file mode 100644 index 00000000..8dfcf1ba --- /dev/null +++ b/.github/scripts/validate-development-image.py @@ -0,0 +1,226 @@ +"""Require completed, same-commit development validation before publishing edge. + +Only explicit runs of the four trusted workflows on develop are accepted. PR +aggregates (including preflight and release-sync fast paths) are not evidence +that the exact commit being packaged passed the full validation suite. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from typing import Any +from urllib.parse import urlencode + +REQUIRED_JOBS: dict[str, dict[str, tuple[str, ...]]] = { + "ci.yml": { + "Quality Gate": ("Ruff lint", "Ruff format check", "Check for uncommitted CSS changes"), + "Type Check": ("Mypy",), + "Migration Check": ( + "Verify migrations apply from empty DB", + "Verify app boots after migration", + ), + "Test (Python 3.12)": ("Run tests with coverage",), + "Test (Python 3.13)": ("Run tests with coverage",), + "Test (Python 3.14)": ("Run tests with coverage",), + "Accessibility Checks": ("Run contrast audit", "Run accessibility browser tests"), + "E2E Tests (chromium)": ("Run E2E tests",), + "E2E Tests (firefox)": ("Run E2E tests",), + "CI Required": (), + }, + "security.yml": { + "Gitleaks": ("Run gitleaks on current tree",), + "pip-audit": ("Run pip-audit",), + "Safety Check": ("Run safety check",), + "Bandit": ("Run Bandit", "Upload Bandit report"), + "Security Required": (), + }, + "workflow-hygiene.yml": { + "actionlint": ("Run actionlint",), + "Workflow Hygiene Required": (), + }, + "docker-validate.yml": { + "Production Docker Validate (trusted)": ( + "Build production Docker image", + "Verify container security runtime", + "Run Grype scan", + "Verify packaged static assets", + "Wait for healthy", + ), + "Docker Validate Required": (), + }, +} + + +class ValidationError(ValueError): + """The available evidence cannot authorize a development publication.""" + + +def _object(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValidationError("GitHub returned an unexpected response shape.") + return value + + +def _positive_id(value: Any) -> int: + if type(value) is not int or value <= 0: + raise ValidationError("GitHub returned an invalid run or check identifier.") + return value + + +def latest_run(runs: list[dict[str, Any]]) -> dict[str, Any]: + if not runs: + raise ValidationError("No manual develop validation run exists for this commit.") + return max(runs, key=lambda run: _positive_id(run.get("id"))) + + +def validate_evidence( + workflow: str, + run: dict[str, Any], + suite: dict[str, Any], + jobs: list[dict[str, Any]], + repository: str, + sha: str, +) -> None: + expected = { + "path": f".github/workflows/{workflow}", + "event": "workflow_dispatch", + "head_sha": sha, + "head_branch": "develop", + "status": "completed", + "conclusion": "success", + } + if any(run.get(key) != value for key, value in expected.items()): + raise ValidationError( + f"{workflow}: latest run is not a successful exact-commit develop run." + ) + for field in ("repository", "head_repository"): + if _object(run.get(field)).get("full_name") != repository: + raise ValidationError(f"{workflow}: validation came from another repository.") + app = _object(suite.get("app")) + if ( + suite.get("id") != _positive_id(run.get("check_suite_id")) + or suite.get("head_sha") != sha + or app.get("slug") != "github-actions" + or _object(app.get("owner")).get("login") != "github" + ): + raise ValidationError(f"{workflow}: missing trusted GitHub Actions check provenance.") + + run_id = _positive_id(run.get("id")) + attempt = _positive_id(run.get("run_attempt")) + for name, required_steps in REQUIRED_JOBS[workflow].items(): + matches = [job for job in jobs if job.get("name") == name] + if len(matches) != 1: + raise ValidationError(f"{workflow}: required job {name!r} is missing or duplicated.") + job = matches[0] + if any( + job.get(key) != value + for key, value in { + "run_id": run_id, + "run_attempt": attempt, + "head_sha": sha, + "status": "completed", + "conclusion": "success", + }.items() + ): + raise ValidationError( + f"{workflow}: required job {name!r} did not succeed in this attempt." + ) + steps = job.get("steps") + if not isinstance(steps, list): + raise ValidationError(f"{workflow}: job {name!r} has no step evidence.") + for step_name in required_steps: + selected = [_object(step) for step in steps if _object(step).get("name") == step_name] + # Preserve the existing advisory Bandit policy, but require that it + # actually ran and that its report job succeeded. Skipped is never OK. + conclusions = {"success"} + if (workflow, name, step_name) == ("security.yml", "Bandit", "Run Bandit"): + conclusions.add("failure") + if ( + len(selected) != 1 + or selected[0].get("status") != "completed" + or selected[0].get("conclusion") not in conclusions + ): + raise ValidationError( + f"{workflow}: required step {step_name!r} did not run successfully." + ) + + +def get_json(endpoint: str) -> dict[str, Any]: + result = subprocess.run( + ["gh", "api", "--hostname", "github.com", "--method", "GET", endpoint], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + if result.returncode: + # Do not forward CLI errors or response bodies into release logs. + raise ValidationError("Unable to read GitHub validation evidence; publication is blocked.") + return _object(json.loads(result.stdout)) + + +def list_all(endpoint: str, field: str) -> list[dict[str, Any]]: + """Fail closed on incomplete pagination instead of accepting a partial job set.""" + records: list[dict[str, Any]] = [] + separator = "&" if "?" in endpoint else "?" + for page in range(1, 11): + payload = get_json(f"{endpoint}{separator}per_page=100&page={page}") + values = payload.get(field) + count = payload.get("total_count") + if not isinstance(values, list) or type(count) is not int or not 0 <= count <= 1000: + raise ValidationError( + "GitHub validation evidence is malformed or exceeds the bounded query." + ) + records.extend(_object(value) for value in values) + if len(records) == count: + return records + if len(values) != 100 or len(records) > count: + break + raise ValidationError("GitHub returned incomplete validation evidence.") + + +def main() -> int: + try: + repository = os.environ["GITHUB_REPOSITORY"] + sha = os.environ["GITHUB_SHA"] + if ( + os.environ.get("GITHUB_EVENT_NAME") != "workflow_dispatch" + or os.environ.get("GITHUB_REF") != "refs/heads/develop" + or not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) + or not re.fullmatch(r"[0-9a-f]{40}", sha) + ): + raise ValidationError("Development publication requires a trusted develop dispatch.") + query = urlencode({"head_sha": sha, "event": "workflow_dispatch", "branch": "develop"}) + for workflow in REQUIRED_JOBS: + base = f"repos/{repository}" + run = latest_run( + list_all(f"{base}/actions/workflows/{workflow}/runs?{query}", "workflow_runs") + ) + run_id = _positive_id(run.get("id")) + attempt = _positive_id(run.get("run_attempt")) + suite_id = _positive_id(run.get("check_suite_id")) + suite = get_json(f"{base}/check-suites/{suite_id}") + jobs = list_all(f"{base}/actions/runs/{run_id}/attempts/{attempt}/jobs", "jobs") + validate_evidence(workflow, run, suite, jobs, repository, sha) + print(f"Validated {workflow}: run {run_id}, attempt {attempt}, commit {sha}") + except ( + ValidationError, + KeyError, + OSError, + subprocess.TimeoutExpired, + json.JSONDecodeError, + ) as exc: + message = ( + str(exc) if isinstance(exc, ValidationError) else "Unable to load validation evidence." + ) + print(f"::error::{message}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c20d4f71..67b53c5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,6 +171,8 @@ jobs: # ────────────────────────────────────────────── # Job 3: Tests + Coverage (matrix: 3.12, 3.13, 3.14) + # Every version runs the complete suite and publishes coverage. The blocking + # 90% release gate follows the production/default Python 3.14 runtime. # ────────────────────────────────────────────── test: name: Test (Python ${{ matrix.python-version }}) @@ -183,7 +185,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12", "3.13", "3.14"] + include: + - python-version: "3.12" + coverage_fail_under: 0 + - python-version: "3.13" + coverage_fail_under: 0 + - python-version: "3.14" + coverage_fail_under: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -206,10 +214,11 @@ jobs: --cov=pullbox \ --cov-report=xml \ --cov-report=term-missing \ - --cov-fail-under=90 \ + --cov-fail-under="${COVERAGE_FAIL_UNDER}" \ --junitxml=test-results.xml \ -v env: + COVERAGE_FAIL_UNDER: ${{ matrix.coverage_fail_under }} PULLBOX_SECRET_KEY: test-ci-key PYTEST_WORKERS: ${{ env.PYTEST_WORKERS }} @@ -391,8 +400,15 @@ jobs: - name: Setup runner-local venv run: .github/scripts/setup-runner-venv.sh "dev,e2e" + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + + - name: Install Node dependencies + run: npm ci + - name: Runner preflight - run: .github/scripts/preflight-runner.sh python playwright + run: .github/scripts/preflight-runner.sh python node playwright - name: Install Playwright browsers run: playwright install ${{ matrix.browser }} diff --git a/.github/workflows/codeql-branch-probe.yml b/.github/workflows/codeql-branch-probe.yml index 3d1a0124..98442c27 100644 --- a/.github/workflows/codeql-branch-probe.yml +++ b/.github/workflows/codeql-branch-probe.yml @@ -33,14 +33,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Initialize CodeQL - uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: languages: python queries: +security-extended config-file: ./.github/codeql/codeql-config.yml - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: category: "/language:python" diff --git a/.github/workflows/docker-release-benchmark.yml b/.github/workflows/docker-release-benchmark.yml index aaf91d68..25858732 100644 --- a/.github/workflows/docker-release-benchmark.yml +++ b/.github/workflows/docker-release-benchmark.yml @@ -353,7 +353,7 @@ jobs: < scripts/verify_container_security_runtime.py - name: Run Grype scan - uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 # v7.4.0 + uses: anchore/scan-action@27805bf3b4e84b4a5c980df22ed233c00390a439 # v7.4.2 with: image: ${{ env.LOCAL_IMAGE }} fail-build: true diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 20d0e1c3..db342658 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -5,8 +5,8 @@ # registries with Cosign, and verifies the published release. # # Security: All actions pinned to full SHA. Release tags or explicit manual -# dispatches are the only publishing triggers. Runnable tags are created only -# after the candidate image passes Grype and smoke validation. +# develop dispatches are the only publishing triggers. Development tags are +# promoted only after exact-commit validation and both registry signatures pass. name: Docker Release @@ -15,11 +15,6 @@ on: tags: - "v*" workflow_dispatch: - inputs: - tag_override: - description: "Custom image tag (default: edge)" - required: false - default: "edge" permissions: contents: read @@ -41,8 +36,12 @@ jobs: timeout-minutes: 10 permissions: contents: read + actions: read + checks: read outputs: image-tags: ${{ steps.meta.outputs.tags }} + candidate-tags: ${{ steps.release-metadata.outputs.candidate_tags }} + development-tag: ${{ steps.release-metadata.outputs.development_tag }} image-labels: ${{ steps.meta.outputs.labels }} image-annotations: ${{ steps.meta.outputs.annotations }} image-version: ${{ steps.version.outputs.version }} @@ -52,6 +51,17 @@ jobs: is-release: ${{ steps.resolve-tag.outputs.is_release }} is-prerelease: ${{ steps.resolve-tag.outputs.is_prerelease }} steps: + - name: Validate publishing trigger + run: | + if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] && [ "${GITHUB_REF}" = "refs/heads/develop" ]; then + exit 0 + fi + if [ "${GITHUB_EVENT_NAME}" = "push" ] && [[ "${GITHUB_REF}" == refs/tags/v* ]]; then + exit 0 + fi + echo "::error::Manual image publication is restricted to develop; releases require a version-tag push." + exit 1 + - name: Resolve correct SHA id: resolve-sha run: | @@ -69,6 +79,12 @@ jobs: fetch-depth: 0 fetch-tags: true + - name: Require validated development commit + if: github.event_name == 'workflow_dispatch' + env: + GH_TOKEN: ${{ github.token }} + run: python3 .github/scripts/validate-development-image.py + - name: Resolve tag from trigger id: resolve-tag run: | @@ -94,11 +110,23 @@ jobs: echo "::error::Unable to extract Pullbox version from src/pullbox/__init__.py" exit 1 fi + if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] && [[ "${VERSION}" != *-dev ]]; then + echo "::error::Development images require a -dev application version." + exit 1 + fi echo "version=${VERSION}" >> "$GITHUB_OUTPUT" - name: Capture release metadata id: release-metadata - run: echo "build_date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT" + run: | + echo "build_date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT" + echo "development_tag=sha-${GITHUB_SHA}-run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + { + echo "candidate_tags<> "$GITHUB_OUTPUT" - name: Docker metadata id: meta @@ -112,8 +140,9 @@ jobs: tags: | type=semver,pattern={{version}},value=${{ steps.resolve-tag.outputs.tag }},enable=${{ steps.resolve-tag.outputs.is_release }} type=raw,value=latest,enable=${{ steps.resolve-tag.outputs.is_release == 'true' && steps.resolve-tag.outputs.is_prerelease != 'true' }} - type=raw,value=${{ inputs.tag_override || 'edge' }},enable=${{ github.event_name == 'workflow_dispatch' }} - type=sha,format=short,prefix=sha- + type=raw,value=edge,enable=${{ github.event_name == 'workflow_dispatch' }} + type=raw,value=sha-${{ github.sha }}-run-${{ github.run_id }}-${{ github.run_attempt }},enable=${{ github.event_name == 'workflow_dispatch' }} + type=sha,format=short,prefix=sha-,enable=${{ github.event_name != 'workflow_dispatch' }} labels: | org.opencontainers.image.title=Pullbox org.opencontainers.image.description=Modern comic book management and acquisition platform @@ -139,6 +168,16 @@ jobs: outputs: digest: ${{ steps.build.outputs.digest }} steps: + - name: Validate development build attempt + if: github.event_name == 'workflow_dispatch' + env: + BUILD_TAG: ${{ needs.build.outputs.development-tag }} + run: | + if [ "${BUILD_TAG}" != "sha-${GITHUB_SHA}-run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" ]; then + echo "::error::Development reruns require Re-run all jobs so preparation uses this build attempt." + exit 1 + fi + - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -244,6 +283,16 @@ jobs: outputs: digest: ${{ steps.build.outputs.digest }} steps: + - name: Validate development build attempt + if: github.event_name == 'workflow_dispatch' + env: + BUILD_TAG: ${{ needs.build.outputs.development-tag }} + run: | + if [ "${BUILD_TAG}" != "sha-${GITHUB_SHA}-run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" ]; then + echo "::error::Development reruns require Re-run all jobs so preparation uses this build attempt." + exit 1 + fi + - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -397,10 +446,11 @@ jobs: < scripts/verify_container_security_runtime.py - name: Run Grype scan - uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 # v7.4.0 + uses: anchore/scan-action@27805bf3b4e84b4a5c980df22ed233c00390a439 # v7.4.2 id: scan with: image: ${{ env.LOCAL_IMAGE }} + grype-version: v0.110.0 fail-build: true severity-cutoff: high output-format: sarif @@ -409,7 +459,7 @@ jobs: - name: Upload SARIF to GitHub Security if: always() continue-on-error: true - uses: github/codeql-action/upload-sarif@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: sarif_file: ${{ steps.scan.outputs.sarif }} category: container-scan @@ -499,7 +549,7 @@ jobs: docker rm -f "${SMOKE_CONTAINER}" || true docker image rm "${LOCAL_IMAGE}" || true - # Assemble and tag the already-built platform digests only after validation. + # Development manifests receive staging tags only. Edge is promoted by sign. push: name: Publish Multi-Arch Manifests runs-on: ubuntu-latest @@ -557,7 +607,7 @@ jobs: - name: Publish GHCR multi-platform manifest env: - TAGS: ${{ needs.build.outputs.image-tags }} + TAGS: ${{ github.event_name == 'workflow_dispatch' && needs.build.outputs.candidate-tags || needs.build.outputs.image-tags }} working-directory: ${{ runner.temp }}/digests run: | sources=() @@ -587,7 +637,7 @@ jobs: - name: Publish Docker Hub multi-platform manifest env: - TAGS: ${{ needs.build.outputs.image-tags }} + TAGS: ${{ github.event_name == 'workflow_dispatch' && needs.build.outputs.candidate-tags || needs.build.outputs.image-tags }} working-directory: ${{ runner.temp }}/digests run: | sources=() @@ -614,7 +664,7 @@ jobs: - name: Validate pushed image metadata id: inspect env: - TAGS: ${{ needs.build.outputs.image-tags }} + TAGS: ${{ github.event_name == 'workflow_dispatch' && needs.build.outputs.candidate-tags || needs.build.outputs.image-tags }} run: | GHCR_TAG=$(printf '%s\n' "${TAGS}" | awk -v prefix="${GHCR_IMAGE}:" 'index($0, prefix) == 1 { print; exit }') if [ -z "${GHCR_TAG}" ]; then @@ -701,6 +751,8 @@ jobs: done - name: Summary + env: + TAGS: ${{ github.event_name == 'workflow_dispatch' && needs.build.outputs.candidate-tags || needs.build.outputs.image-tags }} run: | { echo "## Docker Images Published" @@ -709,7 +761,7 @@ jobs: echo "" echo "**Tags:**" echo '```' - echo "${{ needs.build.outputs.image-tags }}" + echo "${TAGS}" echo '```' } >> "$GITHUB_STEP_SUMMARY" @@ -721,9 +773,21 @@ jobs: timeout-minutes: 20 permissions: contents: read + actions: read + checks: read id-token: write packages: write steps: + - name: Checkout + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.build.outputs.build-sha }} + + - name: Set up Docker Buildx + if: github.event_name == 'workflow_dispatch' + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + - name: Log in to GHCR uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: @@ -764,7 +828,7 @@ jobs: - name: Verify published image signatures env: DIGEST: ${{ needs.push.outputs.image-digest }} - CERTIFICATE_IDENTITY_REGEXP: ^https://github.com/${{ github.repository }}/\.github/workflows/docker-release\.yml@refs/(tags/v.*|heads/.*)$ + CERTIFICATE_IDENTITY: https://github.com/${{ github.workflow_ref }} CERTIFICATE_OIDC_ISSUER: https://token.actions.githubusercontent.com run: | verify_image_signature() { @@ -775,7 +839,7 @@ jobs: for attempt in $(seq 1 "${max_attempts}"); do if cosign verify \ - --certificate-identity-regexp "${CERTIFICATE_IDENTITY_REGEXP}" \ + --certificate-identity "${CERTIFICATE_IDENTITY}" \ --certificate-oidc-issuer "${CERTIFICATE_OIDC_ISSUER}" \ "${image_ref}"; then return 0 @@ -792,6 +856,55 @@ jobs: verify_image_signature "${GHCR_IMAGE}@${DIGEST}" "GHCR" verify_image_signature "${DOCKERHUB_IMAGE}@${DIGEST}" "Docker Hub" + - name: Recheck development validation before promotion + if: github.event_name == 'workflow_dispatch' + env: + GH_TOKEN: ${{ github.token }} + run: python3 .github/scripts/validate-development-image.py + + - name: Promote verified development image + if: github.event_name == 'workflow_dispatch' + env: + DIGEST: ${{ needs.push.outputs.image-digest }} + TAGS: ${{ needs.build.outputs.image-tags }} + BUILD_TAG: ${{ needs.build.outputs.development-tag }} + run: | + if ! [[ "${DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::Cannot promote an invalid image digest." + exit 1 + fi + if [ "${BUILD_TAG}" != "sha-${GITHUB_SHA}-run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" ]; then + echo "::error::Development reruns require Re-run all jobs; retained preparation outputs cannot be promoted." + exit 1 + fi + for registry in "${GHCR_IMAGE}" "${DOCKERHUB_IMAGE}"; do + tag_args=() + while IFS= read -r tag; do + if [[ "${tag}" == "${registry}:"* ]]; then + suffix=${tag#${registry}:} + if [ "${suffix}" != "edge" ] && [ "${suffix}" != "${BUILD_TAG}" ]; then + echo "::error::Unexpected development image tag." + exit 1 + fi + tag_args+=(--tag "${tag}") + fi + done <<< "${TAGS}" + if [ "${#tag_args[@]}" -ne 4 ]; then + echo "::error::Expected edge and a run-specific SHA tag for each registry." + exit 1 + fi + # A single index source with no annotations is a carbon copy: its + # digest, SBOM/provenance, and the verified signature remain intact. + docker buildx imagetools create "${tag_args[@]}" "${registry}@${DIGEST}" + for tag in "${registry}:edge" "${registry}:${BUILD_TAG}"; do + published=$(docker buildx imagetools inspect "${tag}" --format '{{json .Manifest}}' | python3 -c 'import json,sys; print(json.load(sys.stdin)["digest"])') + if [ "${published}" != "${DIGEST}" ]; then + echo "::error::Promoted development tag does not match the signed digest." + exit 1 + fi + done + done + - name: Write release image digest artifact env: DIGEST: ${{ needs.push.outputs.image-digest }} @@ -818,11 +931,20 @@ jobs: retention-days: 30 - name: Summary + env: + TAGS: ${{ needs.build.outputs.image-tags }} + BUILD_SHA: ${{ needs.build.outputs.build-sha }} run: | { echo "## Docker Images Signed" echo "" echo "**Digest:** \`${{ needs.push.outputs.image-digest }}\`" + echo "**Source commit:** \`${BUILD_SHA}\`" echo "" echo "**Signing:** Keyless Sigstore/Cosign signatures verified for GHCR and Docker Hub." + echo "" + echo "**Published tags:**" + echo '```' + echo "${TAGS}" + echo '```' } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/docker-validate.yml b/.github/workflows/docker-validate.yml index 11e07730..7b1c2ee4 100644 --- a/.github/workflows/docker-validate.yml +++ b/.github/workflows/docker-validate.yml @@ -168,9 +168,10 @@ jobs: < scripts/verify_container_security_runtime.py - name: Run Grype scan - uses: anchore/scan-action@e1165082ffb1fe366ebaf02d8526e7c4989ea9d2 # v7.4.0 + uses: anchore/scan-action@27805bf3b4e84b4a5c980df22ed233c00390a439 # v7.4.2 with: image: pullbox:validate + grype-version: v0.110.0 fail-build: true severity-cutoff: high output-format: table diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index f3b11955..e7f5e692 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -8,7 +8,7 @@ # See: docs/development/INFRASTRUCTURE.md # # Tools: -# pip-audit → OSV database (Google-backed) +# pip-audit → PyPI advisory service (shared blocking exception policy) # safety → Safety DB (secondary, different vulnerability coverage) # gitleaks → blocking secret scan # bandit → advisory static analysis during initial rollout @@ -178,7 +178,7 @@ jobs: retention-days: 14 # ────────────────────────────────────────────── - # Job 2: pip-audit (OSV database) + # Job 2: pip-audit (PyPI advisory service) # ────────────────────────────────────────────── dependency-audit: name: pip-audit @@ -204,11 +204,18 @@ jobs: - name: Run pip-audit run: | # Generate requirements from installed packages, excluding editable pullbox - pip freeze --exclude-editable | grep -v "^pullbox==" > /tmp/requirements-audit.txt - # Transitive dep CVE suppressions (not used by pullbox directly): - # - CVE-2026-4539: pygments regex DoS (from bandit/rich, no fix available yet) - pip-audit --strict --desc on -r /tmp/requirements-audit.txt \ - --ignore-vuln CVE-2026-4539 + requirements_file="$(mktemp /tmp/pullbox-requirements-audit.XXXXXX.txt)" + trap 'rm -f "$requirements_file"' EXIT + python -m pip freeze --exclude-editable | grep -v "^pullbox==" > "$requirements_file" + python scripts/run_dependency_audit.py -r "$requirements_file" + + - name: Upload dependency audit report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dependency-audit-report + path: dependency-audit-report.json + retention-days: 14 # ────────────────────────────────────────────── # Job 3: safety (Safety DB — secondary scanner) @@ -310,14 +317,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Initialize CodeQL - uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: languages: python queries: +security-extended config-file: ./.github/codeql/codeql-config.yml - name: Perform CodeQL analysis - uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: category: "/language:python" diff --git a/.gitignore b/.gitignore index 50bc357f..ab7dc7ba 100644 --- a/.gitignore +++ b/.gitignore @@ -113,7 +113,9 @@ node_modules/ # Playwright test artifacts test-results/ .playwright-cli/ +output/playwright/ # Local security artifacts bandit-report.json safety-report.json +dependency-audit-report.json diff --git a/.gitleaksignore b/.gitleaksignore index ca27be46..0d9bbeb4 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -3,3 +3,11 @@ tests/api/test_comicvine_key.py:generic-api-key:253 tests/integration/test_security_endpoints.py:generic-api-key:206 tests/unit/test_secret_validation.py:generic-api-key:103 tests/fixtures/prowlarr_test_fixtures.json:square-access-token:206168 +tests/unit/test_import_story_arc_staging.py:generic-api-key:456 +tests/unit/test_import_story_arc_staging.py:generic-api-key:468 +tests/unit/test_import_story_arc_staging.py:generic-api-key:480 +# Historical synthetic folder cohort IDs, not credentials. Keep commit/path/line scope. +ecd22c27eb7607e072ddb98d921738fb213896e4:tests/unit/test_import_scan_pipeline.py:generic-api-key:502 +ecd22c27eb7607e072ddb98d921738fb213896e4:tests/unit/test_import_story_arc_staging.py:generic-api-key:443 +ecd22c27eb7607e072ddb98d921738fb213896e4:tests/unit/test_import_story_arc_staging.py:generic-api-key:455 +ecd22c27eb7607e072ddb98d921738fb213896e4:tests/unit/test_import_story_arc_staging.py:generic-api-key:467 diff --git a/.grype.yaml b/.grype.yaml index 3127d1fd..0ef4aabf 100644 --- a/.grype.yaml +++ b/.grype.yaml @@ -13,6 +13,104 @@ # stable Python 3.14 or Debian 13 packages become available upstream. ignore: + # Renewed by Adam Hernandez on 2026-09-14 only for this DHI libc6 revision. + # Accepted risk, NOT a fix: Debian 13 still lists this TSIG-printing issue + # as vulnerable/no-DSA. Direct Pullbox callers were not found, but dependency + # reachability and DHI backport status remain unproven. Re-review by + # 2026-09-30 or the next base refresh, whichever comes first; remove when + # a fixed stable package ships. All other High findings remain blocking. + # https://security-tracker.debian.org/tracker/CVE-2026-5435 + - vulnerability: CVE-2026-5435 + package: + name: libc6 + version: 2.41-12+deb13u4+dhi0 + type: deb + + # Approved by Adam Hernandez on 2026-09-15 for this exact runtime revision. + # Accepted risk, NOT a fix: strfmon/strfmon_l right-padding buffer overflow. + # Debian 13 lists this as vulnerable/no-DSA with no fixed stable package. + # No direct Pullbox callers were found; dependency reachability remains unproven. + # Re-review by 2026-09-30 or the next base refresh, whichever comes first; + # remove when a fixed stable package ships. All other High findings block. + # https://security-tracker.debian.org/tracker/CVE-2026-19499 + - vulnerability: CVE-2026-19499 + package: + name: libc6 + version: 2.41-12+deb13u4+dhi0 + type: deb + + # Renewed by Adam Hernandez on 2026-09-13 for eight exact findings in the + # refreshed DHI runtime. These are accepted-risk exceptions, NOT fixes. + # Keep all other High findings blocking, including other package versions. + # Preserve the previous entries for cached images and the review dates below. + # + # Debian 13 still lists CVE-2026-5435 as vulnerable/no-DSA (TSIG printing). + # No direct Pullbox calls were found, but third-party reachability and DHI + # backport status remain unproven. Re-review by 2026-09-30 or the next base + # refresh, whichever comes first; remove when a fixed stable package ships. + # https://security-tracker.debian.org/tracker/CVE-2026-5435 + - vulnerability: CVE-2026-5435 + package: + name: libc6 + version: 2.41-12+deb13u4 + type: deb + + # Debian 13 still has no fixed stable Expat package for these findings. + # The libexpat1 copy supports Fontconfig/Poppler. The separate Python parser + # reports Expat 2.8.2: its >=2.8.1 runtime guard does NOT establish protection + # against these newer advisories. Do not interpret this renewal as proving + # either parser safe. No Python-binary or libexpat1-dev exception is added. + # Re-review by 2026-10-07 or the next base refresh, whichever comes first; + # remove as soon as a fixed stable package is available. + # https://security-tracker.debian.org/tracker/CVE-2026-66046 + # https://security-tracker.debian.org/tracker/CVE-2026-76956 + # https://security-tracker.debian.org/tracker/CVE-2026-76957 + - vulnerability: CVE-2026-66046 + package: + name: libexpat1 + version: 2.8.3-1~deb13u1+dhi3 + type: deb + - vulnerability: CVE-2026-76956 + package: + name: libexpat1 + version: 2.8.3-1~deb13u1+dhi3 + type: deb + - vulnerability: CVE-2026-76957 + package: + name: libexpat1 + version: 2.8.3-1~deb13u1+dhi3 + type: deb + + # These util-linux source-package findings attach to libuuid1, but concern + # privileged mount/nsenter helpers. The inspected ARM64 production image + # contains libuuid1 and neither executable. Other architectures still need + # their normal image validation. Re-review by 2026-10-04 or the next base + # refresh, whichever comes first; remove when fixed stable packages ship. + # https://security-tracker.debian.org/tracker/CVE-2026-76642 + # https://security-tracker.debian.org/tracker/CVE-2026-78408 + # https://security-tracker.debian.org/tracker/CVE-2026-78409 + # https://security-tracker.debian.org/tracker/CVE-2026-78410 + - vulnerability: CVE-2026-76642 + package: + name: libuuid1 + version: 2.41.5-0+deb13u1+dhi3 + type: deb + - vulnerability: CVE-2026-78408 + package: + name: libuuid1 + version: 2.41.5-0+deb13u1+dhi3 + type: deb + - vulnerability: CVE-2026-78409 + package: + name: libuuid1 + version: 2.41.5-0+deb13u1+dhi3 + type: deb + - vulnerability: CVE-2026-78410 + package: + name: libuuid1 + version: 2.41.5-0+deb13u1+dhi3 + type: deb + # Debian Trixie has postponed the OpenSSL 3.5 QUIC listener connection-limit # fix for CVE-2026-14456. Pullbox does not expose an OpenSSL QUIC listener; # retain this exact DHI base-image exception only until Debian ships a fix. @@ -110,6 +208,24 @@ ignore: name: libexpat1-dev version: 2.8.3-1~deb13u1+dhi2 type: deb + + # Expat 2.8.4 fixes CVE-2026-76956 and CVE-2026-76957, but Debian 13 + # currently classifies both as no-DSA minor issues and has no fixed stable + # package. This shared library is copied only for Fontconfig/Poppler; Pullbox + # XML parsing uses Python's separately bundled parser, and pyexpat does not + # expose the custom encoding callback required by CVE-2026-76957. Keep these + # exceptions exact and temporary. Re-review by 2026-10-07 or on the next DHI + # Python refresh, whichever comes first. + - vulnerability: CVE-2026-76956 + package: + name: libexpat1 + version: 2.8.3-1~deb13u1+dhi2 + type: deb + - vulnerability: CVE-2026-76957 + package: + name: libexpat1 + version: 2.8.3-1~deb13u1+dhi2 + type: deb - vulnerability: CVE-2025-59375 package: name: libexpat1 @@ -239,6 +355,63 @@ ignore: version: 2.41-12+deb13u3+dhi1 type: deb + # Accepted risk approved by Adam Hernandez on 2026-08-30 for the DHI +dhi2 + # refresh in PR #133; this is NOT a vulnerability fix or a false-positive claim. + # Debian 13 still lists these as vulnerable with no-dsa/minor-issue status: + # https://security-tracker.debian.org/tracker/CVE-2026-5435 (DNS TSIG printing) + # https://security-tracker.debian.org/tracker/CVE-2026-5450 (scanf %mc width >1024) + # https://security-tracker.debian.org/tracker/CVE-2026-5928 (ungetwc pushback) + # No direct calls were found in Pullbox source/native integration code, but + # reachability through bundled third-party helpers and DHI backport status are + # unproven. Keep the High gate blocking and scope only these nine matches. + # Retain +dhi1 entries above for older/cached image variants. Re-review at the + # next base refresh or by 2026-09-30; remove when fixed Debian 13 packages ship. + - vulnerability: CVE-2026-5435 + package: + name: libc6 + version: 2.41-12+deb13u3+dhi2 + type: deb + - vulnerability: CVE-2026-5435 + package: + name: libc-dev-bin + version: 2.41-12+deb13u3+dhi2 + type: deb + - vulnerability: CVE-2026-5435 + package: + name: libc6-dev + version: 2.41-12+deb13u3+dhi2 + type: deb + - vulnerability: CVE-2026-5450 + package: + name: libc6 + version: 2.41-12+deb13u3+dhi2 + type: deb + - vulnerability: CVE-2026-5450 + package: + name: libc-dev-bin + version: 2.41-12+deb13u3+dhi2 + type: deb + - vulnerability: CVE-2026-5450 + package: + name: libc6-dev + version: 2.41-12+deb13u3+dhi2 + type: deb + - vulnerability: CVE-2026-5928 + package: + name: libc6 + version: 2.41-12+deb13u3+dhi2 + type: deb + - vulnerability: CVE-2026-5928 + package: + name: libc-dev-bin + version: 2.41-12+deb13u3+dhi2 + type: deb + - vulnerability: CVE-2026-5928 + package: + name: libc6-dev + version: 2.41-12+deb13u3+dhi2 + type: deb + - vulnerability: CVE-2021-45346 package: name: libsqlite3-0 @@ -284,6 +457,33 @@ ignore: version: 2.41-5+dhi3 type: deb + # Grype maps these util-linux source-package findings to libuuid1 in the + # current DHI. Their affected code is in privileged mount/nsenter commands, + # which are absent from the Pullbox runtime image. Debian 13 marks all four + # no-dsa/minor and has not published a stable fix. Keep the High gate active; + # retain only these exact package matches. Re-review by 2026-10-04 or at the + # next DHI refresh, whichever comes first. + - vulnerability: CVE-2026-76642 + package: + name: libuuid1 + version: 2.41.5-0+deb13u1+dhi2 + type: deb + - vulnerability: CVE-2026-78408 + package: + name: libuuid1 + version: 2.41.5-0+deb13u1+dhi2 + type: deb + - vulnerability: CVE-2026-78409 + package: + name: libuuid1 + version: 2.41.5-0+deb13u1+dhi2 + type: deb + - vulnerability: CVE-2026-78410 + package: + name: libuuid1 + version: 2.41.5-0+deb13u1+dhi2 + type: deb + - vulnerability: CVE-2025-6141 package: name: libncursesw6 @@ -330,6 +530,23 @@ ignore: name: zlib1g version: 1:1.3.dfsg+really1.3.1-1+dhi2 type: deb + + # Debian 13 has no fixed zlib package for CVE-2026-85091. The affected path + # requires the non-blocking gzwrite API followed by gzprintf/gzvprintf; + # Pullbox does not call that native sequence. The duplicated development + # package metadata comes from the DHI Python build/runtime dependency tree. + # Keep this exact exception only. Re-review by 2026-10-04 or at the next DHI + # refresh, whichever comes first. + - vulnerability: CVE-2026-85091 + package: + name: zlib1g + version: 1:1.3.dfsg+really1.3.1-1+dhi3 + type: deb + - vulnerability: CVE-2026-85091 + package: + name: zlib1g-dev + version: 1:1.3.dfsg+really1.3.1-1+dhi3 + type: deb - vulnerability: CVE-2026-34743 package: name: liblzma5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 12210416..b6e35c98 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: - id: pytest-fast name: fast unit tests - entry: pytest tests/unit/ -x -q -m "not slow" --no-header + entry: scripts/pre_commit_pytest.sh language: system types: [python] pass_filenames: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 962846dd..4390a1b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,160 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.3.0] - 2026-09-15 + +Minor release adding Story Arc management, an optional local Comic Vine catalog, +multi-root and in-place library imports, and substantial large-library import +performance and recovery improvements. + +### Added + +- Added an optional local Comic Vine catalog for series searches and import + matching without live metadata requests. Download it from Metadata settings + with progress, resumable transfers, signed verification, and daily updates. + Failed updates preserve the installed catalog; full metadata enrichment still + uses the user's Comic Vine API key. +- Added import source-layout previews for series folders, publisher/series + folders, and custom folder and issue naming patterns. +- Added independent options to keep existing files in place and use an approved + layout for future managed files without reorganizing an existing library. +- Added first-class Story Arcs with ordered memberships, local issue selection, + monitoring, Mylar Story Arc import, and folder-based arc evidence. +- Added optional separate Story Arc copies or links, reading-order filename + prefixes, and safe previewed reordering while preserving canonical files and + user-owned arc references. +- Added category-scoped bulk import safety review with bounded previews and + audit evidence. +- Added a dedicated import Follow-up workspace for remaining matching, recovery, + and cleanup decisions without restarting a completed import. +- Added previewed clean-library builds with background progress, configurable + naming and conversion, and explicit safeguards for source files and destinations. +- Added safe removal of disabled library roots with dependency checks before + deleting their configuration. +- Added Comic Vine Story Arc discovery, reviewed member ordering, canonical + series/issue reuse, provider-change review, and arc-scoped missing-issue search. +- Added Mylar in-place adoption and original-filename arc copies with optional + leading reading-order numbers. Initial copies can run independently of future + synchronization and expose resumable progress and explicit retries. +- Added a guarded, manually published signed development-image channel for + isolated testing without a general-availability release. + +### Changed + +- Grouped Mylar library-access problems by their underlying root and offered + explicit resolution, skip, or detailed repair actions instead of presenting + every affected series as a separate configuration problem. +- Unified Comic Vine series and Story Arc discovery, added a selectable header + Add action, and simplified Story Arc reading-order review and reordering. +- Centralized library naming and Story Arc file policies, including independent + permissions for referenced existing files and managed library destinations. + +### Fixed + +- Improved completed-import recovery for trusted legacy identities, misplaced + files, and deferred outcomes. Bounded retries preserve successful imports, + revalidate changed sources and approved roots, and resume interrupted work + without losing matching evidence or repeating completed actions. +- Reconciled stale Mylar issue identifiers, renamed files, and unqualified volume + identities when independent evidence proves the target. Mixed folders no longer + require treating every file as belonging to the folder's series. +- Restored actionable recovery controls on completed jobs and kept safety-approved + items, archived history, background activity, and import counters consistent. +- Preserved file ownership and dependent records during in-place adoption and + clean-library execution, and corrected folder imports' reference-root setup. +- Sent torrent metadata through Pullbox to download clients rather than requiring + remote clients to fetch private Prowlarr or indexer download URLs themselves. +- Corrected reader progress display for large compendiums. +- Improved Mylar and folder scan throughput with resource-capped archive + inspection, batched review inserts, and fewer unnecessary duplicate checks. + Corrected current-item scan progress and unknown time estimates, and bounded + pending file-processing work without weakening cancellation or archive safety. +- Reconciled stale Mylar filenames with uniquely verified same-folder comic + files, including `#1` versus `001` naming changes, without weakening identity + or archive safety checks. Added a dry-run-first offline repair for saved + reviews that preserves matches and source files, and clarified missing-path + errors without claiming the source changed after scanning. +- Automatic searches now distinguish valid matches that could not be queued + from empty results. Search history counts only validation rejections, not + unused alternatives or failed downloads. +- Opted-in AirDC++ automatic searches now hand accepted matches to the durable + queue, with restart-safe intervention for lower-confidence matches and + duplicate-download and blocklist checks. +- SABnzbd NZB retrieval now allows slow indexer proxies a separate bounded + timeout without delaying normal client-control requests. +- Recognized unmarked four-digit issue numbers for known series such as 2000 AD + without treating numeric series names as issue numbers. Search confidence now + uses known issue publication/store dates instead of comparing every issue to + the series start year; undated continuing issues use a medium-confidence, + bounded year window while issue, series, and type checks remain required. +- Corrected Mylar `series.json` metadata wrappers and `cvinfo` volume URL + parsing in both Mylar and folder imports, without interpreting another + application's unqualified `series_id` as a ComicVine identity. +- Isolated metadata-only and possible cover-only comic archives in import + review while preserving valid series identities. Single-page approval is + explicit and does not bypass archive safety limits. +- Added an offline, dry-run-first `recheck-import` maintenance command for + affected saved reviews, preserving manual decisions and source files without + repeating the directory scan. +- Raised the database-size health warning to above 1 GiB and the critical + threshold to above 2 GiB to accommodate large collections and retained logs. + Disk-space, integrity, latency, and database-bloat checks remain unchanged. +- Preserved supported Unicode spaces, joiners, and direction marks in Mylar + source paths across preview, mapping, and import without renaming files or + relaxing root-containment protections. Invalid path text no longer reports + a misleading outside-root error. +- Kept Mylar scan counters, progress bars, and saved checkpoints in sync during + file checks and source-page preparation instead of waiting for the full scan. +- Reconciled unambiguous Mylar filename format, case, and spacing changes within + the same folder while retaining original-path evidence and leaving ambiguous + or unavailable files for review. Source files and the Mylar database stay untouched. +- Made Mylar path preflight show searchable, exportable exceptions with exact + paths and repair guidance, distinguishing missing folders from bad mappings. + Added explicit, revalidated continuation with available sources while retaining + unavailable records for review and keeping permission and safety failures blocked. +- Included recent Mylar preflight evidence in diagnostic packages even when an + import cannot start. +- Matched issue-only filenames using series-folder context and handled issue + titles embedded in filenames without treating the title as a new series. +- Preserved exact issue-number identity for large and special issue numbers + instead of allowing floating-point or scientific-notation drift. +- Preserved exact issue numbers in managed filenames, embedded ComicInfo, and + the reader, including suffixes and fractional padded filenames. +- Preserved other series' in-place files when deleting or trashing a shared + folder, without treating literal folder-name characters as SQL wildcards. +- Applied sensitive-directory restrictions to import previews and preserved + literal special characters when opening Mylar databases read-only. +- Preserved Mylar, ComicInfo, and supported series sidecar evidence through + import review and matching without silently overriding stronger identities. +- Rechecked arc search eligibility after provider waits so newly skipped or + removed members cannot be queued from stale search results. +- Fixed import-conflict review queries overflowing the parser stack on older + SQLite builds while retaining server-side sorting and bounded pagination. +- Preserved replaced or edited Story Arc files during failed-publication + cleanup instead of relying only on filesystem device and inode identifiers. + +### Performance + +- Prioritized visible catalog hydration after import while keeping ComicInfo + writes in separate background work. Improved restart recovery and progress + reporting, and prevented duplicate manual refreshes during initial metadata sync. +- Reduced large-library scan and review overhead with bounded worker budgets, + compact projections, and batched database work. Import, rollback, and hydration + progress now use clearer phase-aware completion and time estimates. +- Streamed complete import-review safety summaries using one narrow-field query + instead of repeatedly scanning and sorting blocked files for every batch. + Category counts, examples, and bulk-approval eligibility remain unchanged. +- Moved import archive safety checks off the application event loop and added + throttled progress and cancellation checkpoints between Mylar file checks. +- Removed recursive comic-library sizing from diagnostic generation and moved + blocking filesystem, database snapshot, and ZIP work off the application event + loop. Mylar preflight filesystem analysis also runs on a worker thread. +- Bounded Mylar staging, file matching, and conflict rebuilding so large jobs do + not retain complete ORM result sets or unbounded task queues. +- Moved filesystem inventory and source ordering to a private temporary SQLite + spool, with bounded active workers, archive tasks, progress delivery, and + cooperative cancellation cleanup. + ## [1.2.1] - 2026-08-28 Patch release hardening AirDC++ acquisition recovery so retries, cancellation, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ff8f1e24..64700224 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -161,6 +161,18 @@ DEV_DOCKER_PORT=8586 make dev-docker Local venv development is better for quick test runs and editor integration. +For a newly created Git worktree, prepare its isolated toolchain once: + +```bash +make bootstrap-worktree +``` + +The command is safe to rerun. It reuses the worktree's `.venv`, installs the +declared Python and locked Node development dependencies, ensures Chromium and +Firefox are available to Playwright, and creates `.env` only when it is +missing. It does not reinstall or repoint the Git hook shared by sibling +worktrees. + First-time setup: ```bash @@ -331,6 +343,7 @@ Most common commands: | --- | --- | | `make dev-docker` | One-command Docker development environment | | `make dev-local` | One-command local venv environment | +| `make bootstrap-worktree` | Prepare an isolated worktree toolchain without repointing shared hooks | | `make setup` | Create venv and install Python plus Node dependencies | | `make run` | Start local reload server | | `make css-build` | Compile committed Tailwind CSS | diff --git a/Makefile b/Makefile index 3c731eff..ae01363c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help setup dev dev-local dev-docker dev-docker-up dev-docker-down dev-docker-logs dev-docker-shell dev-docker-seed prod-test-pull prod-test-up prod-test-refresh prod-test-down prod-test-logs prod-test-shell run lint format format-fix typecheck test test-unit test-slow test-integration test-api test-providers test-utilities test-a11y test-e2e test-e2e-chrome test-e2e-firefox coverage coverage-check migrate migration seed seed-full reset-db reset-password reset-import import-fixture performance-baseline direct-download-baseline validate runner-preflight release-changelog-check workflow-hygiene secret-scan security-ci ci-local docker-build-check docker-smoke ci-full ci-clean-room security-check pre-commit css-build css-watch clean +.PHONY: help bootstrap-worktree setup dev dev-local dev-docker dev-docker-up dev-docker-down dev-docker-logs dev-docker-shell dev-docker-seed prod-test-pull prod-test-up prod-test-refresh prod-test-down prod-test-logs prod-test-shell run lint format format-fix typecheck test test-unit test-slow test-integration test-api test-providers test-utilities test-a11y test-e2e test-e2e-chrome test-e2e-firefox coverage coverage-check migrate migration seed seed-full reset-db reset-password reset-import import-fixture performance-baseline direct-download-baseline validate runner-preflight release-changelog-check workflow-hygiene secret-scan security-ci ci-local docker-build-check docker-security-check docker-smoke ci-full ci-clean-room security-check pre-commit css-build css-watch clean VENV := .venv PYTHON_BOOTSTRAP ?= python3 @@ -11,11 +11,15 @@ PULLBOX_IMAGE ?= ghcr.io/pullboxapp/pullbox:latest DEV_DOCKER_PORT ?= 8585 DEV_DOCKER_COMPOSE := PULLBOX_DEV_PORT=$(DEV_DOCKER_PORT) docker compose -f docker/docker-compose.dev.yml DEV_DOCKER_URL ?= http://127.0.0.1:$(DEV_DOCKER_PORT) +# Opt in to stopping without logs or cleanup so a failed smoke container can be reviewed. +DOCKER_SMOKE_KEEP_ON_FAILURE ?= 0 PERFORMANCE_BASELINE_URL ?= $(DEV_DOCKER_URL) PERFORMANCE_BASELINE_ARGS ?= TOOLS_DIR := .cache/tools ACTIONLINT := $(TOOLS_DIR)/actionlint GITLEAKS := $(TOOLS_DIR)/gitleaks +GRYPE := $(TOOLS_DIR)/grype +SECRET_SCAN_BASE ?= origin/develop help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | \ @@ -23,17 +27,26 @@ help: ## Show this help # ─── Setup ─────────────────────────────────────────────── -setup: ## Create venv, install Python + Node dependencies - $(PYTHON_BOOTSTRAP) -m venv $(VENV) +bootstrap-worktree: ## Idempotently prepare this worktree without repointing shared Git hooks + @if [ ! -x "$(PYTHON)" ]; then \ + $(PYTHON_BOOTSTRAP) -m venv $(VENV); \ + else \ + echo "\033[36mℹ️ Reusing $(CURDIR)/$(VENV).\033[0m"; \ + fi $(PIP) install --upgrade "pip>=26.0" wheel $(PIP) install -e ".[dev,e2e]" - $(VENV)/bin/pre-commit install - npm install + npm ci + $(PYTHON) -m playwright install chromium firefox @if [ ! -f .env ]; then \ cp .env.dev.example .env; \ echo "\033[33mℹ️ Created .env from .env.dev.example\033[0m"; \ fi @echo "" + @echo "\033[32m✅ Worktree ready.\033[0m Python, checks, and browser tests resolve locally." + +setup: bootstrap-worktree ## Create venv, install dependencies, and install the shared Git hook + $(VENV)/bin/pre-commit install + @echo "" @echo "\033[32m✅ Setup complete.\033[0m Run \033[36mmake dev-local\033[0m or \033[36mmake dev-docker\033[0m to start developing." dev: dev-local ## Friendly default contributor workflow @@ -204,10 +217,10 @@ workflow-hygiene: ## Run local workflow linting with pinned actionlint @bash scripts/install_ci_tool.sh actionlint "$(TOOLS_DIR)" $(ACTIONLINT) -shellcheck= -pyflakes= -secret-scan: ## Run blocking local gitleaks scan with the repo baseline +secret-scan: ## Scan current files and PR commit history (override SECRET_SCAN_BASE for other bases) @echo "\033[36m──── Secret Scan ────\033[0m" @bash scripts/install_ci_tool.sh gitleaks "$(TOOLS_DIR)" - $(GITLEAKS) dir . --no-banner --redact --timeout=300 + bash scripts/run_secret_scans.sh "$(GITLEAKS)" "$(SECRET_SCAN_BASE)" security-ci: ## Run the local security lane (pip-audit blocking; safety/bandit advisory) @echo "\033[36m──── Security Checks ────\033[0m" @@ -273,7 +286,14 @@ docker-build-check: ## Build the production Docker image and verify it can be in @echo "\033[36m──── Docker Inspect ────\033[0m" @docker image inspect pullbox:local >/dev/null -docker-smoke: docker-build-check ## Run Docker smoke tests against the locally built production image +docker-security-check: docker-build-check ## Verify runtime security and run the blocking Grype High gate + @echo "\033[36m──── Container Security Runtime ────\033[0m" + docker run --rm -i --entrypoint python pullbox:local - < scripts/verify_container_security_runtime.py + @echo "\033[36m──── Container Vulnerability Scan ────\033[0m" + @bash scripts/install_ci_tool.sh grype "$(TOOLS_DIR)" + $(GRYPE) docker:pullbox:local --config .grype.yaml --fail-on high --output table + +docker-smoke: docker-security-check ## Run Docker smoke tests after runtime security and Grype checks @echo "\033[36m──── Start Container ────\033[0m" @docker rm -f pullbox-smoke 2>/dev/null || true docker run -d --name pullbox-smoke -p 18585:8585 \ @@ -288,6 +308,10 @@ docker-smoke: docker-build-check ## Run Docker smoke tests against the locally b fi; \ if [ $$i -eq 30 ]; then \ echo "\033[31m ❌ Container failed to become healthy\033[0m"; \ + if [ "$(DOCKER_SMOKE_KEEP_ON_FAILURE)" = "1" ]; then \ + echo "Container pullbox-smoke preserved; stopped before logs or cleanup."; \ + exit 1; \ + fi; \ docker logs pullbox-smoke; \ docker rm -f pullbox-smoke; \ exit 1; \ @@ -295,8 +319,11 @@ docker-smoke: docker-build-check ## Run Docker smoke tests against the locally b sleep 1; \ done @echo "\033[36m──── Smoke Tests ────\033[0m" - PULLBOX_SMOKE_URL=http://localhost:18585 $(SECRET) $(VENV)/bin/pytest tests/e2e/test_smoke.py -v || \ - (docker logs pullbox-smoke; docker rm -f pullbox-smoke; exit 1) + PYTHONPATH=src PULLBOX_SMOKE_URL=http://localhost:18585 $(SECRET) $(VENV)/bin/pytest tests/e2e/test_smoke.py -v || \ + (if [ "$(DOCKER_SMOKE_KEEP_ON_FAILURE)" = "1" ]; then \ + echo "Container pullbox-smoke preserved; stopped before logs or cleanup."; \ + exit 1; \ + fi; docker logs pullbox-smoke; docker rm -f pullbox-smoke; exit 1) @echo "\033[36m──── Teardown ────\033[0m" @docker rm -f pullbox-smoke @echo "" diff --git a/README.md b/README.md index 3437081c..4ed8dfed 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ read-only or runtime-managed inside Pullbox. | `PULLBOX_BACKUP_DIR` | `/data/backups` | Database backup directory. | | `PULLBOX_BIND_ADDRESS` | `0.0.0.0` | Interface Pullbox binds inside the container. | | `PULLBOX_AIRDCPP_ENABLED` | `false` | Enables the experimental AirDC++ settings, search, queue, and import integration. | +| `PULLBOX_STORY_ARC_MANUAL_CREATE_ENABLED` | `false` | Enables manual creation of empty Story Arcs in the UI and API. Metadata-provider and import creation remain available when disabled. | | `PULLBOX_PORT` | `8585` | Internal listener port. If changed, update the container-side port mapping too. | | `PULLBOX_BASE_URL` | `http://localhost:8585` | Public URL used in generated app links and startup output. | | `PULLBOX_INSTANCE_NAME` | `Pullbox` | Display name for the instance. | @@ -267,8 +268,6 @@ live in the repo docs: - `docs/development/SECURITY_STANDARDS.md` - `docs/development/INFRASTRUCTURE.md` - `docs/development/DESIGN_SYSTEM.md` -- `docs/features/airdcpp.md` -- `docs/features/comic-reader.md` ## Security diff --git a/alembic/env.py b/alembic/env.py index fa533988..c9de7bb9 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -45,10 +45,26 @@ def run_migrations_offline() -> None: def do_run_migrations(connection: Connection) -> None: """Configure context and run migrations within a connection.""" + # SQLite table rebuilds must not cascade into existing child tables. Keep + # embedded Alembic callers consistent with the standalone migration process. + sqlite_fk_enabled = False + if connection.dialect.name == "sqlite": + sqlite_fk_enabled = bool(connection.exec_driver_sql("PRAGMA foreign_keys").scalar()) + connection.exec_driver_sql("PRAGMA foreign_keys=OFF") + connection.commit() context.configure(connection=connection, target_metadata=target_metadata) - with context.begin_transaction(): - context.run_migrations() + try: + with context.begin_transaction(): + context.run_migrations() + except BaseException: + connection.rollback() + raise + else: + connection.commit() + finally: + if sqlite_fk_enabled: + connection.exec_driver_sql("PRAGMA foreign_keys=ON") async def run_async_migrations() -> None: diff --git a/alembic/versions/a2b3c4d5e678_expand_story_arc_domain.py b/alembic/versions/a2b3c4d5e678_expand_story_arc_domain.py new file mode 100644 index 00000000..82bb931e --- /dev/null +++ b/alembic/versions/a2b3c4d5e678_expand_story_arc_domain.py @@ -0,0 +1,964 @@ +"""Expand story arcs into an import-safe first-class domain. + +Revision ID: a2b3c4d5e678 +Revises: z1a2b3c4d567 +Create Date: 2026-08-30 +""" + +from __future__ import annotations + +import unicodedata +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "a2b3c4d5e678" +down_revision: str | Sequence[str] | None = "z1a2b3c4d567" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_BATCH_SIZE = 2_000 +_SOURCE_KINDS = ("legacy", "pullbox", "mylar3", "folder", "comicinfo", "provider") +_LIFECYCLES = ("active", "archived") +_RESOLUTION_STATES = ("pending", "resolved", "missing", "ambiguous", "conflict", "skipped") +_IMPORTED_STATUSES = ( + "detected", + "needs_review", + "ready", + "confirmed", + "skipped", + "imported", + "failed", +) +_MATERIALIZATION_MODES = ("copy", "hardlink", "symlink", "reference_only") +_PLACEMENT_OWNERS = ("managed", "referenced") +_SYMLINK_STYLES = ("absolute", "relative") +_PLACEMENT_STATES = ("current", "missing", "drifted", "failed") + + +def _enum(values: tuple[str, ...], *, name: str) -> sa.Enum: + return sa.Enum( + *values, + name=name, + native_enum=False, + create_constraint=True, + validate_strings=True, + ) + + +def _is_sqlite() -> bool: + return op.get_bind().dialect.name == "sqlite" + + +def _add_story_arc_enum_columns() -> None: + if _is_sqlite(): + op.execute( + sa.text( + "ALTER TABLE story_arcs ADD COLUMN source_kind VARCHAR(9) " + "NOT NULL DEFAULT 'legacy' CONSTRAINT storyarcsourcekind " + "CHECK (source_kind IN " + "('legacy','pullbox','mylar3','folder','comicinfo','provider'))" + ) + ) + op.execute( + sa.text( + "ALTER TABLE story_arcs ADD COLUMN lifecycle VARCHAR(8) " + "NOT NULL DEFAULT 'active' CONSTRAINT storyarclifecycle " + "CHECK (lifecycle IN ('active','archived'))" + ) + ) + return + + op.add_column( + "story_arcs", + sa.Column("source_kind", sa.String(9), nullable=False, server_default="legacy"), + ) + op.create_check_constraint( + "storyarcsourcekind", + "story_arcs", + "source_kind IN ('legacy','pullbox','mylar3','folder','comicinfo','provider')", + ) + op.add_column( + "story_arcs", + sa.Column("lifecycle", sa.String(8), nullable=False, server_default="active"), + ) + op.create_check_constraint( + "storyarclifecycle", + "story_arcs", + "lifecycle IN ('active','archived')", + ) + + +def _add_story_arc_foreign_key_columns() -> None: + if _is_sqlite(): + op.execute( + sa.text( + "ALTER TABLE story_arcs ADD COLUMN target_library_root_id INTEGER " + "CONSTRAINT fk_story_arcs_target_library_root_id_library_roots " + "REFERENCES library_roots(id) ON DELETE SET NULL" + ) + ) + op.execute( + sa.text( + "ALTER TABLE story_arcs ADD COLUMN source_import_job_id INTEGER " + "CONSTRAINT fk_story_arcs_source_import_job_id_import_jobs " + "REFERENCES import_jobs(id) ON DELETE SET NULL" + ) + ) + return + + op.add_column( + "story_arcs", + sa.Column("target_library_root_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_story_arcs_target_library_root_id_library_roots", + "story_arcs", + "library_roots", + ["target_library_root_id"], + ["id"], + ondelete="SET NULL", + ) + op.add_column( + "story_arcs", + sa.Column("source_import_job_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_story_arcs_source_import_job_id_import_jobs", + "story_arcs", + "import_jobs", + ["source_import_job_id"], + ["id"], + ondelete="SET NULL", + ) + + +def _normalize_story_arc_name(value: object) -> str: + normalized = unicodedata.normalize("NFC", str(value)).casefold() + return " ".join(normalized.split()) + + +def _backfill_normalized_story_arc_names() -> None: + connection = op.get_bind() + last_id = 0 + select_batch = sa.text( + "SELECT id, name FROM story_arcs WHERE id > :last_id ORDER BY id LIMIT :batch_size" + ) + update_row = sa.text( + "UPDATE story_arcs SET normalized_name = :normalized_name WHERE id = :story_arc_id" + ) + + while True: + rows = ( + connection.execute( + select_batch, + {"last_id": last_id, "batch_size": _BATCH_SIZE}, + ) + .mappings() + .all() + ) + if not rows: + return + connection.execute( + update_row, + [ + { + "story_arc_id": int(row["id"]), + "normalized_name": _normalize_story_arc_name(row["name"]), + } + for row in rows + ], + ) + last_id = int(rows[-1]["id"]) + + +def _add_story_arc_columns() -> None: + op.add_column( + "story_arcs", + sa.Column( + "normalized_name", + sa.String(500), + nullable=False, + server_default="__legacy__", + ), + ) + _add_story_arc_enum_columns() + for column_name in ("monitored", "search_missing", "include_upcoming", "sync_enabled"): + op.add_column( + "story_arcs", + sa.Column( + column_name, + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + _add_story_arc_foreign_key_columns() + op.add_column( + "story_arcs", + sa.Column("policy_schema_version", sa.Integer(), nullable=True), + ) + op.add_column( + "story_arcs", + sa.Column("policy_snapshot", sa.JSON(), nullable=False, server_default="{}"), + ) + op.add_column( + "story_arcs", + sa.Column("revision", sa.Integer(), nullable=False, server_default="1"), + ) + op.add_column( + "story_arcs", + sa.Column("diagnostics", sa.JSON(), nullable=False, server_default="{}"), + ) + _backfill_normalized_story_arc_names() + + op.create_index( + "ix_story_arcs_normalized_id", + "story_arcs", + ["normalized_name", "id"], + unique=False, + ) + op.create_index( + "ix_story_arcs_lifecycle_monitored_id", + "story_arcs", + ["lifecycle", "monitored", "id"], + unique=False, + ) + op.create_index( + "ix_story_arcs_source_job_id", + "story_arcs", + ["source_import_job_id", "id"], + unique=False, + ) + + +def _create_issue_story_arcs() -> None: + op.create_table( + "issue_story_arcs", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("story_arc_id", sa.Integer(), nullable=False), + sa.Column("issue_id", sa.Integer(), nullable=True), + sa.Column("sequence_number", sa.Integer(), nullable=False), + sa.Column("source_ordinal", sa.Integer(), nullable=False), + sa.Column( + "legacy_sequence_was_null", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.Column( + "resolution_state", + _enum(_RESOLUTION_STATES, name="storyarcresolutionstate"), + nullable=False, + server_default="pending", + ), + sa.Column( + "source_kind", + _enum(_SOURCE_KINDS, name="storyarcsourcekind"), + nullable=False, + server_default="legacy", + ), + sa.Column("source_entry_id", sa.String(255), nullable=True), + sa.Column("source_arc_id", sa.String(255), nullable=True), + sa.Column("source_issue_id", sa.String(255), nullable=True), + sa.Column("source_series_id", sa.String(255), nullable=True), + sa.Column("source_issue_number_text", sa.String(320), nullable=True), + sa.Column("source_series_name", sa.String(500), nullable=True), + sa.Column("source_issue_title", sa.String(500), nullable=True), + sa.Column("source_publisher", sa.String(255), nullable=True), + sa.Column("source_release_date_text", sa.String(50), nullable=True), + sa.Column("source_issue_date_text", sa.String(50), nullable=True), + sa.Column("resolution_confidence", sa.Float(), nullable=True), + sa.Column("resolution_method", sa.String(50), nullable=True), + sa.Column("evidence", sa.JSON(), nullable=False, server_default="{}"), + sa.Column( + "sync_eligible", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.Column( + "last_materialization_result", + sa.JSON(), + nullable=False, + server_default="{}", + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["story_arc_id"], + ["story_arcs.id"], + name="fk_issue_story_arcs_story_arc_id_story_arcs", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["issue_id"], + ["issues.id"], + name="fk_issue_story_arcs_issue_id_issues", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_issue_story_arcs_v2"), + sa.UniqueConstraint( + "story_arc_id", + "issue_id", + name="uq_issue_story_arcs_arc_issue", + ), + ) + op.create_index( + "ix_issue_story_arcs_order", + "issue_story_arcs", + ["story_arc_id", "sequence_number", "source_ordinal", "id"], + unique=False, + ) + op.create_index( + "ix_issue_story_arcs_review", + "issue_story_arcs", + ["story_arc_id", "resolution_state", "sequence_number", "source_ordinal", "id"], + unique=False, + ) + op.create_index( + "ix_issue_story_arcs_issue", + "issue_story_arcs", + ["issue_id", "story_arc_id", "id"], + unique=False, + ) + + +def _rebuild_issue_story_arcs_for_upgrade() -> None: + op.rename_table("issue_story_arcs", "issue_story_arcs_legacy") + _create_issue_story_arcs() + + connection = op.get_bind() + connection.execute( + sa.text( + "WITH arc_max AS (" + "SELECT story_arc_id, COALESCE(MAX(sequence_number), 0) AS max_sequence " + "FROM issue_story_arcs_legacy GROUP BY story_arc_id" + "), assigned AS (" + "SELECT legacy.story_arc_id, legacy.issue_id, legacy.sequence_number, " + "CASE WHEN legacy.sequence_number IS NULL THEN " + "arc_max.max_sequence + ROW_NUMBER() OVER (" + "PARTITION BY legacy.story_arc_id, " + "CASE WHEN legacy.sequence_number IS NULL THEN 1 ELSE 0 END " + "ORDER BY legacy.issue_id) ELSE legacy.sequence_number END AS assigned_sequence " + "FROM issue_story_arcs_legacy AS legacy " + "JOIN arc_max ON arc_max.story_arc_id = legacy.story_arc_id" + "), ranked AS (" + "SELECT assigned.*, ROW_NUMBER() OVER (" + "PARTITION BY assigned.story_arc_id " + "ORDER BY assigned.assigned_sequence, assigned.issue_id) AS assigned_ordinal " + "FROM assigned" + ") " + "INSERT INTO issue_story_arcs (" + "story_arc_id, issue_id, sequence_number, source_ordinal, " + "legacy_sequence_was_null, resolution_state, source_kind, " + "source_issue_number_text, evidence, sync_eligible, " + "last_materialization_result, created_at, updated_at) " + "SELECT ranked.story_arc_id, ranked.issue_id, ranked.assigned_sequence, " + "ranked.assigned_ordinal, " + "CASE WHEN ranked.sequence_number IS NULL THEN TRUE ELSE FALSE END, " + "'resolved', 'legacy', " + "issues.issue_number_text, '{}', FALSE, '{}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP " + "FROM ranked JOIN issues ON issues.id = ranked.issue_id " + "ORDER BY ranked.story_arc_id, ranked.assigned_ordinal" + ) + ) + op.drop_table("issue_story_arcs_legacy") + + +def _create_story_arc_external_identities() -> None: + op.create_table( + "story_arc_external_identities", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("story_arc_id", sa.Integer(), nullable=False), + sa.Column("source", sa.String(50), nullable=False), + sa.Column("namespace", sa.String(100), nullable=False), + sa.Column("external_id", sa.String(255), nullable=False), + sa.Column("source_url", sa.String(1000), nullable=True), + sa.Column("evidence", sa.JSON(), nullable=False, server_default="{}"), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["story_arc_id"], + ["story_arcs.id"], + name="fk_story_arc_external_identities_story_arc_id_story_arcs", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_story_arc_external_identities"), + sa.UniqueConstraint( + "source", + "namespace", + "external_id", + name="uq_story_arc_external_identity", + ), + ) + op.create_index( + "ix_story_arc_external_identities_arc_id", + "story_arc_external_identities", + ["story_arc_id", "id"], + unique=False, + ) + op.get_bind().execute( + sa.text( + "INSERT INTO story_arc_external_identities " + "(story_arc_id, source, namespace, external_id, evidence, created_at, updated_at) " + "SELECT id, 'comicvine', 'story_arc', CAST(comicvine_id AS VARCHAR(255)), " + "'{}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM story_arcs " + "WHERE comicvine_id IS NOT NULL" + ) + ) + + +def _create_import_story_arc_tables() -> None: + op.create_table( + "import_story_arcs", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("import_job_id", sa.Integer(), nullable=False), + sa.Column( + "source_kind", + _enum(_SOURCE_KINDS, name="storyarcsourcekind"), + nullable=False, + ), + sa.Column("source_key", sa.String(255), nullable=False), + sa.Column("source_arc_id", sa.String(255), nullable=True), + sa.Column("source_ordinal", sa.Integer(), nullable=False), + sa.Column("name", sa.String(500), nullable=True), + sa.Column("normalized_name", sa.String(500), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column( + "status", + _enum(_IMPORTED_STATUSES, name="importedstoryarcstatus"), + nullable=False, + server_default="detected", + ), + sa.Column( + "selected_for_import", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.Column("proposed_story_arc_id", sa.Integer(), nullable=True), + sa.Column("materialized_story_arc_id", sa.Integer(), nullable=True), + sa.Column( + "proposed_policy_snapshot", + sa.JSON(), + nullable=False, + server_default="{}", + ), + sa.Column( + "source_settings_snapshot", + sa.JSON(), + nullable=False, + server_default="{}", + ), + sa.Column("diagnostics", sa.JSON(), nullable=False, server_default="{}"), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["import_job_id"], + ["import_jobs.id"], + name="fk_import_story_arcs_import_job_id_import_jobs", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["proposed_story_arc_id"], + ["story_arcs.id"], + name="fk_import_story_arcs_proposed_story_arc_id_story_arcs", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["materialized_story_arc_id"], + ["story_arcs.id"], + name="fk_import_story_arcs_materialized_story_arc_id_story_arcs", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_import_story_arcs"), + sa.UniqueConstraint( + "import_job_id", + "source_key", + name="uq_import_story_arcs_job_source_key", + ), + ) + op.create_index( + "ix_import_story_arcs_job_status_id", + "import_story_arcs", + ["import_job_id", "status", "id"], + unique=False, + ) + op.create_index( + "ix_import_story_arcs_job_normalized_id", + "import_story_arcs", + ["import_job_id", "normalized_name", "id"], + unique=False, + ) + + op.create_table( + "import_story_arc_entries", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("imported_story_arc_id", sa.Integer(), nullable=False), + sa.Column("import_file_id", sa.Integer(), nullable=True), + sa.Column("matched_issue_id", sa.Integer(), nullable=True), + sa.Column("materialized_membership_id", sa.Integer(), nullable=True), + sa.Column("source_ordinal", sa.Integer(), nullable=False), + sa.Column("reading_order", sa.Integer(), nullable=True), + sa.Column("reading_order_raw", sa.String(50), nullable=True), + sa.Column( + "resolution_state", + _enum(_RESOLUTION_STATES, name="storyarcresolutionstate"), + nullable=False, + server_default="pending", + ), + sa.Column( + "source_kind", + _enum(_SOURCE_KINDS, name="storyarcsourcekind"), + nullable=False, + ), + sa.Column("source_entry_id", sa.String(255), nullable=True), + sa.Column("source_arc_id", sa.String(255), nullable=True), + sa.Column("source_issue_id", sa.String(255), nullable=True), + sa.Column("source_series_id", sa.String(255), nullable=True), + sa.Column("source_issue_number_text", sa.String(320), nullable=True), + sa.Column("source_series_name", sa.String(500), nullable=True), + sa.Column("source_issue_title", sa.String(500), nullable=True), + sa.Column("source_publisher", sa.String(255), nullable=True), + sa.Column("source_release_date_text", sa.String(50), nullable=True), + sa.Column("source_issue_date_text", sa.String(50), nullable=True), + sa.Column("resolution_confidence", sa.Float(), nullable=True), + sa.Column("resolution_method", sa.String(50), nullable=True), + sa.Column("evidence", sa.JSON(), nullable=False, server_default="{}"), + sa.Column("source_location", sa.String(1000), nullable=True), + sa.Column( + "selected_for_import", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.Column("diagnostics", sa.JSON(), nullable=False, server_default="{}"), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["imported_story_arc_id"], + ["import_story_arcs.id"], + name="fk_import_story_arc_entries_imported_story_arc_id_import_story_arcs", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["import_file_id"], + ["import_files.id"], + name="fk_import_story_arc_entries_import_file_id_import_files", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["matched_issue_id"], + ["issues.id"], + name="fk_import_story_arc_entries_matched_issue_id_issues", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["materialized_membership_id"], + ["issue_story_arcs.id"], + name="fk_import_story_arc_entries_materialized_membership_id_issue_story_arcs", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_import_story_arc_entries"), + sa.UniqueConstraint( + "imported_story_arc_id", + "source_ordinal", + name="uq_import_story_arc_entries_arc_ordinal", + ), + ) + op.create_index( + "ix_import_story_arc_entries_arc_resolution_order", + "import_story_arc_entries", + [ + "imported_story_arc_id", + "resolution_state", + "reading_order", + "source_ordinal", + "id", + ], + unique=False, + ) + op.create_index( + "ix_import_story_arc_entries_matched_issue_id", + "import_story_arc_entries", + ["matched_issue_id", "id"], + unique=False, + ) + + +def _create_story_arc_placements() -> None: + op.create_table( + "story_arc_placements", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("issue_story_arc_id", sa.Integer(), nullable=False), + sa.Column("library_file_id", sa.Integer(), nullable=True), + sa.Column("library_root_id", sa.Integer(), nullable=True), + sa.Column("placement_path", sa.String(1000), nullable=False), + sa.Column( + "mode", + _enum(_MATERIALIZATION_MODES, name="storyarcplacementmode"), + nullable=False, + server_default="reference_only", + ), + sa.Column( + "ownership", + _enum(_PLACEMENT_OWNERS, name="storyarcplacementownership"), + nullable=False, + server_default="referenced", + ), + sa.Column( + "symlink_style", + _enum(_SYMLINK_STYLES, name="storyarcsymlinkstyle"), + nullable=True, + ), + sa.Column( + "source_kind", + _enum(_SOURCE_KINDS, name="storyarcsourcekind"), + nullable=False, + server_default="legacy", + ), + sa.Column("source_import_job_id", sa.Integer(), nullable=True), + sa.Column("creating_action_id", sa.Integer(), nullable=True), + sa.Column("rendered_reading_order", sa.Integer(), nullable=True), + sa.Column("policy_schema_version", sa.Integer(), nullable=True), + sa.Column("source_fingerprint", sa.JSON(), nullable=False, server_default="{}"), + sa.Column( + "state", + _enum(_PLACEMENT_STATES, name="storyarcplacementstate"), + nullable=False, + server_default="current", + ), + sa.Column("last_result", sa.JSON(), nullable=False, server_default="{}"), + sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.CheckConstraint( + "mode = 'reference_only'", + name="ck_story_arc_placements_reference_only_mode", + ), + sa.CheckConstraint( + "ownership = 'referenced'", + name="ck_story_arc_placements_reference_only_owner", + ), + sa.CheckConstraint( + "symlink_style IS NULL", + name="ck_story_arc_placements_no_symlink_style", + ), + sa.ForeignKeyConstraint( + ["issue_story_arc_id"], + ["issue_story_arcs.id"], + name="fk_story_arc_placements_issue_story_arc_id_issue_story_arcs", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["library_file_id"], + ["library_files.id"], + name="fk_story_arc_placements_library_file_id_library_files", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["library_root_id"], + ["library_roots.id"], + name="fk_story_arc_placements_library_root_id_library_roots", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["source_import_job_id"], + ["import_jobs.id"], + name="fk_story_arc_placements_source_import_job_id_import_jobs", + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["creating_action_id"], + ["import_job_actions.id"], + name="fk_story_arc_placements_creating_action_id_import_job_actions", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_story_arc_placements"), + sa.UniqueConstraint("placement_path", name="uq_story_arc_placements_path"), + ) + op.create_index( + "ix_story_arc_placements_membership", + "story_arc_placements", + ["issue_story_arc_id", "id"], + unique=False, + ) + op.create_index( + "ix_story_arc_placements_library_file", + "story_arc_placements", + ["library_file_id", "id"], + unique=False, + ) + op.create_index( + "ix_story_arc_placements_state", + "story_arc_placements", + ["state", "id"], + unique=False, + ) + + +def _assert_downgrade_is_lossless() -> None: + connection = op.get_bind() + staged = connection.execute(sa.text("SELECT id FROM import_story_arcs LIMIT 1")).first() + if staged is not None: + raise RuntimeError( + "Cannot downgrade to the legacy story-arc schema while staged imports remain." + ) + placement = connection.execute(sa.text("SELECT id FROM story_arc_placements LIMIT 1")).first() + if placement is not None: + raise RuntimeError( + "Cannot downgrade to the legacy story-arc schema while arc placements remain." + ) + cohort = connection.execute( + sa.text( + "SELECT id FROM import_files WHERE source_folder_cohort_key IS NOT NULL " + "OR source_ordinal IS NOT NULL LIMIT 1" + ) + ).first() + if cohort is not None: + raise RuntimeError( + "Cannot downgrade to the legacy story-arc schema while folder cohort data remains." + ) + external = connection.execute( + sa.text( + "SELECT external.id FROM story_arc_external_identities AS external " + "JOIN story_arcs AS arc ON arc.id = external.story_arc_id " + "WHERE external.source != 'comicvine' " + "OR external.namespace != 'story_arc' " + "OR arc.comicvine_id IS NULL " + "OR external.external_id != CAST(arc.comicvine_id AS VARCHAR(255)) LIMIT 1" + ) + ).first() + if external is not None: + raise RuntimeError( + "Cannot downgrade to the legacy story-arc schema while external identities " + "lack a legacy ComicVine mirror." + ) + nonlegacy_arc = connection.execute( + sa.text( + "SELECT id FROM story_arcs WHERE source_kind != 'legacy' " + "OR lifecycle != 'active' OR monitored IS TRUE OR search_missing IS TRUE " + "OR include_upcoming IS TRUE OR sync_enabled IS TRUE " + "OR target_library_root_id IS NOT NULL OR policy_schema_version IS NOT NULL " + "OR CAST(policy_snapshot AS TEXT) != '{}' OR source_import_job_id IS NOT NULL " + "OR revision != 1 OR CAST(diagnostics AS TEXT) != '{}' LIMIT 1" + ) + ).first() + if nonlegacy_arc is not None: + raise RuntimeError( + "Cannot downgrade to the legacy story-arc schema while first-class arc policy " + "or monitoring data remains." + ) + nonlegacy_membership = connection.execute( + sa.text( + "SELECT membership.id FROM issue_story_arcs AS membership " + "LEFT JOIN issues ON issues.id = membership.issue_id " + "WHERE membership.issue_id IS NULL " + "OR membership.resolution_state != 'resolved' " + "OR membership.source_kind != 'legacy' " + "OR membership.source_entry_id IS NOT NULL " + "OR membership.source_arc_id IS NOT NULL " + "OR membership.source_issue_id IS NOT NULL " + "OR membership.source_series_id IS NOT NULL " + "OR (membership.source_issue_number_text IS NOT NULL AND " + "membership.source_issue_number_text != issues.issue_number_text) " + "OR membership.source_series_name IS NOT NULL " + "OR membership.source_issue_title IS NOT NULL " + "OR membership.source_publisher IS NOT NULL " + "OR membership.source_release_date_text IS NOT NULL " + "OR membership.source_issue_date_text IS NOT NULL " + "OR membership.resolution_confidence IS NOT NULL " + "OR membership.resolution_method IS NOT NULL " + "OR CAST(membership.evidence AS TEXT) != '{}' " + "OR membership.sync_eligible IS TRUE " + "OR CAST(membership.last_materialization_result AS TEXT) != '{}' LIMIT 1" + ) + ).first() + if nonlegacy_membership is not None: + raise RuntimeError( + "Cannot downgrade to the legacy story-arc schema while unresolved or " + "provenance-rich memberships remain." + ) + nondeterministic_ordinal = connection.execute( + sa.text( + "SELECT ranked.id FROM (" + "SELECT membership.id, membership.source_ordinal, " + "ROW_NUMBER() OVER (PARTITION BY membership.story_arc_id " + "ORDER BY membership.sequence_number, membership.issue_id) AS expected_ordinal " + "FROM issue_story_arcs AS membership" + ") AS ranked WHERE ranked.source_ordinal != ranked.expected_ordinal LIMIT 1" + ) + ).first() + if nondeterministic_ordinal is not None: + raise RuntimeError( + "Cannot downgrade to the legacy story-arc schema while custom membership " + "tie-break ordering remains." + ) + + +def _rebuild_issue_story_arcs_for_downgrade() -> None: + op.rename_table("issue_story_arcs", "issue_story_arcs_v2") + op.create_table( + "issue_story_arcs", + sa.Column("issue_id", sa.Integer(), nullable=False), + sa.Column("story_arc_id", sa.Integer(), nullable=False), + sa.Column("sequence_number", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["issue_id"], + ["issues.id"], + name="fk_issue_story_arcs_issue_id_issues_legacy", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["story_arc_id"], + ["story_arcs.id"], + name="fk_issue_story_arcs_story_arc_id_story_arcs_legacy", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "issue_id", + "story_arc_id", + name="pk_issue_story_arcs_legacy_restore", + ), + ) + op.get_bind().execute( + sa.text( + "INSERT INTO issue_story_arcs (issue_id, story_arc_id, sequence_number) " + "SELECT issue_id, story_arc_id, " + "CASE WHEN legacy_sequence_was_null THEN NULL ELSE sequence_number END " + "FROM issue_story_arcs_v2 ORDER BY story_arc_id, issue_id" + ) + ) + op.drop_table("issue_story_arcs_v2") + + +def _drop_story_arc_columns() -> None: + op.drop_index("ix_story_arcs_source_job_id", table_name="story_arcs") + op.drop_index("ix_story_arcs_lifecycle_monitored_id", table_name="story_arcs") + op.drop_index("ix_story_arcs_normalized_id", table_name="story_arcs") + + if not _is_sqlite(): + op.drop_constraint( + "fk_story_arcs_source_import_job_id_import_jobs", + "story_arcs", + type_="foreignkey", + ) + op.drop_constraint( + "fk_story_arcs_target_library_root_id_library_roots", + "story_arcs", + type_="foreignkey", + ) + op.drop_constraint("storyarclifecycle", "story_arcs", type_="check") + op.drop_constraint("storyarcsourcekind", "story_arcs", type_="check") + + for column_name in ( + "diagnostics", + "revision", + "policy_snapshot", + "policy_schema_version", + "source_import_job_id", + "target_library_root_id", + "sync_enabled", + "include_upcoming", + "search_missing", + "monitored", + "lifecycle", + "source_kind", + "normalized_name", + ): + op.drop_column("story_arcs", column_name) + + +def upgrade() -> None: + """Add first-class arc identity, import staging, and reference-only placements.""" + _add_story_arc_columns() + op.add_column( + "import_files", + sa.Column("source_folder_cohort_key", sa.String(1000), nullable=True), + ) + op.add_column( + "import_files", + sa.Column("source_ordinal", sa.Integer(), nullable=True), + ) + op.create_index( + "ix_import_files_job_cohort_order", + "import_files", + ["import_job_id", "source_folder_cohort_key", "source_ordinal", "id"], + unique=False, + ) + _rebuild_issue_story_arcs_for_upgrade() + _create_story_arc_external_identities() + _create_import_story_arc_tables() + _create_story_arc_placements() + + +def downgrade() -> None: + """Restore the legacy association only when all new data is representable.""" + _assert_downgrade_is_lossless() + + op.drop_table("story_arc_placements") + op.drop_table("import_story_arc_entries") + op.drop_table("import_story_arcs") + op.drop_table("story_arc_external_identities") + _rebuild_issue_story_arcs_for_downgrade() + + op.drop_index("ix_import_files_job_cohort_order", table_name="import_files") + op.drop_column("import_files", "source_ordinal") + op.drop_column("import_files", "source_folder_cohort_key") + _drop_story_arc_columns() diff --git a/alembic/versions/b3c4d5e6f789_enable_managed_story_arc_placements.py b/alembic/versions/b3c4d5e6f789_enable_managed_story_arc_placements.py new file mode 100644 index 00000000..ae5378b2 --- /dev/null +++ b/alembic/versions/b3c4d5e6f789_enable_managed_story_arc_placements.py @@ -0,0 +1,231 @@ +"""Enable managed story-arc copy and link placement modes. + +Revision ID: b3c4d5e6f789 +Revises: a2b3c4d5e678 +Create Date: 2026-08-30 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "b3c4d5e6f789" +down_revision: str | Sequence[str] | None = "a2b3c4d5e678" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_REFERENCE_ONLY_CONSTRAINTS = ( + "ck_story_arc_placements_reference_only_mode", + "ck_story_arc_placements_reference_only_owner", + "ck_story_arc_placements_no_symlink_style", +) +_MODE_OWNERSHIP_CONSTRAINT = "ck_story_arc_placements_mode_ownership" +_SYMLINK_STYLE_CONSTRAINT = "ck_story_arc_placements_symlink_style" +_MODE_OWNERSHIP_SQL = ( + "((mode = 'reference_only' AND ownership = 'referenced') OR " + "(mode IN ('copy', 'hardlink', 'symlink') AND ownership = 'managed'))" +) +_SYMLINK_STYLE_SQL = ( + "((mode = 'symlink' AND symlink_style IS NOT NULL) OR " + "(mode != 'symlink' AND symlink_style IS NULL))" +) +_SYNC_WORK_STATES = ( + "queued", + "running", + "retry_wait", + "failed", + "completed", + "cancelled", +) +_SYNC_WORK_REASONS = ("canonical_registered", "discrepancy_recovery") + + +def _enable_managed_combinations() -> None: + with op.batch_alter_table("story_arc_placements") as batch_op: + batch_op.add_column(sa.Column("operation_token", sa.String(length=32), nullable=True)) + for constraint_name in _REFERENCE_ONLY_CONSTRAINTS: + batch_op.drop_constraint(constraint_name, type_="check") + batch_op.create_check_constraint( + _MODE_OWNERSHIP_CONSTRAINT, + _MODE_OWNERSHIP_SQL, + ) + batch_op.create_check_constraint( + _SYMLINK_STYLE_CONSTRAINT, + _SYMLINK_STYLE_SQL, + ) + + +def _assert_downgrade_is_lossless() -> None: + managed = ( + op.get_bind() + .execute( + sa.text( + "SELECT id FROM story_arc_placements " + "WHERE mode != 'reference_only' OR ownership != 'referenced' " + "OR symlink_style IS NOT NULL OR operation_token IS NOT NULL LIMIT 1" + ) + ) + .first() + ) + if managed is not None: + raise RuntimeError( + "Cannot downgrade while managed story-arc placements remain; " + "remove or convert them to referenced placements first." + ) + + +def _restore_reference_only_combinations() -> None: + with op.batch_alter_table("story_arc_placements") as batch_op: + batch_op.drop_constraint(_SYMLINK_STYLE_CONSTRAINT, type_="check") + batch_op.drop_constraint(_MODE_OWNERSHIP_CONSTRAINT, type_="check") + batch_op.create_check_constraint( + "ck_story_arc_placements_reference_only_mode", + "mode = 'reference_only'", + ) + batch_op.create_check_constraint( + "ck_story_arc_placements_reference_only_owner", + "ownership = 'referenced'", + ) + batch_op.create_check_constraint( + "ck_story_arc_placements_no_symlink_style", + "symlink_style IS NULL", + ) + batch_op.drop_column("operation_token") + + +def _create_story_arc_sync_work() -> None: + op.create_table( + "story_arc_sync_work", + sa.Column("issue_story_arc_id", sa.Integer(), nullable=False), + sa.Column("library_file_id", sa.Integer(), nullable=False), + sa.Column("desired_generation", sa.String(length=64), nullable=False), + sa.Column("source_signature_hash", sa.String(length=64), nullable=False), + sa.Column("source_file_path", sa.String(length=1000), nullable=False), + sa.Column("source_file_size", sa.BigInteger(), nullable=False), + sa.Column("source_file_modified_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("source_file_hash", sa.String(length=64), nullable=True), + sa.Column("source_signature_schema_version", sa.Integer(), nullable=True), + sa.Column("source_signature_resolved_path", sa.String(length=1000), nullable=True), + sa.Column("source_signature_size", sa.BigInteger(), nullable=True), + sa.Column("source_signature_mtime_ns", sa.BigInteger(), nullable=True), + sa.Column("source_signature_device", sa.BigInteger(), nullable=True), + sa.Column("source_signature_inode", sa.BigInteger(), nullable=True), + sa.Column("story_arc_revision", sa.Integer(), nullable=False), + sa.Column("membership_sequence", sa.Integer(), nullable=False), + sa.Column("policy_schema_version", sa.Integer(), nullable=False), + sa.Column( + "reason", + sa.Enum( + *_SYNC_WORK_REASONS, + name="storyarcsyncreason", + native_enum=False, + create_constraint=True, + ), + server_default="canonical_registered", + nullable=False, + ), + sa.Column( + "state", + sa.Enum( + *_SYNC_WORK_STATES, + name="storyarcsyncworkstate", + native_enum=False, + create_constraint=True, + ), + server_default="queued", + nullable=False, + ), + sa.Column("attempt_count", sa.Integer(), server_default="0", nullable=False), + sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("claim_token", sa.String(length=64), nullable=True), + sa.Column("claimed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_error_code", sa.String(length=100), nullable=True), + sa.Column("last_error_category", sa.String(length=50), nullable=True), + sa.Column("last_error_detail", sa.Text(), nullable=True), + sa.Column("last_result", sa.JSON(), server_default="{}", nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["issue_story_arc_id"], + ["issue_story_arcs.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["library_file_id"], + ["library_files.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "issue_story_arc_id", + "desired_generation", + name="uq_story_arc_sync_work_generation", + ), + ) + op.create_index( + "ix_story_arc_sync_work_ready", + "story_arc_sync_work", + ["state", "next_attempt_at", "id"], + unique=False, + ) + op.create_index( + "ix_story_arc_sync_work_membership", + "story_arc_sync_work", + ["issue_story_arc_id", "id"], + unique=False, + ) + op.create_index( + "ix_story_arc_sync_work_library_file", + "story_arc_sync_work", + ["library_file_id", "id"], + unique=False, + ) + + +def _drop_story_arc_sync_work() -> None: + op.drop_index( + "ix_story_arc_sync_work_library_file", + table_name="story_arc_sync_work", + ) + op.drop_index( + "ix_story_arc_sync_work_membership", + table_name="story_arc_sync_work", + ) + op.drop_index( + "ix_story_arc_sync_work_ready", + table_name="story_arc_sync_work", + ) + op.drop_table("story_arc_sync_work") + + +def upgrade() -> None: + """Admit only internally consistent managed copy/link placements.""" + _enable_managed_combinations() + _create_story_arc_sync_work() + + +def downgrade() -> None: + """Restore IU6-A checks after discarding rebuildable sync-queue state.""" + _assert_downgrade_is_lossless() + # This is operational work state, not audit history. The periodic + # discrepancy sweep recreates eligible intent after a later re-upgrade. + _drop_story_arc_sync_work() + _restore_reference_only_combinations() diff --git a/alembic/versions/c4d5e6f7a890_add_import_owned_story_arc_sync.py b/alembic/versions/c4d5e6f7a890_add_import_owned_story_arc_sync.py new file mode 100644 index 00000000..b6cea0d0 --- /dev/null +++ b/alembic/versions/c4d5e6f7a890_add_import_owned_story_arc_sync.py @@ -0,0 +1,300 @@ +"""Add import provenance and cancellation intent to story-arc sync work. + +Revision ID: c4d5e6f7a890 +Revises: b3c4d5e6f789 +Create Date: 2026-08-30 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "c4d5e6f7a890" +down_revision: str | Sequence[str] | None = "b3c4d5e6f789" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_ORIGIN_FK = "fk_story_arc_sync_work_origin_action_import_job_actions" +_ORIGIN_JOB_FK = "fk_story_arc_sync_work_origin_job_import_jobs" +_ORIGIN_ARC_FK = "fk_story_arc_sync_work_origin_arc_import_story_arcs" +_ORIGIN_ENTRY_FK = "fk_story_arc_sync_work_origin_entry_import_story_arc_entries" +_ORIGIN_UNIQUE = "uq_story_arc_sync_work_origin_import_action" +_ORIGIN_JOB_STATE_INDEX = "ix_story_arc_sync_work_origin_job_state" +_FOLLOWUP_INDEX = "ix_import_jobs_story_arc_followup" +_ROLLBACK_WAITING_FK = "fk_import_jobs_story_arc_rollback_work" +_ROLLBACK_WAITING_INDEX = "ix_import_jobs_story_arc_rollback_waiting" +_ACTION_KEYSET_INDEX = "ix_import_job_actions_job_id_keyset" +_PLACEMENT_ACTION_INDEX = "ix_story_arc_placements_creating_action" +_READY_INDEX = "ix_story_arc_sync_work_ready" +_QUEUED_INDEX = "ix_story_arc_sync_work_queued" +_STALE_CLAIM_INDEX = "ix_story_arc_sync_work_stale_claim" + + +def _add_import_sync_foundation() -> None: + with op.batch_alter_table("story_arc_sync_work") as batch_op: + batch_op.add_column(sa.Column("origin_import_action_id", sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column("origin_import_job_id", sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column("origin_imported_story_arc_id", sa.Integer(), nullable=True)) + batch_op.add_column( + sa.Column("origin_imported_story_arc_entry_id", sa.Integer(), nullable=True) + ) + batch_op.add_column( + sa.Column("cancel_requested_at", sa.DateTime(timezone=True), nullable=True) + ) + batch_op.add_column( + sa.Column( + "claimable", + sa.Boolean(), + server_default=sa.true(), + nullable=False, + ) + ) + batch_op.create_foreign_key( + _ORIGIN_FK, + "import_job_actions", + ["origin_import_action_id"], + ["id"], + ondelete="SET NULL", + ) + batch_op.create_foreign_key( + _ORIGIN_JOB_FK, + "import_jobs", + ["origin_import_job_id"], + ["id"], + ondelete="SET NULL", + ) + batch_op.create_foreign_key( + _ORIGIN_ARC_FK, + "import_story_arcs", + ["origin_imported_story_arc_id"], + ["id"], + ondelete="SET NULL", + ) + batch_op.create_foreign_key( + _ORIGIN_ENTRY_FK, + "import_story_arc_entries", + ["origin_imported_story_arc_entry_id"], + ["id"], + ondelete="SET NULL", + ) + batch_op.create_unique_constraint( + _ORIGIN_UNIQUE, + ["origin_import_action_id"], + ) + op.create_index( + _ORIGIN_JOB_STATE_INDEX, + "story_arc_sync_work", + ["origin_import_job_id", "state", "id"], + unique=False, + ) + op.add_column( + "import_jobs", + sa.Column( + "story_arc_placement_followup_pending", + sa.Boolean(), + server_default=sa.false(), + nullable=False, + ), + ) + if op.get_bind().dialect.name == "sqlite": + # SQLite can add one nullable REFERENCES column without rebuilding the + # parent table. A batch rebuild would cascade-delete existing import + # children when the old import_jobs table is dropped. + op.execute( + sa.text( + "ALTER TABLE import_jobs ADD COLUMN story_arc_rollback_waiting_work_id " + "INTEGER REFERENCES story_arc_sync_work(id) ON DELETE SET NULL" + ) + ) + else: + op.add_column( + "import_jobs", + sa.Column("story_arc_rollback_waiting_work_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + _ROLLBACK_WAITING_FK, + "import_jobs", + "story_arc_sync_work", + ["story_arc_rollback_waiting_work_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_index( + _FOLLOWUP_INDEX, + "import_jobs", + ["status", "story_arc_placement_followup_pending", "id"], + unique=False, + ) + op.create_index( + _ROLLBACK_WAITING_INDEX, + "import_jobs", + ["status", "story_arc_rollback_waiting_work_id", "id"], + unique=False, + ) + op.create_index( + _ACTION_KEYSET_INDEX, + "import_job_actions", + ["import_job_id", "id"], + unique=False, + ) + op.create_index( + _PLACEMENT_ACTION_INDEX, + "story_arc_placements", + ["creating_action_id", "id"], + unique=False, + ) + op.create_index( + _QUEUED_INDEX, + "story_arc_sync_work", + ["claimable", "state", "created_at", "id"], + unique=False, + ) + op.drop_index(_READY_INDEX, table_name="story_arc_sync_work") + op.create_index( + _READY_INDEX, + "story_arc_sync_work", + ["claimable", "state", "next_attempt_at", "id"], + unique=False, + ) + op.create_index( + _STALE_CLAIM_INDEX, + "story_arc_sync_work", + ["claimable", "state", "claimed_at", "id"], + unique=False, + ) + + +def _assert_downgrade_is_lossless() -> None: + held_work = ( + op.get_bind() + .execute(sa.text("SELECT id FROM story_arc_sync_work WHERE claimable = false LIMIT 1")) + .first() + ) + if held_work is not None: + raise RuntimeError( + "Cannot downgrade while held story-arc sync work remains; releasing the " + "claimability fence would revive work that has not been safely published." + ) + + linked_work = ( + op.get_bind() + .execute( + sa.text( + "SELECT id FROM story_arc_sync_work " + "WHERE origin_import_action_id IS NOT NULL LIMIT 1" + ) + ) + .first() + ) + if linked_work is not None: + raise RuntimeError( + "Cannot downgrade while import-owned story-arc sync work remains; " + "finish, cancel, or detach the linked work before removing its provenance." + ) + + typed_provenance = ( + op.get_bind() + .execute( + sa.text( + "SELECT id FROM story_arc_sync_work WHERE " + "origin_import_job_id IS NOT NULL " + "OR origin_imported_story_arc_id IS NOT NULL " + "OR origin_imported_story_arc_entry_id IS NOT NULL LIMIT 1" + ) + ) + .first() + ) + if typed_provenance is not None: + raise RuntimeError( + "Cannot downgrade while typed import-owned story-arc sync provenance remains; " + "finish, cancel, or detach the linked work before removing its provenance." + ) + + pending_cancellation = ( + op.get_bind() + .execute( + sa.text( + "SELECT id FROM story_arc_sync_work WHERE cancel_requested_at IS NOT NULL LIMIT 1" + ) + ) + .first() + ) + if pending_cancellation is not None: + raise RuntimeError( + "Cannot downgrade while a pending story-arc sync cancellation remains; " + "consume or clear it before removing durable cancellation intent." + ) + + pending_import_lifecycle = ( + op.get_bind() + .execute( + sa.text( + "SELECT id FROM import_jobs WHERE story_arc_placement_followup_pending " + "OR story_arc_rollback_waiting_work_id IS NOT NULL LIMIT 1" + ) + ) + .first() + ) + if pending_import_lifecycle is not None: + raise RuntimeError( + "Cannot downgrade while an import Story Arc follow-up or rollback wait remains; " + "finish the pending lifecycle work before removing its durable routing state." + ) + + +def _remove_import_sync_foundation() -> None: + op.drop_index(_ACTION_KEYSET_INDEX, table_name="import_job_actions") + op.drop_index(_ROLLBACK_WAITING_INDEX, table_name="import_jobs") + op.drop_index(_FOLLOWUP_INDEX, table_name="import_jobs") + if op.get_bind().dialect.name == "sqlite": + op.execute( + sa.text("ALTER TABLE import_jobs DROP COLUMN story_arc_rollback_waiting_work_id") + ) + op.execute( + sa.text("ALTER TABLE import_jobs DROP COLUMN story_arc_placement_followup_pending") + ) + else: + op.drop_constraint(_ROLLBACK_WAITING_FK, "import_jobs", type_="foreignkey") + op.drop_column("import_jobs", "story_arc_rollback_waiting_work_id") + op.drop_column("import_jobs", "story_arc_placement_followup_pending") + op.drop_index(_ORIGIN_JOB_STATE_INDEX, table_name="story_arc_sync_work") + op.drop_index(_STALE_CLAIM_INDEX, table_name="story_arc_sync_work") + op.drop_index(_READY_INDEX, table_name="story_arc_sync_work") + op.create_index( + _READY_INDEX, + "story_arc_sync_work", + ["state", "next_attempt_at", "id"], + unique=False, + ) + op.drop_index(_QUEUED_INDEX, table_name="story_arc_sync_work") + op.drop_index(_PLACEMENT_ACTION_INDEX, table_name="story_arc_placements") + with op.batch_alter_table("story_arc_sync_work") as batch_op: + batch_op.drop_constraint(_ORIGIN_UNIQUE, type_="unique") + batch_op.drop_constraint(_ORIGIN_ENTRY_FK, type_="foreignkey") + batch_op.drop_constraint(_ORIGIN_ARC_FK, type_="foreignkey") + batch_op.drop_constraint(_ORIGIN_JOB_FK, type_="foreignkey") + batch_op.drop_constraint(_ORIGIN_FK, type_="foreignkey") + batch_op.drop_column("claimable") + batch_op.drop_column("cancel_requested_at") + batch_op.drop_column("origin_imported_story_arc_entry_id") + batch_op.drop_column("origin_imported_story_arc_id") + batch_op.drop_column("origin_import_job_id") + batch_op.drop_column("origin_import_action_id") + + +def upgrade() -> None: + """Add optional action provenance and durable cancellation requests.""" + _add_import_sync_foundation() + + +def downgrade() -> None: + """Remove additive fields only when doing so cannot erase provenance.""" + _assert_downgrade_is_lossless() + _remove_import_sync_foundation() diff --git a/alembic/versions/d5e6f7a8b901_add_import_story_arc_intent.py b/alembic/versions/d5e6f7a8b901_add_import_story_arc_intent.py new file mode 100644 index 00000000..5f32bdd6 --- /dev/null +++ b/alembic/versions/d5e6f7a8b901_add_import_story_arc_intent.py @@ -0,0 +1,66 @@ +"""Add durable Step 1 story-arc import intent. + +Revision ID: d5e6f7a8b901 +Revises: c4d5e6f7a890 +Create Date: 2026-08-30 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "d5e6f7a8b901" +down_revision: str | Sequence[str] | None = "c4d5e6f7a890" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _intent_columns() -> tuple[sa.Column[bool], sa.Column[bool]]: + """Return fresh column objects for SQLite/native and batch operations.""" + return ( + sa.Column( + "story_arc_import_requested", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + sa.Column( + "story_arc_materialization_requested", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + + +def upgrade() -> None: + """Record compatible, non-authorizing Story Arc wizard choices.""" + import_requested, materialization_requested = _intent_columns() + if op.get_bind().dialect.name == "sqlite": + # Both fields are simple constant-default additions. Native ALTER keeps + # c4's rollback-work foreign key exactly as introduced; a batch rebuild + # here changes that older FK into a shape SQLite cannot later drop. + op.add_column("import_jobs", import_requested) + op.add_column("import_jobs", materialization_requested) + return + with op.batch_alter_table("import_jobs") as batch_op: + batch_op.add_column(import_requested) + batch_op.add_column(materialization_requested) + + +def downgrade() -> None: + """Remove Story Arc wizard intent fields.""" + if op.get_bind().dialect.name == "sqlite": + op.drop_column("import_jobs", "story_arc_materialization_requested") + op.drop_column("import_jobs", "story_arc_import_requested") + return + with op.batch_alter_table("import_jobs") as batch_op: + batch_op.drop_column("story_arc_materialization_requested") + batch_op.drop_column("story_arc_import_requested") diff --git a/alembic/versions/e6f7a8b9c012_drop_legacy_issue_number_uniqueness.py b/alembic/versions/e6f7a8b9c012_drop_legacy_issue_number_uniqueness.py new file mode 100644 index 00000000..fb6391bf --- /dev/null +++ b/alembic/versions/e6f7a8b9c012_drop_legacy_issue_number_uniqueness.py @@ -0,0 +1,78 @@ +"""Make exact issue-number text the canonical per-series identity. + +Revision ID: e6f7a8b9c012 +Revises: d5e6f7a8b901 +Create Date: 2026-08-30 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "e6f7a8b9c012" +down_revision: str | Sequence[str] | None = "d5e6f7a8b901" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_LEGACY_NUMERIC_UNIQUE = "uq_series_issue" + + +def _drop_legacy_numeric_unique() -> None: + if op.get_bind().dialect.name == "sqlite": + # SQLite stores a table-level UNIQUE constraint in an auto-index, so + # removing it requires a bounded table rebuild. Alembic copies rows and + # recreates the exact-text and ordering indexes during the batch. + with op.batch_alter_table("issues", recreate="always") as batch_op: + batch_op.drop_constraint(_LEGACY_NUMERIC_UNIQUE, type_="unique") + return + op.drop_constraint(_LEGACY_NUMERIC_UNIQUE, "issues", type_="unique") + + +def _assert_numeric_uniqueness_can_be_restored() -> None: + duplicate = ( + op.get_bind() + .execute( + sa.text( + "SELECT series_id, issue_number FROM issues " + "GROUP BY series_id, issue_number HAVING COUNT(*) > 1 LIMIT 1" + ) + ) + .first() + ) + if duplicate is not None: + raise RuntimeError( + "Cannot downgrade while exact issue-number siblings share a numeric value." + ) + + +def _restore_legacy_numeric_unique() -> None: + if op.get_bind().dialect.name == "sqlite": + with op.batch_alter_table("issues", recreate="always") as batch_op: + batch_op.create_unique_constraint( + _LEGACY_NUMERIC_UNIQUE, + ["series_id", "issue_number"], + ) + return + op.create_unique_constraint( + _LEGACY_NUMERIC_UNIQUE, + "issues", + ["series_id", "issue_number"], + ) + + +def upgrade() -> None: + """Remove the float identity constraint while retaining its sort index.""" + _drop_legacy_numeric_unique() + + +def downgrade() -> None: + """Restore float uniqueness only when no exact siblings would be lost.""" + _assert_numeric_uniqueness_can_be_restored() + _restore_legacy_numeric_unique() diff --git a/alembic/versions/f7a8b9c0d123_add_story_arc_covers.py b/alembic/versions/f7a8b9c0d123_add_story_arc_covers.py new file mode 100644 index 00000000..1c381ed2 --- /dev/null +++ b/alembic/versions/f7a8b9c0d123_add_story_arc_covers.py @@ -0,0 +1,43 @@ +"""Add first-class Story Arc cover storage. + +Revision ID: f7a8b9c0d123 +Revises: e6f7a8b9c012 +Create Date: 2026-08-31 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "f7a8b9c0d123" +down_revision: str | Sequence[str] | None = "e6f7a8b9c012" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Store provider and locally cached Story Arc artwork explicitly.""" + with op.batch_alter_table("story_arcs") as batch_op: + batch_op.add_column(sa.Column("cover_path", sa.String(length=500), nullable=True)) + batch_op.add_column(sa.Column("cover_url", sa.String(length=500), nullable=True)) + + +def downgrade() -> None: + """Remove first-class Story Arc artwork fields.""" + if op.get_bind().dialect.name == "sqlite": + # Native DROP preserves a2's inline foreign-key definitions. A batch + # rebuild turns them into table-level constraints that a2's later + # downgrade cannot remove column-by-column. + op.drop_column("story_arcs", "cover_url") + op.drop_column("story_arcs", "cover_path") + return + with op.batch_alter_table("story_arcs") as batch_op: + batch_op.drop_column("cover_url") + batch_op.drop_column("cover_path") diff --git a/alembic/versions/g8b9c0d1e234_add_mylar_path_map_confirmation.py b/alembic/versions/g8b9c0d1e234_add_mylar_path_map_confirmation.py new file mode 100644 index 00000000..e30edc78 --- /dev/null +++ b/alembic/versions/g8b9c0d1e234_add_mylar_path_map_confirmation.py @@ -0,0 +1,40 @@ +"""Add durable Mylar path-mapping confirmation. + +Revision ID: g8b9c0d1e234 +Revises: f7a8b9c0d123 +Create Date: 2026-08-31 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "g8b9c0d1e234" +down_revision: str | Sequence[str] | None = "f7a8b9c0d123" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Distinguish a confirmed identity map from legacy auto-detection state.""" + op.add_column( + "import_jobs", + sa.Column( + "mylar3_path_map_confirmed", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + + +def downgrade() -> None: + """Remove durable Mylar path-mapping confirmation.""" + op.drop_column("import_jobs", "mylar3_path_map_confirmed") diff --git a/alembic/versions/h9c0d1e2f345_add_library_root_management.py b/alembic/versions/h9c0d1e2f345_add_library_root_management.py new file mode 100644 index 00000000..9eafbae0 --- /dev/null +++ b/alembic/versions/h9c0d1e2f345_add_library_root_management.py @@ -0,0 +1,114 @@ +"""Add explicit multi-library root roles and managed default. + +Revision ID: h9c0d1e2f345 +Revises: g8b9c0d1e234 +Create Date: 2026-08-31 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "h9c0d1e2f345" +down_revision: str | Sequence[str] | None = "g8b9c0d1e234" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_DEFAULT_INDEX = "uq_library_roots_default_managed_destination" +_LEGACY_DEFAULT_ROOT = sa.text( + "SELECT roots.id FROM library_roots AS roots " + "LEFT JOIN system_config AS config " + "ON config.key = 'comics_directory' AND config.value = roots.path " + "WHERE roots.enabled = true AND roots.allow_managed_writes = true " + "ORDER BY CASE WHEN config.key IS NULL THEN 1 ELSE 0 END, roots.id " + "LIMIT 1" +) + + +def upgrade() -> None: + """Add root roles, backfill one legacy default, and constrain uniqueness.""" + op.add_column( + "library_roots", + sa.Column( + "allow_referenced_registrations", + sa.Boolean(), + nullable=False, + server_default=sa.true(), + ), + ) + op.add_column( + "library_roots", + sa.Column( + "allow_managed_writes", + sa.Boolean(), + nullable=False, + server_default=sa.true(), + ), + ) + op.add_column( + "library_roots", + sa.Column( + "is_default_managed_destination", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + + bind = op.get_bind() + selected_id = bind.execute(_LEGACY_DEFAULT_ROOT).scalar_one_or_none() + if selected_id is not None: + bind.execute( + sa.text( + "UPDATE library_roots SET is_default_managed_destination = true WHERE id = :root_id" + ), + {"root_id": selected_id}, + ) + + op.create_index( + _DEFAULT_INDEX, + "library_roots", + ["is_default_managed_destination"], + unique=True, + sqlite_where=sa.text("is_default_managed_destination = 1"), + postgresql_where=sa.text("is_default_managed_destination"), + ) + + +def downgrade() -> None: + """Remove explicit root-management fields and the default constraint.""" + bind = op.get_bind() + nonrepresentable_roles = bind.execute( + sa.text( + "SELECT COUNT(*) FROM library_roots " + "WHERE allow_referenced_registrations = false " + "OR allow_managed_writes = false" + ) + ).scalar_one() + if nonrepresentable_roles: + raise RuntimeError( + "Cannot downgrade library-root capabilities while a root disables " + "referenced registrations or managed writes." + ) + + selected_id = bind.execute(_LEGACY_DEFAULT_ROOT).scalar_one_or_none() + current_default_id = bind.execute( + sa.text("SELECT id FROM library_roots WHERE is_default_managed_destination = true") + ).scalar_one_or_none() + if current_default_id != selected_id: + raise RuntimeError( + "Cannot downgrade library-root capabilities because the default managed " + "destination cannot be reconstructed by the legacy schema." + ) + + op.drop_index(_DEFAULT_INDEX, table_name="library_roots") + op.drop_column("library_roots", "is_default_managed_destination") + op.drop_column("library_roots", "allow_managed_writes") + op.drop_column("library_roots", "allow_referenced_registrations") diff --git a/alembic/versions/i0d1e2f3a456_add_series_preferred_library_root.py b/alembic/versions/i0d1e2f3a456_add_series_preferred_library_root.py new file mode 100644 index 00000000..5ca5dd73 --- /dev/null +++ b/alembic/versions/i0d1e2f3a456_add_series_preferred_library_root.py @@ -0,0 +1,71 @@ +"""Add a durable preferred managed destination to series. + +Revision ID: i0d1e2f3a456 +Revises: h9c0d1e2f345 +Create Date: 2026-08-31 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "i0d1e2f3a456" +down_revision: str | Sequence[str] | None = "h9c0d1e2f345" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_FOREIGN_KEY = "fk_series_preferred_library_root_id" + + +def upgrade() -> None: + """Add the preferred root and retain managed current roots as the default.""" + with op.batch_alter_table("series") as batch_op: + batch_op.add_column(sa.Column("preferred_library_root_id", sa.Integer(), nullable=True)) + batch_op.create_foreign_key( + _FOREIGN_KEY, + "library_roots", + ["preferred_library_root_id"], + ["id"], + ondelete="SET NULL", + ) + + op.execute( + sa.text( + "UPDATE series SET preferred_library_root_id = library_root_id " + "WHERE library_root_id IN (" + "SELECT id FROM library_roots WHERE allow_managed_writes = true" + ")" + ) + ) + + +def downgrade() -> None: + """Remove the independent preferred destination from series.""" + nonrepresentable_count = ( + op.get_bind() + .execute( + sa.text( + "SELECT COUNT(*) FROM series " + "WHERE preferred_library_root_id IS NOT NULL " + "AND (library_root_id IS NULL " + "OR preferred_library_root_id != library_root_id)" + ) + ) + .scalar_one() + ) + if nonrepresentable_count: + raise RuntimeError( + "Cannot downgrade series preferred roots while a series uses a preferred " + "managed destination different from its current library root." + ) + + with op.batch_alter_table("series") as batch_op: + batch_op.drop_constraint(_FOREIGN_KEY, type_="foreignkey") + batch_op.drop_column("preferred_library_root_id") diff --git a/alembic/versions/j1e2f3a4b567_index_import_files_matched_issue.py b/alembic/versions/j1e2f3a4b567_index_import_files_matched_issue.py new file mode 100644 index 00000000..28a990ee --- /dev/null +++ b/alembic/versions/j1e2f3a4b567_index_import_files_matched_issue.py @@ -0,0 +1,38 @@ +"""Index import-file references to matched issues. + +Revision ID: j1e2f3a4b567 +Revises: i0d1e2f3a456 +Create Date: 2026-09-01 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "j1e2f3a4b567" +down_revision: str | Sequence[str] | None = "i0d1e2f3a456" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Avoid full import-file scans when an issue is deleted or rolled back.""" + op.create_index( + "ix_import_files_matched_issue_id", + "import_files", + ["matched_issue_id"], + unique=False, + ) + + +def downgrade() -> None: + """Remove the matched-issue lookup index.""" + op.drop_index( + "ix_import_files_matched_issue_id", + table_name="import_files", + ) diff --git a/alembic/versions/k2f3a4b5c678_index_import_files_import_series.py b/alembic/versions/k2f3a4b5c678_index_import_files_import_series.py new file mode 100644 index 00000000..85cb6663 --- /dev/null +++ b/alembic/versions/k2f3a4b5c678_index_import_files_import_series.py @@ -0,0 +1,38 @@ +"""Index import-file references to their staged series. + +Revision ID: k2f3a4b5c678 +Revises: j1e2f3a4b567 +Create Date: 2026-09-01 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "k2f3a4b5c678" +down_revision: str | Sequence[str] | None = "j1e2f3a4b567" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Avoid full import-file scans for per-series matching and cleanup.""" + op.create_index( + "ix_import_files_import_series_id", + "import_files", + ["import_series_id"], + unique=False, + ) + + +def downgrade() -> None: + """Remove the staged-series lookup index.""" + op.drop_index( + "ix_import_files_import_series_id", + table_name="import_files", + ) diff --git a/alembic/versions/l3f4a5b6c789_index_import_file_delete_references.py b/alembic/versions/l3f4a5b6c789_index_import_file_delete_references.py new file mode 100644 index 00000000..9aff8736 --- /dev/null +++ b/alembic/versions/l3f4a5b6c789_index_import_file_delete_references.py @@ -0,0 +1,48 @@ +"""Index staged-file references used by database-side cascade deletes. + +Revision ID: l3f4a5b6c789 +Revises: k2f3a4b5c678 +Create Date: 2026-09-01 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "l3f4a5b6c789" +down_revision: str | Sequence[str] | None = "k2f3a4b5c678" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Avoid full child-table scans while staged import files are deleted.""" + op.create_index( + "ix_import_files_duplicate_of_file_id", + "import_files", + ["duplicate_of_file_id"], + unique=False, + ) + op.create_index( + "ix_import_story_arc_entries_import_file_id", + "import_story_arc_entries", + ["import_file_id"], + unique=False, + ) + + +def downgrade() -> None: + """Remove staged-file cascade support indexes.""" + op.drop_index( + "ix_import_story_arc_entries_import_file_id", + table_name="import_story_arc_entries", + ) + op.drop_index( + "ix_import_files_duplicate_of_file_id", + table_name="import_files", + ) diff --git a/alembic/versions/m4g5h6i7j890_add_import_job_archived_at.py b/alembic/versions/m4g5h6i7j890_add_import_job_archived_at.py new file mode 100644 index 00000000..c73095fb --- /dev/null +++ b/alembic/versions/m4g5h6i7j890_add_import_job_archived_at.py @@ -0,0 +1,42 @@ +"""Add non-destructive import history archival. + +Revision ID: m4g5h6i7j890 +Revises: l3f4a5b6c789 +Create Date: 2026-09-04 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "m4g5h6i7j890" +down_revision: str | Sequence[str] | None = "l3f4a5b6c789" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Add the nullable archive timestamp and history lookup index.""" + op.add_column( + "import_jobs", + sa.Column("archived_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "ix_import_jobs_archived_created", + "import_jobs", + ["archived_at", "created_at"], + unique=False, + ) + + +def downgrade() -> None: + """Remove non-destructive import archival support.""" + op.drop_index("ix_import_jobs_archived_created", table_name="import_jobs") + op.drop_column("import_jobs", "archived_at") diff --git a/alembic/versions/n5h6i7j8k901_protect_library_root_removal.py b/alembic/versions/n5h6i7j8k901_protect_library_root_removal.py new file mode 100644 index 00000000..4fddf2ee --- /dev/null +++ b/alembic/versions/n5h6i7j8k901_protect_library_root_removal.py @@ -0,0 +1,149 @@ +"""Protect root dependencies and retain removed import destinations. + +Revision ID: n5h6i7j8k901 +Revises: m4g5h6i7j890 +""" + +from __future__ import annotations + +import sqlalchemy as sa + +from alembic import op + +revision = "n5h6i7j8k901" +down_revision = "m4g5h6i7j890" +branch_labels = None +depends_on = None + +_TABLES = ("library_files", "series", "story_arcs", "story_arc_placements", "import_jobs") +_NAMING = {"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s"} +_SQLITE_INLINE_KEYS = { + "import_jobs": {"story_arc_rollback_waiting_work_id": "story_arc_sync_work"}, + "story_arcs": { + "source_import_job_id": "import_jobs", + "target_library_root_id": "library_roots", + }, +} +_STORY_ARC_INLINE_CHECKS = { + "source_kind": "VARCHAR(9) NOT NULL DEFAULT 'legacy' CONSTRAINT storyarcsourcekind " + "CHECK (source_kind IN ('legacy','pullbox','mylar3','folder','comicinfo','provider'))", + "lifecycle": "VARCHAR(8) NOT NULL DEFAULT 'active' CONSTRAINT storyarclifecycle " + "CHECK (lifecycle IN ('active','archived'))", +} + + +def _restore_inline_keys(bind: sa.Connection, table: sa.Table) -> None: + """Restore ADD COLUMN FKs expected by immutable older SQLite downgrades.""" + definitions = { + column: f'INTEGER REFERENCES "{target}"(id) ON DELETE SET NULL' + for column, target in _SQLITE_INLINE_KEYS.get(table.name, {}).items() + } + if table.name == "story_arcs": + definitions.update(_STORY_ARC_INLINE_CHECKS) + for column, definition in definitions.items(): + if column not in table.c: + continue + indexes = [index for index in table.indexes if column in index.columns] + bind.exec_driver_sql( + f'CREATE TEMP TABLE root_removal_refs AS SELECT id, "{column}" FROM "{table.name}"' + ) + bind.exec_driver_sql("CREATE UNIQUE INDEX root_removal_refs_id ON root_removal_refs(id)") + for index in indexes: + index.drop(bind) + op.drop_column(table.name, column) + bind.exec_driver_sql(f'ALTER TABLE "{table.name}" ADD COLUMN "{column}" {definition}') + bind.exec_driver_sql( + f'UPDATE "{table.name}" SET "{column}" = (SELECT "{column}" FROM root_removal_refs ' + f'WHERE root_removal_refs.id = "{table.name}".id)' + ) + bind.exec_driver_sql("DROP TABLE root_removal_refs") + for index in indexes: + index.create(bind) + + +def _change_constraints(*, upgrading: bool) -> None: + bind = op.get_bind() + # Batch recreation with enforcement enabled can cascade into child tables. + if bind.dialect.name == "sqlite" and bind.exec_driver_sql("PRAGMA foreign_keys").scalar(): + raise RuntimeError( + "Run root-removal migration with the standalone Alembic connection (foreign_keys=OFF)." + ) + for table in _TABLES: + metadata = sa.MetaData(naming_convention=_NAMING) + reflected = sa.Table(table, metadata, autoload_with=bind) + if bind.dialect.name == "sqlite": + # SQLAlchemy's SQL-text reflection can lose ON DELETE on columns + # originally added with ALTER TABLE. SQLite's own FK list is exact. + actions = { + row[3]: (row[5], row[6]) + for row in bind.exec_driver_sql(f'PRAGMA foreign_key_list("{table}")') + } + for fk in list(reflected.foreign_key_constraints): + if len(fk.columns) == 1: + column = next(iter(fk.columns)).name + fk.onupdate, fk.ondelete = actions[column] + if not upgrading and column in _SQLITE_INLINE_KEYS.get(table, {}): + reflected.constraints.remove(fk) + if not upgrading and table == "story_arcs": + for constraint in list(reflected.constraints): + if isinstance(constraint, sa.CheckConstraint) and constraint.name in { + "storyarcsourcekind", + "storyarclifecycle", + }: + reflected.constraints.remove(constraint) + keys = [ + key + for key in sa.inspect(bind).get_foreign_keys(table) + if key["referred_table"] == "library_roots" + ] + with op.batch_alter_table( + table, + naming_convention=_NAMING, + copy_from=reflected, + recreate="always" if bind.dialect.name == "sqlite" else "auto", + ) as batch: + for key in keys: + columns = key["constrained_columns"] + if ( + not upgrading + and bind.dialect.name == "sqlite" + and columns[0] in _SQLITE_INLINE_KEYS.get(table, {}) + ): + continue + name = key["name"] or f"fk_{table}_{columns[0]}_library_roots" + batch.drop_constraint(name, type_="foreignkey") + batch.create_foreign_key( + name, + "library_roots", + columns, + ["id"], + ondelete="RESTRICT" + if upgrading + else ("CASCADE" if table == "library_files" else "SET NULL"), + ) + if not upgrading and bind.dialect.name == "sqlite": + _restore_inline_keys(bind, reflected) + if bind.dialect.name == "sqlite" and bind.exec_driver_sql("PRAGMA foreign_key_check").first(): + raise RuntimeError("Foreign key validation failed after root-removal migration.") + + +def upgrade() -> None: + _change_constraints(upgrading=True) + op.add_column( + "import_jobs", sa.Column("removed_library_root_snapshot", sa.JSON(), nullable=True) + ) + + +def downgrade() -> None: + if ( + op.get_bind() + .execute( + sa.text( + "SELECT 1 FROM import_jobs WHERE removed_library_root_snapshot IS NOT NULL LIMIT 1" + ) + ) + .first() + ): + raise RuntimeError("Cannot downgrade while import history records removed library roots.") + _change_constraints(upgrading=False) + op.drop_column("import_jobs", "removed_library_root_snapshot") diff --git a/alembic/versions/w8x9y0z1a234_add_import_job_layout_snapshots.py b/alembic/versions/w8x9y0z1a234_add_import_job_layout_snapshots.py new file mode 100644 index 00000000..4f6a6656 --- /dev/null +++ b/alembic/versions/w8x9y0z1a234_add_import_job_layout_snapshots.py @@ -0,0 +1,82 @@ +"""Add durable import-job layout and handling snapshots. + +Revision ID: w8x9y0z1a234 +Revises: v7w8x9y0z123 +Create Date: 2026-08-29 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "w8x9y0z1a234" +down_revision: str | Sequence[str] | None = "v7w8x9y0z123" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_AUTO_LAYOUT_JSON = ( + '{"schema_version":1,"mode":"auto","preset":null,' + '"series_path_template":null,"issue_filename_template":null,' + '"selected_cluster_id":null,"fallback_to_auto":true}' +) + + +def upgrade() -> None: + """Add compatible defaults for existing and newly-created jobs.""" + with op.batch_alter_table("import_jobs") as batch_op: + batch_op.add_column( + sa.Column( + "file_handling_mode", + sa.Enum( + "managed_copy", + "in_place", + name="importfilehandlingmode", + native_enum=False, + create_constraint=True, + ), + nullable=False, + server_default="managed_copy", + ) + ) + batch_op.add_column( + sa.Column( + "source_layout_snapshot", + sa.JSON(), + nullable=False, + server_default=_AUTO_LAYOUT_JSON, + ) + ) + batch_op.add_column( + sa.Column( + "future_layout_requested", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ) + ) + batch_op.add_column(sa.Column("future_root_policy_snapshot", sa.JSON(), nullable=True)) + batch_op.add_column( + sa.Column( + "future_root_policy_applied_at", + sa.DateTime(timezone=True), + nullable=True, + ) + ) + + +def downgrade() -> None: + """Remove durable import-job snapshot fields.""" + with op.batch_alter_table("import_jobs") as batch_op: + batch_op.drop_constraint("importfilehandlingmode", type_="check") + batch_op.drop_column("future_root_policy_applied_at") + batch_op.drop_column("future_root_policy_snapshot") + batch_op.drop_column("future_layout_requested") + batch_op.drop_column("source_layout_snapshot") + batch_op.drop_column("file_handling_mode") diff --git a/alembic/versions/x9y0z1a2b345_add_library_file_storage_ownership.py b/alembic/versions/x9y0z1a2b345_add_library_file_storage_ownership.py new file mode 100644 index 00000000..4e3df5bd --- /dev/null +++ b/alembic/versions/x9y0z1a2b345_add_library_file_storage_ownership.py @@ -0,0 +1,78 @@ +"""Add durable library-file ownership and source signatures. + +Revision ID: x9y0z1a2b345 +Revises: w8x9y0z1a234 +Create Date: 2026-08-29 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "x9y0z1a2b345" +down_revision: str | Sequence[str] | None = "w8x9y0z1a234" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Backfill existing artifacts as managed and add signature storage.""" + with op.batch_alter_table("library_files") as batch_op: + batch_op.add_column( + sa.Column( + "storage_mode", + sa.Enum( + "managed", + "referenced", + name="libraryfilestoragemode", + native_enum=False, + create_constraint=True, + ), + nullable=False, + server_default="managed", + ) + ) + batch_op.add_column( + sa.Column( + "source_signature", + sa.JSON(), + nullable=False, + server_default="{}", + ) + ) + + with op.batch_alter_table("import_files") as batch_op: + batch_op.add_column( + sa.Column( + "source_signature", + sa.JSON(), + nullable=False, + server_default="{}", + ) + ) + + +def downgrade() -> None: + """Remove ownership fields after referenced-row safety is handled externally.""" + referenced_count = ( + op.get_bind() + .execute(sa.text("SELECT COUNT(*) FROM library_files WHERE storage_mode = 'referenced'")) + .scalar_one() + ) + if referenced_count: + raise RuntimeError("Cannot downgrade library-file ownership while referenced files remain.") + + with op.batch_alter_table("import_files") as batch_op: + batch_op.drop_column("source_signature") + + with op.batch_alter_table("library_files") as batch_op: + batch_op.drop_constraint("libraryfilestoragemode", type_="check") + batch_op.drop_column("source_signature") + batch_op.drop_column("storage_mode") diff --git a/alembic/versions/y0z1a2b3c456_add_library_root_policies.py b/alembic/versions/y0z1a2b3c456_add_library_root_policies.py new file mode 100644 index 00000000..f5630628 --- /dev/null +++ b/alembic/versions/y0z1a2b3c456_add_library_root_policies.py @@ -0,0 +1,86 @@ +"""Add complete per-root library naming policies. + +Revision ID: y0z1a2b3c456 +Revises: x9y0z1a2b345 +Create Date: 2026-08-29 +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "y0z1a2b3c456" +down_revision: str | Sequence[str] | None = "x9y0z1a2b345" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Create root policy storage without inventing overrides for existing roots.""" + op.create_table( + "library_root_policies", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("library_root_id", sa.Integer(), nullable=False), + sa.Column("schema_version", sa.Integer(), nullable=False), + sa.Column("series_path_template", sa.String(length=1024), nullable=False), + sa.Column("comic_file_template", sa.String(length=1024), nullable=False), + sa.Column("annual_file_template", sa.String(length=1024), nullable=False), + sa.Column("non_standard_file_template", sa.String(length=1024), nullable=False), + sa.Column( + "single_non_standard_file_template", + sa.String(length=1024), + nullable=False, + ), + sa.Column("replace_illegal_characters", sa.Boolean(), nullable=False), + sa.Column("colon_replacement", sa.String(length=16), nullable=False), + sa.Column( + "source", + sa.Enum( + "global_default", + "import_adoption", + "manual", + name="libraryrootpolicysource", + native_enum=False, + create_constraint=True, + ), + nullable=False, + ), + sa.Column("source_import_job_id", sa.Integer(), nullable=True), + sa.Column("revision", sa.Integer(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["library_root_id"], + ["library_roots.id"], + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["source_import_job_id"], + ["import_jobs.id"], + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("library_root_id"), + ) + + +def downgrade() -> None: + """Remove per-root naming policies and return to global fallback only.""" + op.drop_table("library_root_policies") diff --git a/alembic/versions/z1a2b3c4d567_add_issue_number_text.py b/alembic/versions/z1a2b3c4d567_add_issue_number_text.py new file mode 100644 index 00000000..528432db --- /dev/null +++ b/alembic/versions/z1a2b3c4d567_add_issue_number_text.py @@ -0,0 +1,154 @@ +"""Add exact issue-number compatibility storage. + +Revision ID: z1a2b3c4d567 +Revises: y0z1a2b3c456 +Create Date: 2026-08-30 +""" + +from __future__ import annotations + +from decimal import Decimal, InvalidOperation +from typing import TYPE_CHECKING + +import sqlalchemy as sa + +from alembic import op + +if TYPE_CHECKING: + from collections.abc import Sequence + +revision: str = "z1a2b3c4d567" +down_revision: str | Sequence[str] | None = "y0z1a2b3c456" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_BATCH_SIZE = 2_000 +_MAX_TEXT_LENGTH = 320 + + +def _format_issue_number(value: object) -> str: + """Frozen migration formatter for legacy numeric issue values.""" + try: + decimal_value = Decimal(str(value)) + except InvalidOperation: + raise RuntimeError("Cannot backfill an invalid issue number.") from None + if not decimal_value.is_finite(): + raise RuntimeError("Cannot backfill a non-finite issue number.") + if decimal_value == 0: + return "0" + rendered = format(decimal_value, "f") + if "." in rendered: + rendered = rendered.rstrip("0").rstrip(".") + if not rendered or len(rendered) > _MAX_TEXT_LENGTH: + raise RuntimeError("Cannot backfill an unsupported issue number.") + return rendered + + +def _backfill_issue_number_text() -> None: + connection = op.get_bind() + last_id = 0 + select_batch = sa.text( + "SELECT id, issue_number FROM issues WHERE id > :last_id ORDER BY id LIMIT :batch_size" + ) + update_row = sa.text( + "UPDATE issues SET issue_number_text = :issue_number_text WHERE id = :issue_id" + ) + + while True: + rows = ( + connection.execute( + select_batch, + {"last_id": last_id, "batch_size": _BATCH_SIZE}, + ) + .mappings() + .all() + ) + if not rows: + return + updates = [ + { + "issue_id": int(row["id"]), + "issue_number_text": _format_issue_number(row["issue_number"]), + } + for row in rows + ] + connection.execute(update_row, updates) + last_id = int(rows[-1]["id"]) + + +def _assert_exact_text_is_unique() -> None: + duplicate = ( + op.get_bind() + .execute( + sa.text( + "SELECT series_id, issue_number_text FROM issues " + "WHERE issue_number_text IS NOT NULL " + "GROUP BY series_id, issue_number_text HAVING COUNT(*) > 1 LIMIT 1" + ) + ) + .first() + ) + if duplicate is not None: + raise RuntimeError("Cannot add exact issue-number uniqueness while duplicates remain.") + + +def _assert_downgrade_is_lossless() -> None: + connection = op.get_bind() + last_id = 0 + select_batch = sa.text( + "SELECT id, issue_number, issue_number_text FROM issues " + "WHERE id > :last_id ORDER BY id LIMIT :batch_size" + ) + while True: + rows = ( + connection.execute( + select_batch, + {"last_id": last_id, "batch_size": _BATCH_SIZE}, + ) + .mappings() + .all() + ) + if not rows: + return + for row in rows: + exact_text = row["issue_number_text"] + if exact_text is not None and str(exact_text) != _format_issue_number( + row["issue_number"] + ): + raise RuntimeError( + "Cannot downgrade while divergent exact issue-number text remains." + ) + last_id = int(rows[-1]["id"]) + + +def upgrade() -> None: + """Add nullable exact text, backfill legacy rows, and index ordering.""" + op.add_column( + "issues", + sa.Column("issue_number_text", sa.String(320), nullable=True), + ) + + _backfill_issue_number_text() + _assert_exact_text_is_unique() + + op.create_index( + "uq_series_issue_number_text", + "issues", + ["series_id", "issue_number_text"], + unique=True, + ) + op.create_index( + "ix_issues_series_number_order", + "issues", + ["series_id", "issue_number", "issue_number_text", "id"], + unique=False, + ) + + +def downgrade() -> None: + """Drop exact text only when doing so cannot erase divergent semantics.""" + _assert_downgrade_is_lossless() + + op.drop_index("ix_issues_series_number_order", table_name="issues") + op.drop_index("uq_series_issue_number_text", table_name="issues") + op.drop_column("issues", "issue_number_text") diff --git a/docker/.env.example b/docker/.env.example index 3a099091..de8864dc 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -97,6 +97,9 @@ PULLBOX_RUNTIME_GID=65532 # PULLBOX_DEBUG=false # PULLBOX_STARTUP_UPDATE_CHECK_ENABLED=true +# Manual empty Story Arc creation (provider/import creation is unaffected) +# PULLBOX_STORY_ARC_MANUAL_CREATE_ENABLED=false + # Embedded comic reader # Set false for an immediate rollback that hides Read and disables reader APIs # without deleting source comics or private resume state. @@ -142,9 +145,15 @@ PULLBOX_RUNTIME_GID=65532 # PULLBOX_NAMING_SERIES_FORMAT="{series} ({year})" # PULLBOX_NAMING_ISSUE_FORMAT="{series} ({year}) #{issue:03d}" -# Import debug controls. These are for troubleshooting only and should stay -# disabled for normal production use. +# Import concurrency. Scan 0 = auto (up to 4 inspectors, capped by visible +# CPU/container and available-memory limits). Use 1 for slow/shared disks or +# explicitly test 2-16; an explicit value never exceeds the resource ceiling. +# Inspection never changes file-safety policy. Step 4 workers remain separately +# budgeted because conversions can require substantial temporary disk space. +# PULLBOX_IMPORT_SCAN_WORKER_COUNT=0 # PULLBOX_IMPORT_FILE_WORKER_COUNT=2 + +# Import debug controls. These should stay disabled for normal production use. # PULLBOX_IMPORT_DEBUG_SLOW_MODE=false # PULLBOX_IMPORT_DEBUG_PHASE_DELAY_SECONDS=1.25 # PULLBOX_IMPORT_DEBUG_ITEM_DELAY_SECONDS=0.4 diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index cd3b3488..8860ffc8 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -41,6 +41,7 @@ services: PULLBOX_TEMP_DIR: /data/tmp PULLBOX_BACKUP_DIR: /data/backups PULLBOX_READER_ENABLED: ${PULLBOX_READER_ENABLED:-true} + PULLBOX_STORY_ARC_MANUAL_CREATE_ENABLED: ${PULLBOX_STORY_ARC_MANUAL_CREATE_ENABLED:-false} PULLBOX_BIND_ADDRESS: 0.0.0.0 PULLBOX_PORT: 8585 # Native HTTPS uses the normal Pullbox port; remap the host port above if needed. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 6eb41cc7..7c76a285 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -42,6 +42,7 @@ services: - PULLBOX_LIBRARY_ROOT=/comics - PULLBOX_COVERS_DIR=/comics/.covers - PULLBOX_READER_ENABLED=${PULLBOX_READER_ENABLED:-true} + - PULLBOX_STORY_ARC_MANUAL_CREATE_ENABLED=${PULLBOX_STORY_ARC_MANUAL_CREATE_ENABLED:-false} # Optional overrides (config.xml handles the rest): # - PULLBOX_PORT=8585 # - PULLBOX_BASE_URL=https://comics.example.com diff --git a/docs/development/ARCHITECTURE_OVERVIEW.md b/docs/development/ARCHITECTURE_OVERVIEW.md index b2c72592..b9949ba7 100644 --- a/docs/development/ARCHITECTURE_OVERVIEW.md +++ b/docs/development/ARCHITECTURE_OVERVIEW.md @@ -316,7 +316,8 @@ session and WebSocket lifecycle are supervised per exact configured client. **Current repo nuances** - Prowlarr-synced Torznab indexers are aggregated through a single Prowlarr - search path. + search path. Persisted indexer IDs resolve to that aggregate for acquisition, + without duplicating searches or adding manager-owned per-indexer health checks. - Prowlarr-synced Newznab indexers are kept as individual Newznab proxy endpoints because the direct proxy behavior can produce better category and result fidelity. @@ -326,6 +327,14 @@ session and WebSocket lifecycle are supervised per exact configured client. policy and retires missing manager rows instead of deleting their history. - Download clients may be cached across task cycles when config values have not changed. +- HTTP torrent metadata is fetched and validated inside Pullbox, then uploaded + to the selected qBittorrent, Transmission, or Deluge client. This is independent + of browser-resolver opt-in and applies to manual grabs, automatic acquisition, + intervention approval, and retries. Magnets remain URL submissions. Descriptor + fetching is bounded by the configured indexer origin, redirect count, timeout, + response size, and shared bencode validation. A failed fetch never falls back + to forwarding the private indexer URL to a remote client. Legacy HTTP downloads + without an available originating indexer require a new search. - Pullbox Data is not a general metadata proxy. Installed clients default to the public `https://api.pullbox.app` release API, while deployments may use `PULLBOX_DATA_API_BASE_URL` for an intentional private-network override. @@ -342,6 +351,16 @@ Search and acquisition are coordinated through several focused service modules: - `search_indexers.py` calls configured indexers. - `release_parser.py` and matching helpers parse release titles. - `release_validator.py` rejects mismatches and unsafe candidates. +- Date confidence is separate from query identity. Serial queries retain the + series start year, but indexer, direct, and Direct Connect results share + `release_year_matching.py`: known issue cover/store years use the configured + year tolerance; explicitly labeled volume years must match series identity. + Only undated regular issues from a continuing series can use the inclusive + start-year-to-current-year fallback, capped at medium confidence. This uses + existing catalog data without extra metadata requests and never overrides + series, issue-number, or issue-type mismatches. Search diagnostics record the + date-evidence basis. Bare four-digit issues require a known series prefix so + numeric titles such as `Marvel 1602` remain intact. - `search_scoring.py` and `search_evaluation.py` rank acceptable releases. The configured Search Priority orders the `usenet`, `torrent`, and `direct` source lanes. Within each lane, the deterministic quality score includes the @@ -393,6 +412,17 @@ Wanted issue or manual search - Search behavior is intentionally split into smaller modules so parsing, validation, scoring, indexing, and orchestration can be tested separately. - Manual search may fan out more broadly than automated wanted search. +- Single-issue automatic search and scheduled search both attach opted-in + AirDC++ candidates before shared source ranking. The handoff rechecks the exact + client's enabled/search/automatic flags and readiness. High-confidence results + use the durable AirDC++ acquisition service; lower-confidence routes are stored + encrypted in intervention and can be approved after a restart. Active issue + downloads suppress another queue mutation, and ambiguous responses remain owned + by reconciliation instead of falling through to a second source. +- Search history counts validation rejections independently of acquisition. + Exhausted source handoffs report `source_unavailable`, not `no_results`, with + safe provider notices. SABnzbd NZB retrieval has a 60-second read timeout and + 90-second total deadline; ordinary client API calls retain their 10-second timeout. - Search Wanted snapshots every eligible Wanted issue, processes at most 100 per batch, and resumes the same restart-safe sweep hourly until complete. Never-searched issues run first, followed by least-recently searched issues; @@ -437,12 +467,56 @@ Step 3 is the active decision point: - Matched and In Library rows can be selected only by the user; state changes must not silently auto-select files for import. +Mylar Step 1 analyzes stored locations before creating a job. Missing folders +inside a visible root are distinct from unmapped prefixes, unreadable paths, +ambiguous roots, and unsafe paths. The preview exposes series identity, stored +and attempted paths, the reason, and a suggested action for every exception. +Its searchable 25-row pages and full JSON export read a saved report instead of +rescanning the filesystem. The most recent report is included in diagnostic +packages even when preflight prevented job creation. + +If some locations are accessible and the only exceptions are missing or +unmapped sources, the user can explicitly acknowledge continuing with available +sources. The acknowledgement is bound to the reviewed exception set and is +revalidated at job creation. Unavailable Mylar records remain review exceptions; +they are not silently dropped or treated as owned files. Permission failures, +unsafe paths, ambiguous roots, unavailable configured roots, and truncated +previews still block starting. Preflight never modifies the Mylar database or +creates, renames, or changes permissions on source folders. + +Both Mylar and folder imports use the same sidecar parser and comic-content +review policy. Mylar's nested `series.json` metadata and explicit `cvinfo` +ComicVine volume URLs are authoritative local evidence; an unqualified +`series_id` is not. Conflicting explicit identities remain review exceptions. +Archive member inventories, not declared ComicInfo page counts or minimum file +sizes, identify metadata-only and possible cover-only CBZ/CBR/CB7/CBT files. +No-page archives cannot be imported; single-page archives require individual +approval without disabling resource or dangerous-file checks. PDF and EPUB do +not use this image-member heuristic. These checks neither extract page payloads +nor fetch provider metadata. + +Affected saved reviews can be rechecked with the offline, dry-run-first +maintenance procedure in [Import Review Recovery](IMPORT_REVIEW_RECOVERY.md). +It does not silently repair existing reviews during an upgrade. + Step 4 is the only place files move into the library. It may reuse Step 2/3 matched summaries, create targeted series and issue rows first, and then mark the issue catalog as hydrating while full ComicVine issue metadata is fetched in the background. This keeps imports responsive without pretending the catalog is complete before hydration finishes. +Completed reference-only imports support two bounded follow-up paths. Exact +ComicInfo or trusted sidecar identity can repair a misplaced file's logical +series and issue ownership without changing its source path. After those +corrections, the operator may create a separate managed-copy import from the +completed results screen. That job reuses the reviewed issue identity, applies +the destination root's current naming, conversion, and ComicInfo policy, and +replaces each old referenced registration only after its managed copy is +published. Its signed preview covers the exact source lineage and non-overlapping +target root. Rollback restores the original reference and removes only a +verified unchanged managed artifact; it never deletes or renames the Mylar +source. + **Required standard** - Preserve matching quality before optimizing import speed. diff --git a/docs/development/DATABASE_STANDARDS.md b/docs/development/DATABASE_STANDARDS.md index 8070c1b1..b011280e 100644 --- a/docs/development/DATABASE_STANDARDS.md +++ b/docs/development/DATABASE_STANDARDS.md @@ -532,6 +532,14 @@ Pullbox also includes sidecar recovery logic for stale or corrupt `-wal` and ### 5.2 Maintenance Coordination +Database-size health thresholds allow for large collections and retained logs: +the size sub-check reports the current file size as informational data. Database +health is determined by integrity, representative latency, bloat, and available +storage rather than a fixed size threshold. The file-size observation does not +impose a storage limit or an automatic cleanup policy. Disk-space, integrity, query +latency, and database-bloat checks remain independent; a smaller database does +not suppress failures in those checks. + **Current Pullbox implementation** - Database maintenance windows coordinate app traffic through the shared diff --git a/docs/development/DEPENDENCY_AUDIT_EXCEPTIONS.md b/docs/development/DEPENDENCY_AUDIT_EXCEPTIONS.md new file mode 100644 index 00000000..eb542036 --- /dev/null +++ b/docs/development/DEPENDENCY_AUDIT_EXCEPTIONS.md @@ -0,0 +1,76 @@ +# Dependency Audit Exceptions + +This is a risk-acceptance record, not a claim that an upstream vulnerability +has been fixed. All unexcepted pip-audit findings and audit collection failures +remain blocking. Safety and Bandit retain their existing advisory status. + +## NLTK Used By Safety + +| Field | Reviewed scope | +| --- | --- | +| Package | `nltk==3.10.3` | +| Development tool | `safety==3.8.1` | +| Advisory | `PYSEC-2026-3740`, `GHSA-8mgp-746c-j5xp`, `CVE-2026-81726` | +| Approval | Maintainer-approved temporary development-toolchain acceptance | +| Review date | 2026-09-03 UTC (2026-09-02 local development date) | +| Expires | **2026-10-03 at 00:00 UTC**, exclusive; no automatic renewal | +| Owner | Pullbox maintainer | +| Production applicability | None; production image checks reject Safety or NLTK | + +### Exposure And Rationale + +The [NLTK advisory](https://github.com/nltk/nltk/security/advisories/GHSA-8mgp-746c-j5xp) +covers model-artifact APIs that bypass path containment when callers supply +untrusted filesystem paths. At review time, the maintainer advisory lists +versions through 3.10.3 as affected and no patched version. Do not change feeds +or assume 3.10.3 is fixed merely because a secondary database disagrees. + +Safety 3.8.1 imports NLTK in `safety/tool/typosquatting.py` and calls +`nltk.edit_distance()` on package names. The reviewed flow does not use the +vulnerable model read/write APIs. Pullbox has no NLTK application imports. +Safety and NLTK are development dependencies; the production Docker build +installs `.[prod]` into a fresh environment, not `.[dev]`. The local production +image was also checked for both packages' absence during review. + +Development-only does not mean harmless or sandboxed: an added model-path +consumer could expose developer/runner filesystem access. This exception +accepts the presently reviewed exposure, not arbitrary future use. + +### Enforcement And Evidence + +- Both `scripts/security_check.sh` and `.github/workflows/security.yml` call + `scripts/run_dependency_audit.py`, using the actual selected Python environment. +- The wrapper runs pip-audit with `--strict`, explicit PyPI service, aliases, + descriptions, and JSON output. It does **not** pass a global NLTK ignore flag. +- The complete pinned `pip freeze` inventory includes transitive dependencies. + `--no-deps --disable-pip` audits those exact versions directly instead of + recreating an environment whose preinstalled packaging tools can be omitted + from pip's installation report. This does not exclude transitive packages. +- Only this finding for NLTK 3.10.3 alongside Safety 3.8.1 can be accepted, + before the fixed expiry and while neither is declared in runtime dependency + groups. A changed version, newly reported fixed version, or other finding + blocks the gate. Non-development optional dependency groups are checked too. +- Invalid, skipped, duplicate, empty, stale, partial, or inconsistent scanner + evidence cannot turn a failed audit into success. The scanned inventory must + include every package at its exact frozen version in the requirements export. + Expiry is checked when the completed audit is evaluated, not when it starts. +- `dependency-audit-report.json` retains the raw report, including this finding; + GitHub uploads it even when the audit fails. Logs explicitly identify accepted + findings, expiry, and any remaining blocking findings. +- `scripts/verify_container_security_runtime.py` independently checks actual + production images for Safety/NLTK absence before accepting image validation. +- The pre-existing Pygments `CVE-2026-4539` exception is unchanged. + +### Removal And Re-Review + +Upgrade when an upstream fix is available and remove the NLTK exception after +verifying the actual toolchain. Re-review any Safety/NLTK version change, +application dependency/use change, or new model-artifact consumer. At expiry, +an affected toolchain fails again until upgraded or explicitly re-reviewed; +do not extend the date merely to obtain green CI. A clean audit with no +remaining NLTK finding continues to pass after expiry. + +Regression coverage lives in `tests/unit/test_dependency_audit_policy.py`, +`tests/unit/test_verify_container_security_runtime.py`, and the shared-workflow +contracts. Run those first, then `make security-check`, `make workflow-hygiene`, +and `make ci-full` before merging. diff --git a/docs/development/DEVELOPMENT_IMAGES.md b/docs/development/DEVELOPMENT_IMAGES.md new file mode 100644 index 00000000..fefb52dc --- /dev/null +++ b/docs/development/DEVELOPMENT_IMAGES.md @@ -0,0 +1,154 @@ +# Signed development images + +The opt-in `edge` channel packages selected, validated commits from `develop`. +It is not a general-availability release and does not update `latest`, create a +version tag, or create a GitHub Release. Builds are manual; merging to `develop` +does not automatically run full validation or publish an image. + +This workflow change must be merged into `develop` before using this runbook. +Local workflow tests do not demonstrate that an image has been published. + +## Maintainer publication + +1. Select a reviewed commit on `develop`. The application version must end in + `-dev`. Coordinate merges while validating so all four runs test the same SHA. +2. Explicitly run the existing full validation workflows against `develop`: + + ```bash + gh workflow run ci.yml --ref develop + gh workflow run security.yml --ref develop + gh workflow run workflow-hygiene.yml --ref develop + gh workflow run docker-validate.yml --ref develop + ``` + + These are full runs, including the Python matrix, browser/accessibility + tests, migration checks, security checks, and trusted production Docker + validation. They consume CI resources. The image workflow never dispatches + these checks on its own. + +3. Wait for all four to complete successfully and confirm they tested the exact + same commit that is still selected on `develop`. For each workflow, inspect + its manual runs, for example: + + ```bash + gh run list --workflow ci.yml --branch develop --event workflow_dispatch \ + --json databaseId,headSha,status,conclusion + ``` + +4. Explicitly publish that validated development snapshot: + + ```bash + gh workflow run docker-release.yml --ref develop + ``` + + There is no custom tag input. A manual run on a feature branch, `main`, or a + version tag is rejected before checkout and before using a self-hosted Docker + runner. Stable releases still use the existing signed version-tag push flow. + +5. Wait for the complete Docker Release workflow, including signature + verification and promotion, to succeed. Share its run URL, source SHA, and + published digest with testers. The `release-image-digest` artifact contains + `digest.txt`; `tag.txt` is intentionally empty for a development build. + +The development gate reads GitHub Actions evidence with read-only `actions` and +`checks` permissions. For each of the four workflows it requires the latest +manual `develop` run for the exact SHA, from this repository, with trusted +GitHub Actions check-suite provenance. The workflow must have succeeded and its +required jobs and meaningful steps must have completed successfully. PR +aggregates, preflight-only checks, release-sync shortcuts, skipped jobs, older +successful runs superseded by failures, and another commit's checks do not +qualify. Missing or malformed API evidence blocks publication. + +The existing advisory Bandit findings policy remains advisory, but Bandit must +actually run and upload its report. The development channel requires successful +whole workflow runs: an informational CodeQL job failure may therefore prevent +publication even when the aggregate Security Required check is green. + +If `develop` advances between validation and dispatch, validate the newly +selected commit before retrying. The exact-SHA gate is also checked again before +promoting `edge`, so a new failed or unfinished validation run blocks promotion. + +For a failed development image run, use **Re-run all jobs** or start a new manual +dispatch after resolving the failure. Do not selectively rerun a platform build, +failed jobs, or the signing/promotion job: GitHub retains earlier preparation +outputs during partial reruns, which could otherwise reuse an old build tag for +a newly rebuilt digest. The platform jobs and final promotion reject any +preparation tag that does not match the current run attempt. A full rerun creates +fresh preparation metadata and a distinct build-attempt tag. + +## Tags, digests, and signatures + +Both registries carry the same multi-platform digest: + +- `ghcr.io/pullboxapp/pullbox` +- `docker.io/pullbox/pullbox` + +Development publication uses three kinds of reference: + +| Reference | Purpose | +| --- | --- | +| `edge` | Rolling opt-in channel, updated only after both registry signatures verify. | +| `sha--run--` | Unique build reference; a new build or attempt gets a new tag instead of replacing a plain SHA alias. | +| `@sha256:` | Definitive immutable artifact identity; preferred for reproducible tester reports. | + +The builders first publish untagged platform digests. After Grype, smoke, and +runtime checks, the manifest job creates `candidate--` staging +tags. These staging tags are not the tester channel and may be unsigned if a +later step fails. The signing job signs and verifies the multi-platform digest +in both registries, then promotes that exact index to the development tags +without rebuilding it. SBOM/provenance attestations remain attached. + +Run-specific tags avoid accidentally reusing a commit tag when rebuilding the +same source. This is not a claim of bit-for-bit reproducible builds or registry +enforced tag immutability: timestamps, build provenance, dependencies, or base +image updates can change the digest. Pin the digest to reproduce the exact +artifact. Staging references are retained on failure for investigation; registry +cleanup is a separate deliberate maintenance operation. + +Verify the digest before running it, substituting the digest reported by the +successful workflow: + +```bash +cosign verify \ + --certificate-identity 'https://github.com/pullboxapp/pullbox/.github/workflows/docker-release.yml@refs/heads/develop' \ + --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \ + 'ghcr.io/pullboxapp/pullbox@sha256:' +``` + +For Docker Hub, use the same command with +`docker.io/pullbox/pullbox@sha256:`. Signing establishes artifact origin +and integrity, not stability or freedom from bugs. Docker does not automatically +enforce this Cosign verification when pulling an image. + +Promotion is not a cross-registry transaction. If the second registry update +fails, one registry's `edge` may advance while the other remains on the previous +snapshot; the workflow fails and both candidate signatures have already been +verified. Do not advertise the build as successfully published until the entire +workflow is green. Investigate and use **Re-run all jobs** or a new manual +dispatch; do not bypass verification or manually point `edge` at an unsigned +candidate. + +## Tester isolation and rollback + +Use a separate development container with separate host port, appdata/database, +library, downloads, and import-source copies. Do not share a production `/data` +mount or let development automation modify the production library or downloader +queue. Use a dedicated downloader category/destination if testing acquisition. + +Prefer pinning the supplied digest over following `edge` with an automatic +container updater. Keep the source SHA, image digest, architecture, and run URL +with each test report. Before upgrading a test instance, take a consistent full +appdata backup (including `config.xml`) and protect any test files you need. + +Database migrations run at startup. Changing the image back to a GA tag does +not roll the database or filesystem back. Restore the matching pre-upgrade +appdata backup and necessary files into the isolated test environment, or start +again with fresh test state. Do not attach a development-migrated database to an +older production image. + +## Verification references + +- [GitHub workflow-run API](https://docs.github.com/en/rest/actions/workflow-runs) +- [GitHub job evidence for a specific run attempt](https://docs.github.com/en/rest/actions/workflow-jobs#list-jobs-for-a-workflow-run-attempt) +- [Docker manifest-index copying](https://docs.docker.com/reference/cli/docker/buildx/imagetools/create/) +- [Cosign verification](https://docs.sigstore.dev/cosign/verifying/verify/) diff --git a/docs/development/IMPORT_PERFORMANCE.md b/docs/development/IMPORT_PERFORMANCE.md new file mode 100644 index 00000000..7de632bb --- /dev/null +++ b/docs/development/IMPORT_PERFORMANCE.md @@ -0,0 +1,82 @@ +# Import Performance + +## Runtime Contract + +Mylar and folder imports share bounded, read-only archive inspection. A worker +receives paths and immutable safety settings, never an AsyncSession. Completion +and diagnostic updates flow through one coordinator. Review file rows are +inserted in batches of at most 500 in the existing transaction; source pages +retain their durable checkpoints. A singleton issue group cannot contain a +duplicate or conflict, so it no longer loads files and commits just to check. +Archive page-name matching runs off the event loop and uses a bounded local +cache for repeated page-title parsing. It still counts every page toward +consensus and does not reuse safety decisions between scans. + +Completed-import source recovery inspects each bounded page before its database +mutation phase, then commits before inspecting the next page. Slow files +therefore do not hold SQLite's writer lock, and results do not accumulate in an +unbounded in-memory collection. +Deferred catalog recovery publishes live provider progress without rewriting +the full recovery snapshot, then writes one JSON-safe checkpoint after each +completed catalog. + +`PULLBOX_IMPORT_SCAN_WORKER_COUNT=0` selects automatic inspection concurrency, +up to four workers. CPU affinity, cgroup v2 CPU quotas and parent limits, +cgroup memory headroom, and OS available memory cap the budget. Common cgroup +v1 mounts are supported too. Missing resource information falls back +conservatively. Explicit values 1-16 are also capped. The budget is reevaluated +between batches; this is not a throughput-learning autotuner or a guarantee +against other host workloads consuming resources. + +The resource ceiling reserves at least one CPU (25% on larger machines) and +512 MiB of available memory, with an additional 512 MiB allowance per inspector. +Automatic mode deliberately does not saturate high-core machines: local ZIP +header parsing did not improve with more than 2-4 workers. Docker Desktop's +VM resources are the relevant limits, not the Mac's advertised RAM. + +Step 4 keeps `PULLBOX_IMPORT_FILE_WORKER_COUNT=2` and its existing temporary-space +preflight, target-collision serialization, per-worker sessions, and rollback +journal. It now bounds submitted tasks as well as active workers, rather than +creating one waiting task per file. Exiting or canceling either worker pool +drains active work before the job can transition; no orphan filesystem work +may continue after cancellation is reported complete. + +## Progress and Evidence + +Unknown inventory totals are indeterminate. Completed series report 100% for +the current item, independent of overall phase weights. The browser must not +invent an ETA when the backend reports an unknown estimate. Matching emits +lightweight completion updates between durable checkpoints without adding a +database commit per item. + +`import_archive_inspection_batch` reports actual workers, effective CPUs, +available memory, files checked, and elapsed milliseconds. Mylar and folder +batch events separately report inspection and persistence durations; existing +Step 2 timing events retain discovery/matching/total durations. These metrics +are observations, not substitutes for transaction-wait or storage profiling. +Inspection batch wall time includes policy reads, reconciliation, and progress +callbacks. Persistence timing covers row materialization/flush, not the later +checkpoint commit. Do not interpret either value as exclusive disk I/O time. + +## Reproducible Benchmarks + +```bash +.venv/bin/python scripts/benchmark_import_scan.py \ + --series-count 100 --files-per-series 12 --trusted-comicinfo \ + --archive-pages 32 --inspection-workers 4 +``` + +The benchmark creates an isolated temporary source tree and database and makes +no external provider calls. Repeat with workers 1, 2, 4, 8, and 16; use medians +and retain the effective worker count, not only the requested count. Compare +identical matched, blocked, conflict, and missing-file outcomes before speed. + +`--inspection-delay-ms 10` adds controlled per-archive latency to evaluate I/O +overlap. It is a simulation, not a NAS measurement. Archive fixtures exercise +member indexes and bounded ComicInfo reads, not full image decoding or physical +multi-gigabyte payloads. Run representative CBR/conversion workloads and real +storage samples before claiming a user's end-to-end speedup. + +Do not add persistent cross-scan safety caches, speculative provider concurrency, +or process pools without evidence and new invalidation/recovery tests. Existing +compact archive metadata reuse remains intact; safety is freshly evaluated. diff --git a/docs/development/IMPORT_REVIEW_RECOVERY.md b/docs/development/IMPORT_REVIEW_RECOVERY.md new file mode 100644 index 00000000..77943e6d --- /dev/null +++ b/docs/development/IMPORT_REVIEW_RECOVERY.md @@ -0,0 +1,409 @@ +# Recheck A Saved Import Review + +## Guided Import Contract + +Collection imports use the same five stages for Mylar and folder sources: +Source, Analyze, Review, Import, and Finish. The normal path is intentionally +task-oriented: + +- Step 1 asks for the source and copy versus keep-in-place behavior. Pullbox + automatically uses the sole or default writable managed root for copied + imports. **Where new files go** appears only when there is a real choice or + no writable destination is available; layout overrides and manual path + mappings remain under progressive disclosure. +- Mylar path analysis groups a shared root problem into one actionable card. + When at least one source is available, missing or stale Mylar references are + retained as non-blocking follow-up instead of requiring an acknowledgement. +- Step 3 shows Ready, Needs attention, and Deferred follow-up first. **Import + all ready comics** selects the safe canonical set without requiring the user + to visit every deferred group. Detailed status tables remain available under + **Review details**. +- Trusted, complete Mylar or ComicInfo Story Arc evidence may create a logical + Story Arc automatically. Inferred or incomplete arc evidence is retained for + later review and never blocks canonical comic import. +- Source-mutating actions are not presented during Review. Import Follow-up + owns unresolved matching and cleanup actions. Import History owns the + optional clean-library organizer. +- In-place imports keep existing files associated with their current roots. If + a selected series spans multiple roots, Step 3 requires one writable root for + future downloads and replacements without relocating the existing files. + +Use this maintenance procedure when a review was generated before the Mylar +sidecar parser and comic-content checks were corrected. Normal completed-job +recovery is available in Import Follow-up and does not require an +offline command. It works for Mylar and folder imports. This is not a full +rescan, a database restore, or an import. + +## Import Follow-up + +The Follow-up tab groups actionable work by import job rather than rendering +one flat, cross-import backlog. Selecting an import opens its matching, +recovery, cleanup, Story Arc, failed-file, and exhausted metadata work in one +bounded workspace. Each bulk action has an exact count, three representative +filenames, and an on-demand detail view limited to 25 files per page. Imports +with only successful, duplicate, skipped, or still-hydrating metadata outcomes +do not appear. + +Step 5 remains a concise completion receipt. When follow-up exists, it shows an +above-fold count and a direct link to that import's Follow-up workspace. + +Available recovery actions are intentionally narrow: + +A later task or optional Story Arc failure does not hide these actions after +the canonical import has durably completed. Pullbox also requires the job to be +unarchived, idle, and free of rollback work before exposing or applying any +recovery action. An import that failed before durable completion remains +ineligible. + +- **Dismiss stale Mylar references** marks missing database references skipped. + This includes references confirmed missing by a later source recheck. It does + not delete a review record or touch Mylar's database. +- **Skip one-page archives** excludes one-page image archives while leaving the + source files intact. A one-page archive may be cover art, a damaged archive, + or an intentional one-page comic, so Pullbox does not delete it automatically. +- **Move a reviewed source to Trash** is an individual post-import option for a + confirmed cover or unwanted one-page source. It appears only in Follow-up, + uses a red warning modal and an actor-bound signed preview, and + requires configured Trash plus source write permission. Reference-only Mylar + roots cannot use it; the non-destructive skip remains available instead. +- **Skip unusable files** excludes empty, unsupported, and page-less files, + including those confirmed unusable by a later source recheck. +- **Allow oversized files once** retries only decompression-size blocks marked + overrideable. It does not change the global archive safety policy or approve + dangerous archive content. +- **Retry source inspection** rechecks files that were unreadable, changed, or + temporarily could not be inspected, then resumes only work that now passes. +- **Recognize already-owned issues** clears conflicts whose issue already has a + registered library file. +- **Accept recommended conflict choices** applies only when a conflict group has + exactly one high-confidence preferred file and the issue is not already + owned. Alternatives in that group are skipped and the preferred file alone + is retried. +- **Resolve mixed-folder files** uses exact ComicInfo or trusted sidecar series + and issue identity to correct files assigned to the wrong imported series or + issue. Ambiguous titles, filename-only guesses, conflicting target files, + stale references, and managed files remain review-only. The source path and + source artifact are never changed. +- **Recover known series** re-evaluates legacy series-level identity rejections + using saved evidence, without a new scan or provider requests. A unique Mylar + series ID, or a retained trusted folder/ComicInfo series match, can restore + only files whose saved series and issue identities agree. Existing catalog + ownership, issue numbers, per-file conflicts, manual decisions, skips, and + safety blocks are checked before an actor-bound preview is issued. Ambiguous + duplicate candidates and a series containing an unpreviewed ready file are + excluded. The confirmed scope runs through normal background Step 4 source + validation and import rules. Successful files and source paths are untouched; + unresolved files remain in Follow-up. This is not a blanket repair of stale + IDs or a replacement for manual review when trusted evidence disagrees. +- **Recheck deferred files** checks the remaining unmatched files in a resumable + background pass. The preview counts distinct physical paths; its signed scope + still includes every underlying record. Repeated records are consolidated + only when their path, size, portable timestamp, and available content digests + agree. The retained record links back to every superseded record. Files already + registered at the same path and exact issue are recognized without importing + them again. A different file for an owned issue remains a review decision; + it is never automatically substituted for the owned copy. + +The deferred pass uses complete local catalogs first. Exact issue identity may +correct stale Mylar ownership only when the file's title, issue number, type, +and embedded identity agree with the target. Conflicting embedded IDs remain +blocked. Filename-only recovery additionally requires an exact series title or +alias, one issue target, matching issue type, a publication year within one year +of the target issue date, and no pack, volume, or identity conflict. It does not +relax the ordinary search matcher. + +Missing candidate catalogs are fetched once per candidate series per pass, not +once per file. Only trusted saved Mylar, ComicInfo, and sidecar series IDs are +candidates; membership in one catalog plus agreeing file evidence is required +before staging a target. Completed catalog checks are checkpointed. A provider +failure pauses the pass, and Resume continues without repeating completed +checks. Network requests do not hold a database write transaction. +Catalog summaries are converted to JSON-safe checkpoint payloads before they +are stored. Live provider progress does not rewrite the full durable recovery +snapshot; each completed catalog produces one durable checkpoint, so a worker +restart resumes after the last completed catalog without replaying it. + +Recovered files run through normal Step 4 safety, current-source validation, +ownership checks, and the original copy or keep-in-place settings. Only newly +prepared recovery groups execute, not unrelated ready files or Story Arcs. +Cancellation stops this pass without rolling back the original import or +completed recovery files. Manual choices, skips, ambiguous targets, and safety +blocks remain protected. Empty missing-location groups with no file records +are archived with their evidence retained. These rules are shared by Mylar and +folder imports; no source file is moved, renamed, or deleted by reconciliation. + +For older jobs, **Retry failed** first repairs terminal bookkeeping before it +retries file work. A series with any successfully imported file retains its +imported outcome even when another file failed. A series that failed because it +has no ComicVine identity moves to Follow-up instead of being retried without a +target. Files that previously failed to resolve to a library issue are retried +only when their saved identity resolves to one unambiguous issue in the known +Pullbox series; contradictory provider IDs fail closed. Every other unresolved +target becomes an explicit Follow-up decision. Prior errors remain in +diagnostics, counters are rebuilt, and source files remain unchanged. A +status-only correction does not launch another import. + +A completed source recheck reports a file ready only after both archive safety +and saved target identity checks pass. Missing, empty, or otherwise blocked +sources are counted as blocked even when the archive-level inspection itself +completed successfully. +Completed-import source rechecks inspect one bounded page before writing its +refreshed evidence, then commit that page before reading more archives. This +keeps slow archive I/O outside SQLite's single-writer window, bounds memory, and +leaves completed pages durable if a later source needs another attempt. + +Recovery queries must not expand an entire library into SQL bind parameters. +Mixed-folder lookups join existing references and discard exact same-title +rows before loading archive diagnostics; the final shared identity rules still +decide eligibility. Known-series recovery batches catalog lookups and fails +closed if saved or current ownership evidence changes after preview. + +During a Mylar scan, Pullbox can also reconcile one stale recorded path with a +file found in another series folder when the embedded ComicInfo issue ID is an +exact match. If Mylar's issue ID is stale, Pullbox can use a stricter fallback +only when the embedded ComicInfo series ID, series title, issue number, and +issue ID are trusted; the recorded filename is exact; and the missing record +and candidate are both unique. The real embedded issue ID is preserved. Pullbox +links one canonical file in place for import and classifies only byte-identical +extra copies from the exact-ID path as duplicates. Filename guesses, ambiguous +records, conflicting embedded identity, and non-identical candidates remain +untouched for review. + +Follow-up keeps the optional physical cleanup separate from the safe +recovery actions above: + +- **Move misplaced file** moves one canonical file to the exact missing path + already recorded by Mylar. It requires an empty destination, revalidates the + source fingerprint, and updates the Pullbox reference and rollback journal + together. Its signed confirmation is a one-time authorization for that exact + move even when the root is otherwise reference-only; the root's permanent + managed-write policy and Mylar's database remain unchanged. +- **Move all verified files** applies the same checks to the complete current + exact-identity scope. The signed preview covers the scope digest and actor; + changed, ambiguous, occupied, or inaccessible candidates remain untouched. +- **Move duplicate to Trash** is a separate per-file choice available only for + an extra copy that remains byte-identical to its canonical issue. It requires + a configured Trash folder and managed-write permission. Pullbox never removes + these copies automatically. + +All actions use an actor-bound signed preview and restore the physical source +if their database update cannot commit. A reference-only Mylar root remains +non-destructive during import and does not become managed after an explicitly +confirmed cleanup move. + +Every mutation requires a fresh, actor-bound signed preview. Pullbox rejects an +expired preview or any action whose row set changed after preview. Bounded, +recoverable actions use a normal confirmation; typed confirmation remains +reserved for permanent deletion. Mutations run in bounded database pages, +recompute import counters, and create import and security audit records. +Dangerous, unknown, and genuinely ambiguous outcomes remain manual-review +items. + +Once cleanup is complete, **Archive results** hides the finished job from the +current history view without deleting its rows, logs, decisions, or rollback +evidence. Archived jobs can be restored later. **Clear History** never deletes +archived jobs. + +## Building A Clean Pullbox Library + +A completed reference-only import can be used as the reviewed source for a +separate clean managed library. Import History exposes this optional organizer +for eligible jobs. Its modal shows the exact file count, series count, and +source size, then requires the operator to choose an enabled writable library +root that does not overlap any source root. + +The clean-library build creates a normal background Step 4 import. It copies +only files with an exact, current imported-file to library-file to issue +lineage. The destination root's current folder naming, file naming, CBZ +conversion, and ComicInfo policy are applied. Existing skip-existing behavior +does not prevent this explicit adoption, because each verified source reference +is being replaced by its newly managed Pullbox copy. + +The operation is intentionally source-preserving: + +- The Mylar database, folders, filenames, permissions, and file content remain + unchanged. +- The destination must have sufficient capacity under the ordinary managed-copy + preflight, including conversion workspace reserve when conversion is enabled. +- The preview token is actor-bound, expires after 15 minutes, and covers the + exact source rows and destination root. Changed scope requires a new preview. +- Each completed placement records the prior reference and source identity. + Rollback removes only an unchanged Pullbox-managed destination, restores the + original referenced library record, and leaves the Mylar file in place. +- A missing or changed source, occupied old identity/path, stale database + lineage, or modified managed destination fails closed and preserves the clean + managed file for review. + +Validate the managed library before disabling or retiring the legacy Mylar +root. Source retirement is a separate operator decision; this workflow never +deletes the legacy tree automatically. + +## Safety And Scope + +- Stop Pullbox and back up its database before running the command. `--offline` + is the operator's acknowledgement, not an automatic container stop. +- The job must be idle at Step 3 (`REVIEW`) or finished (`COMPLETED`). Do not + run this against a scan, import, rollback, or job with a pending control + request. +- For a `REVIEW` job, only automatically rejected series with + `trusted_source_identity_conflict` are examined by default. Repeat + `--series-id` to narrow the operation to specific **import-review series + IDs**, not ComicVine or library series IDs. +- For a `COMPLETED` job, prefer the matching in-app recovery action. **Retry + failed** remains available for ordinary import failures; **Retry source + inspection** handles completed safety rows whose source should be checked + again. +- An entire series is left untouched if it has manual overrides, selected + files, explicit skips, approved exceptions, or other completed file decisions. + The report counts these as `skipped_series`; discuss them individually. +- `--source-root` must name a specific directory visible inside the container. + Repeat it for multiple mounts. Both lexical and resolved paths must remain + inside an explicitly permitted root; traversal, sensitive paths, and escaping + symlinks are not accepted. +- No sources, Mylar databases, ComicInfo files, or library files are changed. + The command inspects only saved candidates, using bounded database pages and + one folder-sidecar read per folder in a series. It makes no provider requests. +- This is deliberately transactional. Without `--apply` no changes persist. + With `--apply`, failure before the final commit rolls back the recheck. + +## Docker Compose Example + +Replace `pullbox.yml`, the service name, job ID, and source root with the actual +deployment values. The image must contain this command. Keep all existing data, +source, and config mounts on the one-off maintenance container. + +```bash +docker compose -f pullbox.yml stop pullbox +``` + +Back up the stopped instance's database with the deployment's normal backup +procedure. Keep any SQLite WAL sidecars together with a filesystem backup; +do not copy a database file alone while writers are running. + +Preview first: + +```bash +docker compose -f pullbox.yml run --rm --no-deps --entrypoint python pullbox \ + -m pullbox.cli recheck-import --job 1 --source-root /mnt/comics --offline +``` + +For a Step 3 review, the JSON result reports `series_prepared`, +`files_checked`, `blocked_files`, and `skipped_series`. For a completed import, +it reports `files_prepared`, `files_checked`, `blocked_files`, and +`skipped_files`. `applied: false` confirms it was only a preview. Review the +counts before running the same command with `--apply`: + +```bash +docker compose -f pullbox.yml run --rm --no-deps --entrypoint python pullbox \ + -m pullbox.cli recheck-import --job 1 --source-root /mnt/comics --offline --apply +docker compose -f pullbox.yml start pullbox +``` + +The successful command stages affected series for local matching. Normal +startup recovery resumes from `MATCHING`, preserving the directory inventory +and unaffected review decisions, and returns to Step 3. It does not select +files or start Step 4. Genuine source-identity conflicts remain in review. + +For a completed import, restart Pullbox, open that import in Follow-up, and +choose the relevant recovery action. Only the previewed scope is prepared; successful +files and series remain untouched. A file that is still missing, outside an +approved root, unreadable, or unsafe remains excluded with refreshed +diagnostics. + +## Replaced Files + +Changed or missing scan signatures normally remain blocked. After deliberately +replacing a defective file, preview a targeted recheck with +`--accept-replaced-files` and, when useful, one or more `--series-id` filters. +This explicitly accepts new scan evidence only after containment and archive +checks; it is not an archive-safety override. Then repeat the reviewed command +with `--apply` if appropriate. + +A renamed file at a different path is not automatically discovered by this +command. Do not use broad filename guessing to repair ownership. + +## Stale Mylar Filenames + +Use `reconcile-import-paths` for a different problem: Mylar remembers a filename +such as `Firefly Bad Company #1 (2019).cbr`, while the saved review already has +`Firefly Bad Company 001 (2019).cbz` matched in the same folder. This command +does not enumerate the library or restart matching. The job stays in Step 3. + +The image must include this command. Stop Pullbox and back up its database as +above, keeping the deployment's existing mounts on the maintenance container. +Preview first: + +```bash +docker compose -f pullbox.yml run --rm --no-deps --entrypoint python pullbox \ + -m pullbox.cli reconcile-import-paths --job 1 --source-root /mnt/comics --offline +``` + +Repeat `--source-root` for additional approved mounts. Use `--series-id` to +limit the preview to particular **import-review series IDs**, not ComicVine IDs. +The report includes: + +- `missing_references`: missing entries in the requested scope. +- `candidates_checked`: entries with one matched same-folder counterpart and + the same stored ComicVine issue ID. This alone does not authorize a repair. +- `references_reconciled`: entries that pass current filesystem, signature, + archive safety, content, and independent ComicInfo identity checks. +- `remaining_missing_references`: entries that would remain after applying. +- `retained_reasons`: counts explaining why entries remain, including + `no_unique_matched_counterpart`, `review_or_source_protected`, + `source_check_failed`, `file_safety_review`, and `identity_unconfirmed`. +- `samples` and `retained_samples`: bounded examples, including original paths. + +After reviewing the preview, repeat with `--apply`: + +```bash +docker compose -f pullbox.yml run --rm --no-deps --entrypoint python pullbox \ + -m pullbox.cli reconcile-import-paths --job 1 --source-root /mnt/comics --offline --apply +docker compose -f pullbox.yml start pullbox +``` + +Only the obsolete review reference is removed. Its original path and review ID +remain in the real file's reconciliation diagnostics. The real file keeps its +match and selection state; counters are recomputed and a summary is logged. +An interrupted command rolls back; repeating a successful command is safe. +The grouped candidate query runs once and streams bounded batches rather than +rescanning the database for every page. + +Safeguards: + +- Require independent, equal ComicVine issue IDs from Mylar and inspected + ComicInfo, compatible series/issue/type evidence, and no identity conflicts. +- Require one missing reference and one existing counterpart in the same + folder. Ambiguous copies and cross-folder guesses remain unresolved. +- Refuse changed files, symlinks, root escapes, unreadable files, corrupt or + content-blocked archives, and files requiring a safety override. +- Leave an entire series alone when it has manual matches, selections, skips, + approvals, or completed decisions. Never delete a referenced review row. +- Do not edit Mylar's database, rename files, rewrite ComicInfo, download + metadata, or import anything. Existing source files remain untouched. + +New scans perform the same identity check after archive inspection, reusing +the cached member evidence. Folder imports share the identity/content safety +rules but have no stale Mylar database references to repair. Missing-path copy +does not assume a file disappeared after the scan: it may never have existed +under the database's recorded name. + +## Content Outcomes + +- `archive_no_pages`: there are no non-empty supported image members. A + `ComicInfo.xml` declaring 27 pages does not establish that those pages exist. + Replace or skip the file; it cannot be allowed once. +- `single_page_comic`: possibly an alternate cover, but also possibly an + intentional one-page comic. Inspect it and approve individually or skip it. + Bulk archive-size approval does not approve these files. Moving the source to + Trash is never automatic and is available only from completed-import cleanup + after an explicit red warning on a writable source. +- Archive read failures remain distinct from empty archives. An unavailable + RAR backend, corrupt archive, permissions problem, or disappearing source is + not evidence that the archive has zero pages. +- Two or more image members pass this content heuristic, not a full image + integrity guarantee. It deliberately avoids decoding every page or using an + arbitrary file-size cutoff. PDF/EPUB keep their existing validation paths. + +Verify the resulting review and diagnostic logs before asking the user to +confirm an import. A reported host-restart recovery is useful evidence, but +does not replace checking the current job's durable state. diff --git a/docs/development/INFRASTRUCTURE.md b/docs/development/INFRASTRUCTURE.md index 5ea07823..d4acc11b 100644 --- a/docs/development/INFRASTRUCTURE.md +++ b/docs/development/INFRASTRUCTURE.md @@ -24,7 +24,9 @@ requirements. automation. - Pull request pushes run a cheap GitHub Actions preflight by default. Full PR CI/security/workflow-hygiene checks run after maintainers apply `ci:full`. -- CI tests Python 3.12, 3.13, and 3.14. +- CI runs the complete test suite and uploads coverage for Python 3.12, 3.13, + and 3.14. The blocking 90% coverage gate applies to the production/default + Python 3.14 runtime and local full CI. - Self-hosted Python matrix jobs use five pytest workers per Python version. - Self-hosted functional E2E jobs use three isolated pytest workers per browser. - Normal PR E2E runs disable video encoding; manual CI dispatches can enable @@ -140,7 +142,9 @@ Key gates: |---|---| | `make validate` | CSS build, lint, format, typecheck, and non-E2E tests | | `make ci-local` | GitHub-aligned local CI shape | -| `make ci-full` | CI-local plus security and Docker smoke validation | +| `make ci-full` | CI-local, security, current-tree/history secret scans, container runtime/Grype checks, and Docker smoke validation | +| `make secret-scan` | Blocking Gitleaks scan of current files and the PR commit range | +| `make docker-security-check` | Build the production image, verify its security runtime, and run Grype with the blocking High cutoff | | `make test-a11y` | Contrast gate plus accessibility browser checks | | `make workflow-hygiene` | Local workflow linting | | `make security-check` | Local security checks | @@ -160,6 +164,17 @@ Key gates: - Full validation can be slow. Focused tests are still expected during development. +- Before `make secret-scan` or `make ci-full`, refresh the intended PR base + with `git fetch origin develop`. The default history range is + `origin/develop..HEAD`, excluding merge commits as in PR CI. For another + base, fetch it and set `SECRET_SCAN_BASE=origin/main` (or the intended ref). + A missing base fails the gate; current uncommitted files are also scanned. +- `make docker-smoke` first runs the same container security runtime script, + pinned Grype version, `.grype.yaml`, and blocking High cutoff as Docker CI. + Local architecture, cached base layers, and vulnerability DB freshness may + differ from hosted CI, so a local pass does not replace the remote gate. +- Set `DOCKER_SMOKE_KEEP_ON_FAILURE=1` when a failure must stop before smoke + logs or cleanup. Runtime and Grype failures stop before smoke startup. - CSS drift is a real failure and should not be hand-waved. - E2E failures should be diagnosed from logs, traces, screenshots, and artifacts before guessing at a fix. @@ -195,7 +210,9 @@ tracked in the active CI/CD path. - Branch rulesets should require only stable aggregate checks: `CI Required`, `Security Required`, `Workflow Hygiene Required`, and `Docker Validate Required`. -- Python tests run across the supported matrix. +- Python tests run across the supported matrix. Every version publishes a + coverage report; the production/default Python 3.14 job enforces the 90% + release threshold while compatibility jobs report coverage without gating it. - Migration checks must validate upgrade and downgrade behavior. - Accessibility checks must stay separate from functional browser checks. - E2E runs should upload useful artifacts on failure. @@ -456,6 +473,21 @@ TZ - The container runtime user must be able to read mounted import paths and read or write the mounted library and downloads paths based on the configured workflow. +- Mylar Step 1 reports the exact unavailable locations. A missing child folder + under an accessible root is not itself evidence of a broken Docker mount. + Verify the stored path and container-visible path before adding a mapping; + do not create empty directories simply to make preflight pass. +- Recent Mylar preflight reports are private app data under + `/data/diagnostics/mylar-preflight` (or the configured data directory). + Authenticated report/export requests expire after 24 hours; at most five + reports of up to 32 MiB each are retained. They contain library paths and + series names, so treat exports as private support data. +- Diagnostic collection probes directory permissions and filesystem capacity + without recursively sizing the comic library. `dir_size_bytes` is `null` + with an explicit `not_collected` reason, not a zero-byte library. Blocking + filesystem collection, sanitized database copying, and ZIP compression run + on worker threads rather than the application event loop. Generation time + can still depend on database/log size and storage responsiveness. - Library permission management is chmod-only. It can normalize file and folder modes, but it cannot repair ownership, group, NAS ACL, or mount-option problems. diff --git a/docs/development/LOCAL_CATALOG_V2.md b/docs/development/LOCAL_CATALOG_V2.md new file mode 100644 index 00000000..d14e507a --- /dev/null +++ b/docs/development/LOCAL_CATALOG_V2.md @@ -0,0 +1,138 @@ +# Local Comic Vine catalog v2 client + +## Scope + +The catalog is an optional, derived SQLite database on the **Pullbox server**, +separate from the user's library database. Settings → Metadata → Download catalog +starts the first download. Browsers display status and progress; they do not +download or unzip the database onto the browser device. + +The Pullbox Data API serves only signed publications and catalog artifacts. This +feature adds no remote search or individual metadata endpoints. ComicRack/v1, +MEGA, arbitrary file imports, and user-configurable signing keys are not supported. + +After installation: + +- Add Series, automatic import matching, manual match search, orphan recovery, + explicit Comic Vine ID lookup, and basic issue-list hydration use the catalog. +- A local miss does not silently fall back to online requests. The user can check + for a catalog update; existing review and override decisions remain intact. +- Without an installed catalog, existing Comic Vine discovery behavior remains. +- Full metadata refresh and post-import ComicInfo enrichment retain their direct + Comic Vine provider path, including its batch cache and rate controls. They + still need the user's API key. Basic local matching does not need that key. +- Catalog installation changes no library rows or files. Imports still require + the existing review/confirmation steps before Step 4 materializes files. +- New basic records use `metadata_source=pullbox_catalog`. Series freshness uses + the publication's source cutoff, not download time. A completed full Comic Vine + series refresh is not demoted or overwritten by basic catalog hydration. + Existing live Comic Vine issue records likewise retain their identity and + fields; basic hydration adds missing issues without reverting live metadata. + +## Download and activation contract + +`services/catalog/contract.py` owns the Ed25519 public verification key ring. +The release currently trusts `catalog-2026-09`, verified against the publisher's +public key and production publication. A future signing-key rotation must ship +the next public key in a Pullbox release before the publisher switches. Unknown +keys fail closed. No signing secret is distributed with Pullbox. + +1. Fetch `/api/v2/catalog/latest` from `PULLBOX_DATA_API_BASE_URL` (defaults to + `https://api.pullbox.app`), using the cached ETag when available. +2. Verify the signature over compact sorted JSON, payload SHA-256, schema, + versions, lineage, sizes, and exact same-API download coordinates. Reverify + the cached signed envelope after a 304. Do not follow redirects. +3. Stream the required snapshot or cumulative patch into a checksum-named + `.part` file. Interrupted transfers resume with Range/If-Range. A server that + returns 200 starts a fresh transfer; 206 must match the expected byte range. +4. Verify compressed byte count and SHA-256 **before decompression**. Decompress + with a bounded window and output ceiling, checking free space as it expands. +5. Validate the SQLite application/user versions, dataset identity, source + cutoff, row counts, logical content hash, foreign keys, integrity, and FTS + index. Reject unsupported tables, views, or triggers. +6. Reconstruct each daily cumulative patch from a **fresh copy of its immutable + weekly base**, never from yesterday's patched database. Apply child-first + deletes and parent-first upserts transactionally, replace the dataset + manifest, rebuild FTS, and verify the target logical hash. +7. Flush and atomically move the validated generation into place, then atomically + replace `active.json`. Retain the previous reference and file. Readers open a + read-only immutable generation for each query; long disk work is offloaded + from the event loop. Cancellation waits for an owned disk operation to finish + before releasing the update lock. + +When a new weekly base is published, it is downloaded in full. Between weekly +bases, only the latest cumulative patch is needed. If no current patch exists, +the signed publication's full snapshot is installed. A healthy newer local +version is never downgraded by a stale API response. + +Limits: manifest 1 MiB, compressed artifact 2 GiB, decompressed artifact 4 GiB, +zstd window 128 MiB; search input 256 characters / 16 words; up to 1,000 results. +Disk-space checks reserve a safety margin, and patching checks space for copying +the weekly base. The byte-progress bar applies to transfer; unpacking, +verification, and installation are separate indeterminate stages. + +## Persistent storage and recovery + +Under `/catalog/`: + +```text +active.json installed generation and source cutoff +previous.json last installed generation before a successful update +state.json opt-in, update preference, coarse status, timestamps +manifest.json signed publication cache and ETag +update.lock cross-process advisory update lock +bases/.db immutable weekly snapshots +versions/.db validated reconstructed daily generations +downloads/.part resumable compressed transfer +staging/catalog-*.db uncommitted work, never used for searches +``` + +An async lock and filesystem lock prevent simultaneous installers. Failed +downloads or patches leave the installed catalog active. Retry resumes a matching +partial transfer. Checksum failures discard the bad transfer. Retrying repairs a +missing or invalid file at the currently published version. Unknown signing keys +or formats require a client update, not bypassing verification. An unreadable +active catalog reports an actionable error rather than making live metadata +requests. Symlinked catalog storage is rejected. + +Cleanup runs under the update lock: abandon unfinished staging files, remove +compressed downloads after success, and expire unreferenced generations/partials +older than two days. Keep active, previous, and both referenced weekly bases. +Only recognized catalog-owned filenames are eligible; no library paths are used. +The grace period accommodates readers that started before activation. + +## Scheduling and local control API + +`catalog_update` appears as **Local Catalog Update** in the normal task system. +It checks daily at 06:30 in the scheduler timezone, with up to 30 minutes of jitter. +Automatic runs do nothing until the user requests the first download or when +automatic updates are disabled. A startup check catches overdue work; a 23-hour +freshness guard allows the next day's jitter to be earlier than yesterday's. +Manual checks bypass the age guard. Startup checks use the same installer lock +and status but do not create a separate scheduler history entry. + +| Local route | Access | Result | +|---|---|---| +| `GET /api/v1/catalog` | Authenticated | Safe status, version, cutoff, byte progress, timestamps, error | +| `POST /api/v1/catalog/sync` | Interactive operator + CSRF | 202 queued/already queued/already running; 503 if unavailable | +| `PATCH /api/v1/catalog/preferences` | Interactive operator + CSRF | `{ "automatic_updates": true/false }`; updated status | + +These routes control this Pullbox instance; they are not new Pullbox Data API +distribution routes. Machine API keys cannot trigger downloads or change the +preference. Settings polls only while the page is mounted and live updates are +enabled; a download continues when the page is closed. Errors never include +credentials or raw response bodies. + +## Verification + +`tests/unit/test_catalog_*` covers signed-manifest failures, hash and lineage +checks, corrupted/unsupported SQLite artifacts, cumulative reversion, interrupted +transfers, installation failure preservation, overlap, repair, cleanup, realistic +zstd windows, search escaping, exact issue IDs and fractions, source provenance, +and live enrichment separation. `tests/ui/test_catalog_controls.py` covers +settings, no-key local search, session/CSRF and machine-key boundaries. + +The external-artifact SQLite adapter is deliberately separate from the ORM +application database. Its SQL identifiers come exclusively from a closed contract +allowlist; all search terms, IDs and filters are bound parameters. Existing +application database access remains SQLAlchemy-based. diff --git a/docs/development/SECURITY_STANDARDS.md b/docs/development/SECURITY_STANDARDS.md index 5dd0c181..c72d1fbd 100644 --- a/docs/development/SECURITY_STANDARDS.md +++ b/docs/development/SECURITY_STANDARDS.md @@ -747,6 +747,10 @@ default-src 'self'; script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; at least 2.8.1 before accepting the image. - `.grype.yaml` limits reviewed container exceptions to exact package versions; the Grype High-severity gate remains blocking. +- Local `make ci-full` scans both current files and the PR commit range with + Gitleaks, then runs the container runtime and blocking Grype checks before + Docker smoke tests. Grype is version-pinned in both local installation and + Docker workflows; exceptions are shared, not separate local bypasses. - GitHub Actions are SHA-pinned with version comments. - Workflows define explicit default permissions and per-job permissions. - `.github/dependabot.yml` covers `pip`, `github-actions`, `docker`, and `npm`. @@ -786,6 +790,12 @@ default-src 'self'; script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; becomes public, but remains informational unless `PULLBOX_REQUIRE_CODEQL=true`. - Safety's legacy command still needs valid JSON artifact generation while it remains advisory. +- Local and GitHub dependency audits use `scripts/run_dependency_audit.py`. + Temporary exceptions must be scoped to the reviewed package/version, + advisory, dependency use, and a fixed UTC expiry. See + [Dependency Audit Exceptions](DEPENDENCY_AUDIT_EXCEPTIONS.md) for the current + Safety/NLTK development-only risk acceptance. Production image validation + rejects either package; neither belongs in the shipped runtime. - Before public visibility, scan the current tree, full Git history, release notes, PR/issue metadata, and refs for secrets and internal tool/provenance references. diff --git a/docs/features/airdcpp.md b/docs/features/airdcpp.md deleted file mode 100644 index 86c6a5ba..00000000 --- a/docs/features/airdcpp.md +++ /dev/null @@ -1,120 +0,0 @@ -# AirDC++ Integration - -AirDC++ support is experimental and disabled by default. It adds Direct -Connect as an independent acquisition protocol without changing the existing -Usenet, torrent, or direct-download providers. - -Enable it with `PULLBOX_AIRDCPP_ENABLED=true`, restart Pullbox, and then add one -or more AirDC++ clients under **Settings > Download Clients**. - -## AirDC++ API user - -Create a dedicated, non-administrator AirDC++ Web API user for Pullbox. Grant -only these permissions: - -- `search` -- `download` -- `queue_view` -- `queue_edit` -- `hubs_view` -- `settings_view` - -Do not grant `admin`, filesystem, share, chat, event-edit, web-user management, -or `transfers`. Pullbox uses the queue API for whole-file progress and -completion, so `transfers` is neither needed nor supported. - -The supported compatibility floor is Web API version 1 and feature level 10. -Set AirDC++'s **Minimum search interval** to at least 45 seconds before testing -the connection. Pullbox checks the API version, permissions, WebSocket setup, -and minimum interval without changing AirDC++ settings or starting a search. - -## Client settings - -For each AirDC++ client, configure: - -- **URL**: the AirDC++ Web API base URL, without credentials, a query string, - or a fragment. -- **Username and password**: the dedicated API account. Pullbox encrypts the - password at rest and never returns it from the API. -- **Remote path**: the absolute completion root reported by AirDC++, commonly - `/Downloads`. -- **Download directory**: the absolute path for that same directory inside the - Pullbox container, such as `/downloads/airdcpp`. -- **Hub allowlist**: optional normalized `adc://` or `adcs://` hub URLs, one per - line. Leave it empty to search every currently connected hub. - -The remote and local roots must identify the same bounded directory through -the two containers' mounts. Completed files outside the exact remote root, -paths containing traversal, symlinks, non-regular files, and unsupported -extensions are rejected. Do not configure `/` or another broad filesystem -root. - -Multiple AirDC++ clients may be enabled. A manual search fans out once to every -enabled and ready AirDC++ client; each client then searches all of its connected -hubs unless its allowlist narrows the scope. Failure or cooldown on one client -does not hide usable results from another client or acquisition source. - -## Search cooldown and manual searches - -Pullbox enforces one durable search gate per configured AirDC++ client. Manual -searches, automatic searches, different query text, and queue alternate-source -searches all share the same 45-second-or-longer interval. The gate survives a -Pullbox restart. - -If a manual search reaches a client during its cooldown, Pullbox keeps the -search spinner active and displays: - -> Direct Connect search will resume in {seconds} seconds to respect the -> 45-second hub cooldown. - -The search resumes automatically when the client becomes eligible. The live -region announces the wait when it begins and the resume transition, not every -countdown tick. - -Manual results are grouped and ranked using normalized file identity, size, -online source availability, and route freshness. The browser receives only a -short-lived opaque route token; user identities, hub identities, raw result -IDs, TTH values, and remote paths are not exposed in the page. - -## Queue, progress, and import - -Choose **Grab** on an AirDC++ result to create the durable Pullbox acquisition -record before the remote queue is mutated. Pullbox then reconciles the exact -AirDC++ bundle from both WebSocket events and bounded REST snapshots. The -Downloads page shows durable progress, speed, ETA, and terminal state without -polling AirDC++ separately for every row. - -Cancel removes the exact known bundle and records the cancellation locally. -Retry creates a fresh queue attempt from retained provenance. Pullbox does not -guess when a bundle identity is ambiguous. - -After AirDC++ reports a completed/shared bundle, Pullbox maps the completed path -through the configured roots and sends the archive through the normal comic -validation, naming, ComicInfo, library registration, and cleanup pipeline. A -database lease prevents duplicate imports after overlapping scheduler runs or -restarts. The raw AirDC++ completion path is redacted from diagnostic exports. - -Automatic wanted searches may include AirDC++ results only when **Automatic -search** is enabled for that client. During the experimental soak period those -results are evaluation-only: Pullbox records selection diagnostics but does not -automatically mutate the AirDC++ queue. Manual Grab is the supported test path. - -## Troubleshooting - -Use **Test** on the client card and **System > Health > Download Clients**. -Common states are: - -- **Authentication failed**: verify the dedicated username and password. -- **Permissions incomplete**: grant exactly the six required permissions. -- **API incompatible**: upgrade AirDC++ Web Client to an API version 1, - feature-level-10-or-newer release. -- **Minimum interval below 45 seconds**: change the AirDC++ setting, then test - again. -- **Reconnecting or unavailable**: verify the API URL and container/network - reachability; the supervisor retries with bounded backoff. -- **Completed path unavailable or unsafe**: verify the remote/local root pair - and the shared volume mount. Do not bypass the path safety checks. - -When diagnosing a missing result, first confirm the hub is connected in -AirDC++, the client's search toggle is enabled, its cooldown has elapsed, and -the optional hub allowlist exactly matches the connected hub URL. diff --git a/docs/features/comic-reader.md b/docs/features/comic-reader.md deleted file mode 100644 index da503bad..00000000 --- a/docs/features/comic-reader.md +++ /dev/null @@ -1,160 +0,0 @@ -# Embedded comic reader - -Pullbox can open an owned issue directly from its issue-details page. The reader is a private, -full-viewport, single-page experience intended both for ordinary reading and for quickly checking -that a download is the correct comic. - -## Supported files - -The reader supports CBZ, CBR, CB7, CBT, and PDF. The production image includes the official UnRAR -backend for CBR, py7zr/7-Zip support for CB7, TAR support for CBT, and Poppler for PDF rendering. -The file signature must match the format recorded in Pullbox. EPUB is not supported. - -Archive pages are naturally ordered, unsafe members are ignored or rejected, and extraction, -decoded pixels, rendering time, worker concurrency, and generated cache storage are bounded. Large -images are resized to the configured rendition ceiling before delivery to keep browser memory use -predictable. Source comic files are never rewritten by the reader. - -## Controls - -- Use the visible previous/next buttons, the one-based page field, the outer tap zones, or a - horizontal touch swipe while the page is fitted. -- Fit page is the default. Fit width, fit height, actual size, and stepped zoom are available. -- LTR/RTL changes arrow, tap-zone, and swipe meaning without changing canonical page numbers. -- Close returns to the same issue URL, scroll position, page state, and Read button focus. -- `?` opens the complete keyboard shortcut reference. The main shortcuts are arrow keys, - Page Up/Down, Space, Home/End, `G`, `W`, `H`, `0`, `+`/`-`, `R`, `F`, and Escape. - -Pullbox saves only the last page that remained visible after decoding for the settle interval. -Progress is private to the signed-in user. Prefetching and opening directly on the final page do not -mark completion; deliberately navigating to and viewing the final page does. - -## Reading state and workspace - -Reading state belongs to the signed-in user and the canonical Pullbox issue. It has three -independent dimensions: - -- resume position records the last settled page for the current file revision and page count; -- Read/Unread records deliberate completion intent; and -- Want to Read maintains a private reading queue. - -**Want to Read is not the acquisition status Wanted.** The reading queue does not trigger a search -or download, and acquisition state does not mark a comic read. Marking an issue Read removes it from -Want to Read but preserves its last known page. Marking it Unread clears completion without erasing -resume history. - -The **Reading** workspace at `/reading` presents bounded Continue, Want to Read, and Read tabs. The -dashboard shows the eight most recently opened incomplete issues. Issue details expose the current -reading state and explicit actions; series details show per-issue state and a read aggregate; the -series registry reports readable/completed totals without opening comic archives. Unavailable -queued or read issues remain visible but cannot be opened until a supported owned file is restored. - -## Moving between issues - -Previous issue and Next issue are separate from page navigation. They consider only owned issues in -the same series with a registered CBZ, CBR, CB7, CBT, or PDF file. Pullbox waits for the active -issue's progress write before requesting the next manifest. A failed save, target manifest, or first -target page leaves the current issue open and reports an in-reader recovery message. - -Only one adjacent page in the active issue may be prefetched; another issue's pages are not fetched -until the user explicitly switches. Reader fit, direction, and zoom preferences carry across the -switch. Reaching the final page shows a completion panel with **Read next issue**, **Mark unread**, -or a caught-up message as appropriate. - -## File identity and continuity - -Private state is keyed by `(user_id, issue_id)`, never a path or `LibraryFile` identifier. File -operations therefore follow these rules: - -| Operation | Result | -|---|---| -| Rename or relocate | Position, completion, queue, clocks, and state version remain exact. | -| Convert to CBZ with the same page count | The saved ordinal position and issue-level intent remain. | -| Replace or re-import with the same page count | The saved ordinal position and issue-level intent remain. | -| Replace or re-import with a different page count | The next open starts on page 1; completion and queue intent remain. | -| Remove the registered file | Continue hides the issue; Want to Read/Read preserve it as unavailable. | -| Re-import against the same Pullbox issue | Readability returns using the existing private state row. | -| Delete the canonical Pullbox issue | Its private reading state is deleted by database cascade. | - -The manifest is side-effect free: page-count reconciliation controls the initial page returned to -the client but does not rewrite saved state merely because a file was inspected. - -## Errors and recovery - -The reader keeps failures inside the full-viewport shell and never exposes library paths, archive -member names, renderer output, or stack traces. A page-level failure can be retried without closing -the reader, and the original Download action remains available. Missing, mismatched, corrupt, empty, -oversized, or temporarily busy sources return stable private errors. - -If PDF or CBR support is unexpectedly unavailable, verify that the running image is the supported -Pullbox production image rather than a locally reduced Python environment. The authenticated -`GET /api/v1/reader/capabilities` diagnostic reports readiness for all five formats plus path-free -cache and worker limits. - -## Cache and operations - -Generated pages live under the Pullbox data directory in `reader-cache`; losing them affects only -latency because they are rebuilt from source comics. The authenticated, CSRF-protected -`DELETE /api/v1/reader/cache` operation clears only generated reader files and does not follow -symlinks or touch the comic library. Normal deployments use a 512 MiB quota, two expensive workers, -and a short bounded worker wait. - -Archive reads retain hard entry, expanded-size, page-size, page-count, path, pixel, and concurrency -budgets. The 250:1 compression-ratio guard applies only to readable page entries that expand to at -least 4 MiB, so small solid-color pages are not mistaken for archive bombs. Operators can adjust the -floor with `PULLBOX_READER_COMPRESSION_RATIO_MIN_MB`; large high-ratio pages still fail before -extraction, with path-free structured diagnostics. - -For an immediate feature rollback, set `PULLBOX_READER_ENABLED=false` in the Pullbox service -environment and recreate/restart that service. This hides Read and makes the private reader routes -return not found while preserving source comics, generated cache files, and private resume state. -Set the value back to `true` to restore the feature. - -Reader state is stored in the main Pullbox database, so a full `/data` backup includes it. Generated -`reader-cache` files do not need to be backed up. Back up `/data` before any migration downgrade. -Downgrading the independent-state migration deletes queue-only, completion-only, and explicit-unread -rows that the old progress-only schema cannot represent; downgrading past the original reader-state -migration deletes all reader state. Disabling the feature gate is the preferred rollback because it -is non-destructive. - -## Performance contract and acceptance snapshot - -The reader indexes a source once per revision, reads or renders only the requested page, prefetches -only the adjacent page, and keeps generated pages in the bounded disk cache. Archive decoding and -PDF rendering never run on the event loop. The initial acceptance objectives are a cached manifest -or page under 150 ms, a cold common CBZ page under one second, and a cold PDF page under two seconds -on documented hardware. These are engineering objectives, not a network-inclusive release SLA. - -The 2026-08-03 development acceptance run used the production ARM64 container on an Apple M4 Pro -MacBook Pro with 24 GB of memory. Each synthetic format fixture contained the same two JPEG pages; -the CBR fixture used RAR5. Times are single-run server measurements and RSS deltas are retained -process memory after garbage collection, so they are deliberately reported as observations rather -than statistically significant benchmarks. - -| Format | Manifest | Cold first page | Repeated cache hit | RSS delta | FD/child leak | -|---|---:|---:|---:|---:|---:| -| CBZ | 5.5 ms | 10.6 ms | 0.13 ms | 2.4 MiB | 0 / 0 | -| CBR | 6.2 ms | 11.6 ms | 0.14 ms | 8.1 MiB | 0 / 0 | -| CB7 | 39.0 ms | 35.2 ms | 0.11 ms | 3.4 MiB | 0 / 0 | -| CBT | 3.4 ms | 4.1 ms | 0.17 ms | 1.6 MiB | 0 / 0 | -| PDF | 9.5 ms | 356.6 ms | 0.15 ms | 10.5 MiB | 0 / 0 | - -A real 74.3 MB, 52-page CBZ from the mounted development library produced its manifest in 5.9 ms, -its cover in 129.5 ms, its next page in 4.0 ms, and a repeated page in 0.14 ms. The two generated -pages occupied 1.78 MiB on disk; retained process RSS grew by 52.4 MiB after decoding the large -source artwork. Cancellation, single-flight generation, worker saturation, cache quota/clear, and -source-preservation behavior have dedicated regression coverage. - -The 2026-08-25 reader-enhancement acceptance run on an Apple M5 Pro with 48 GB of memory seeded -10,000 issues across 500 series and 5,000 overlapping state rows per user in SQLite. The warmed -Continue query used exactly two SQL statements, returned eight rows, and improved from roughly -2,154 ms to 3 ms after adding the measured `library_files.issue_id` join index. A 50-issue manifest -run retained three bounded page sources, added no file descriptors or child processes, and retained -about 1.2 MiB RSS. These are local development observations, not release SLAs. - -## Future OPDS and mobile clients - -`IssueReaderState` plus the adapter-neutral reader state and bounded query services are the source -of truth for future OPDS and mobile integrations. Those adapters must call the same commands and -projections rather than creating another state table or deriving state from file paths. Web URLs -remain a web-adapter concern and are not stored in domain projections. diff --git a/docs/library-root-removal.md b/docs/library-root-removal.md new file mode 100644 index 00000000..5bb44fd3 --- /dev/null +++ b/docs/library-root-removal.md @@ -0,0 +1,43 @@ +# Removing an unused library root + +In **Settings > Media Management > Library roots**, disable a root and choose +**Remove root**. Review the checks, choose **Confirm removal**, then confirm the +root's name and path in the standard confirmation dialog. + +This removes configuration, including root-specific naming settings. It never +deletes, moves, scans, or changes permissions on folders or comics. An offline +or missing directory can be removed if nothing still depends on its root. + +Removal is blocked while a root is enabled, is the default destination, or has +registered files, current/preferred series associations, Story Arc destinations +or placements, or default destination settings. The preview gives exact counts +and the next step for each blocker. Disabling a root does not detach its files. +Relocate or explicitly remove the associated library entries first; root +removal is not a bulk-delete-files shortcut. + +Finish or cancel active/paused imports and file utilities before removal. +Pending import placement or rollback work also blocks removal. Confirmation +rechecks dependencies; a changed or expired preview must be reviewed again. + +Historical imports do not trap unused roots forever. Their original root name, +path, and ID are preserved, and their history and rollback records remain. +After removal, retry/recovery cannot silently use the current default root: +start a new import and explicitly choose a valid destination. Existing imported +library contents are not removed by this operation. + +## Developer safeguards + +All live root foreign keys use `RESTRICT`, and the ORM does not delete registered +files when their root is deleted. Historical import destinations are explicitly +detached with a retained snapshot in the same transaction as removal. Only the +root's own naming policy is discarded. Previews are operator-bound, signed, and +valid for 15 minutes. SQLite confirmation takes a short write reservation before +rechecking; final deletion also checks dependencies and relies on FK enforcement. + +The migration rebuilds affected SQLite tables on a dedicated Alembic +connection with foreign-key enforcement disabled for migrations only, then +checks referential integrity. Back up the database before applying migrations. +Runtime connections keep enforcement enabled. Existing inline constraints are +preserved when downgrading so older migrations continue to work correctly. +Downgrade is refused once history contains a removed-root snapshot, because old +code cannot honor its retry guard. PostgreSQL uses ordinary FK alterations. diff --git a/pyproject.toml b/pyproject.toml index 0c89c99a..a1f74e8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dependencies = [ "pillow~=12.3", "pdf2image~=1.17", "tzlocal~=5.3", + "zstandard>=0.25,<1.0", ] [project.optional-dependencies] diff --git a/scripts/benchmark_import_directory_classification.py b/scripts/benchmark_import_directory_classification.py new file mode 100644 index 00000000..18cc9ec3 --- /dev/null +++ b/scripts/benchmark_import_directory_classification.py @@ -0,0 +1,67 @@ +"""Benchmark import series-directory classification without filesystem I/O. + +Usage: + .venv/bin/python scripts/benchmark_import_directory_classification.py \ + --series-count 50000 --publisher-count 500 +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +from pullbox.core.collection_scanner import CollectionScanner +from pullbox.performance.baseline import current_process_peak_rss_bytes + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--series-count", type=int, default=10_000) + parser.add_argument("--publisher-count", type=int, default=100) + args = parser.parse_args() + + if args.series_count < 1: + parser.error("--series-count must be at least 1") + if args.publisher_count < 1: + parser.error("--publisher-count must be at least 1") + + root = Path("/synthetic-import-root") + publishers = [root / f"Publisher {index:05d}" for index in range(args.publisher_count)] + dir_files = {publisher: [publisher / "Publisher Special 001.cbz"] for publisher in publishers} + for index in range(args.series_count): + publisher = publishers[index % len(publishers)] + series = publisher / f"Series {index:06d} (2026)" + dir_files[series] = [series / "Issue 001.cbz"] + + scanner = CollectionScanner() + started_at = time.perf_counter() + series_dirs = scanner._identify_series_dirs(dir_files) + elapsed = time.perf_counter() - started_at + + if len(series_dirs) != args.series_count: + msg = f"expected {args.series_count} series directories, found {len(series_dirs)}" + raise RuntimeError(msg) + + print( + json.dumps( + { + "profile": "directory_classification_only", + "series_count": args.series_count, + "publisher_count": args.publisher_count, + "input_comic_directory_count": len(dir_files), + "series_directory_count": len(series_dirs), + "classification_seconds": elapsed, + "directories_per_second": len(dir_files) / elapsed if elapsed else None, + "peak_rss_bytes": current_process_peak_rss_bytes(), + "filesystem_scan_count": 0, + }, + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark_import_execute.py b/scripts/benchmark_import_execute.py index 9893b56d..e3ab7350 100644 --- a/scripts/benchmark_import_execute.py +++ b/scripts/benchmark_import_execute.py @@ -8,6 +8,7 @@ import argparse import asyncio +import hashlib import json import tempfile import time @@ -21,6 +22,7 @@ from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from pullbox.core.library_file_ownership import build_file_identity_signature from pullbox.core.library_policy import LibraryIngestPolicy, serialize_library_ingest_policy from pullbox.models import config as _config_models # noqa: F401 from pullbox.models import import_job as _import_job_models # noqa: F401 @@ -34,6 +36,7 @@ ImportedFile, ImportedFileStatus, ImportedSeries, + ImportFileHandlingMode, ImportJob, ImportJobStatus, ImportSeriesStatus, @@ -46,6 +49,73 @@ from pullbox.performance.baseline import current_process_peak_rss_bytes from pullbox.services.import_service import ImportService +_REPORT_DIGEST_PAGE_SIZE = 1_000 +_MAX_REPORT_SAMPLE_LIMIT = 100 + + +def _bounded_sample_limit(value: str) -> int: + limit = int(value) + if not 0 <= limit <= _MAX_REPORT_SAMPLE_LIMIT: + raise argparse.ArgumentTypeError( + f"report sample limit must be between 0 and {_MAX_REPORT_SAMPLE_LIMIT}" + ) + return limit + + +async def _summarize_library_files( + session: AsyncSession, + *, + sample_limit: int, +) -> dict[str, object]: + """Return bounded examples and a digest without materializing every row.""" + library_file_count = int(await session.scalar(select(func.count(LibraryFile.id))) or 0) + format_rows = ( + await session.execute( + select(LibraryFile.file_format, func.count(LibraryFile.id)) + .group_by(LibraryFile.file_format) + .order_by(LibraryFile.file_format) + ) + ).all() + library_format_counts = { + file_format.value.lower(): int(count) for file_format, count in format_rows + } + name_samples = list( + ( + await session.scalars( + select(LibraryFile.file_name) + .order_by(LibraryFile.file_name, LibraryFile.id) + .limit(sample_limit) + ) + ).all() + ) + + digest = hashlib.sha256() + last_id = 0 + while True: + page = ( + await session.execute( + select(LibraryFile.id, LibraryFile.file_name) + .where(LibraryFile.id > last_id) + .order_by(LibraryFile.id) + .limit(_REPORT_DIGEST_PAGE_SIZE) + ) + ).all() + if not page: + break + for library_file_id, file_name in page: + encoded = file_name.encode("utf-8") + digest.update(len(encoded).to_bytes(8, byteorder="big")) + digest.update(encoded) + last_id = library_file_id + + return { + "library_file_count": library_file_count, + "library_file_name_sample_limit": sample_limit, + "library_file_name_samples": name_samples, + "library_file_name_digest_sha256": digest.hexdigest(), + "library_format_counts": dict(sorted(library_format_counts.items())), + } + class FakeSeriesService: """Deterministic Step 4 series service used for import benchmarking.""" @@ -233,6 +303,12 @@ async def main() -> None: "and ComicInfo writes on small valid archives." ), ) + parser.add_argument( + "--report-sample-limit", + type=_bounded_sample_limit, + default=20, + help="Maximum library filenames included in the bounded JSON report (0-100).", + ) args = parser.parse_args() fake_series_service = FakeSeriesService(args.files_per_series) @@ -333,6 +409,7 @@ async def counted_commit() -> None: else {} ), target_library_root_id=root.id, + file_handling_mode=ImportFileHandlingMode.MANAGED_COPY, ) session.add(job) await session.flush() @@ -346,6 +423,7 @@ async def counted_commit() -> None: status=ImportSeriesStatus.CONFIRMED, cv_id=cv_id, file_count=len(file_paths), + selected_for_import=True, ) session.add(imported_series) await session.flush() @@ -365,6 +443,8 @@ async def counted_commit() -> None: matched_issue_cv_id=cv_id * 1000 + issue_idx, match_confidence="high", match_method="benchmark", + include_in_import=True, + source_signature=build_file_identity_signature(path), ) ) await session.commit() @@ -400,10 +480,9 @@ async def counted_commit() -> None: await asyncio.gather(*pending_enrichment, return_exceptions=True) await session.refresh(job) - library_files = ( - (await session.execute(select(LibraryFile).order_by(LibraryFile.file_name))) - .scalars() - .all() + library_file_summary = await _summarize_library_files( + session, + sample_limit=args.report_sample_limit, ) operation_progress_count = int( await session.scalar(select(func.count(OperationProgress.id))) or 0 @@ -413,9 +492,6 @@ async def counted_commit() -> None: for file_paths in series_files for path in file_paths ) - library_format_counts = Counter( - library_file.file_format.value.lower() for library_file in library_files - ) report = { "file_work_profile": args.file_work_profile, "real_file_work": real_file_work, @@ -426,10 +502,8 @@ async def counted_commit() -> None: "prefetch_calls": fake_series_service.prefetch_calls, "series_add_calls": fake_series_service.add_calls, "register_calls": register_calls, - "library_file_count": len(library_files), "operation_progress_count": operation_progress_count, - "library_file_names": [library_file.file_name for library_file in library_files], - "library_format_counts": dict(sorted(library_format_counts.items())), + **library_file_summary, "source_format_counts": dict(sorted(source_format_counts.items())), "peak_rss_bytes": current_process_peak_rss_bytes(), "final_status": job.status.value, diff --git a/scripts/benchmark_import_metadata_scale.py b/scripts/benchmark_import_metadata_scale.py new file mode 100644 index 00000000..e55cc424 --- /dev/null +++ b/scripts/benchmark_import_metadata_scale.py @@ -0,0 +1,331 @@ +"""Exercise import review/rollback at metadata scale without archive payloads. + +The default profile represents 50,000 series and 200,000 files entirely as +database staging rows. It also confirms 10,000 staged story arcs, then restores +all review state through the rollback path. No provider or filesystem scanner +is constructed, so both call counts are structurally zero. + +Usage: + .venv/bin/python scripts/benchmark_import_metadata_scale.py + .venv/bin/python scripts/benchmark_import_metadata_scale.py \ + --series-count 100 --files-per-series 4 --story-arc-count 100 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import tempfile +import time +from pathlib import Path + +from sqlalchemy import event, func, insert, select, text, update +from sqlalchemy.ext.asyncio import ( + AsyncConnection, + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from pullbox.models import Base +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobStatus, + ImportSeriesStatus, + ImportSourceType, +) +from pullbox.models.story_arc import ( + ImportedStoryArcStatus, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.performance.baseline import current_process_peak_rss_bytes +from pullbox.services.import_rollback_state import restore_review_state_after_rollback +from pullbox.services.import_story_arc_review import confirm_import_story_arcs + +_POSTGRESQL_URL_ENV = "PULLBOX_IMPORT_BENCHMARK_POSTGRESQL_URL" + + +async def _prepare_database( + connection: AsyncConnection, + *, + backend: str, + reset_dedicated_database: bool, +) -> None: + """Prepare an empty schema without attempting cyclic table-by-table drops.""" + if reset_dedicated_database: + if backend != "postgresql": + raise ValueError("dedicated schema reset requires PostgreSQL") + await connection.execute(text("DROP SCHEMA public CASCADE")) + await connection.execute(text("CREATE SCHEMA public")) + await connection.run_sync(Base.metadata.create_all) + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("value must be positive") + return parsed + + +async def _seed_metadata( + session: AsyncSession, + *, + job_id: int, + series_count: int, + files_per_series: int, + story_arc_count: int, + insert_batch_size: int, +) -> int: + commits = 0 + for start in range(1, series_count + 1, insert_batch_size): + stop = min(start + insert_batch_size, series_count + 1) + await session.execute( + insert(ImportedSeries), + [ + { + "id": series_id, + "import_job_id": job_id, + "raw_series_name": f"Synthetic Series {series_id}", + "file_count": files_per_series, + "status": ImportSeriesStatus.IMPORTED, + "files_imported": files_per_series, + } + for series_id in range(start, stop) + ], + ) + await session.commit() + commits += 1 + + total_files = series_count * files_per_series + for start in range(1, total_files + 1, insert_batch_size): + stop = min(start + insert_batch_size, total_files + 1) + await session.execute( + insert(ImportedFile), + [ + { + "id": file_id, + "import_job_id": job_id, + "import_series_id": ((file_id - 1) // files_per_series) + 1, + "file_path": f"/metadata-only/{file_id}.cbz", + "file_name": f"{file_id}.cbz", + "file_size": 0, + "file_format": "cbz", + "status": ImportedFileStatus.IMPORTED, + "include_in_import": True, + } + for file_id in range(start, stop) + ], + ) + await session.commit() + commits += 1 + + for start in range(1, story_arc_count + 1, insert_batch_size): + stop = min(start + insert_batch_size, story_arc_count + 1) + await session.execute( + insert(ImportedStoryArc), + [ + { + "id": arc_id, + "import_job_id": job_id, + "source_kind": StoryArcSourceKind.MYLAR3, + "source_key": f"metadata-scale:{arc_id}", + "source_arc_id": f"source-{arc_id}", + "source_ordinal": arc_id, + "name": f"Synthetic Arc {arc_id}", + "status": ImportedStoryArcStatus.READY, + "selected_for_import": True, + } + for arc_id in range(start, stop) + ], + ) + await session.execute( + insert(ImportedStoryArcEntry), + [ + { + "id": arc_id, + "imported_story_arc_id": arc_id, + "source_ordinal": 1, + "reading_order": arc_id, + "reading_order_raw": str(arc_id), + "resolution_state": StoryArcResolutionState.MISSING, + "source_kind": StoryArcSourceKind.MYLAR3, + "source_entry_id": f"entry-{arc_id}", + "source_arc_id": f"source-{arc_id}", + "source_issue_number_text": "1AU", + "selected_for_import": True, + } + for arc_id in range(start, stop) + ], + ) + await session.commit() + commits += 1 + return commits + + +def _install_select_counter(engine: AsyncEngine, counts: dict[str, int], phase: list[str]) -> None: + def record_statement(*args: object) -> None: + if str(args[2]).lstrip().upper().startswith("SELECT"): + counts[phase[0]] = counts.get(phase[0], 0) + 1 + + event.listen(engine.sync_engine, "before_cursor_execute", record_statement) + + +async def _run(args: argparse.Namespace) -> dict[str, object]: + total_started = time.monotonic() + with tempfile.TemporaryDirectory(prefix="pullbox-metadata-scale-") as tmp: + db_path = Path(tmp) / "metadata-scale.db" + if args.backend == "postgresql": + db_url = os.environ.get(_POSTGRESQL_URL_ENV, "") + if not db_url.startswith("postgresql+asyncpg://"): + raise RuntimeError( + f"{_POSTGRESQL_URL_ENV} must contain a dedicated PostgreSQL async URL" + ) + else: + db_url = f"sqlite+aiosqlite:///{db_path}" + engine = create_async_engine(db_url) + async with engine.begin() as connection: + await _prepare_database( + connection, + backend=args.backend, + reset_dedicated_database=args.reset_dedicated_database, + ) + + session_factory = async_sessionmaker(engine, expire_on_commit=False) + select_counts: dict[str, int] = {} + phase = ["idle"] + _install_select_counter(engine, select_counts, phase) + async with session_factory() as session: + job = ImportJob( + source_path="/metadata-only", + source_type=ImportSourceType.MYLAR3, + status=ImportJobStatus.REVIEW, + ) + session.add(job) + await session.commit() + + seed_started = time.monotonic() + seed_commit_count = await _seed_metadata( + session, + job_id=int(job.id), + series_count=args.series_count, + files_per_series=args.files_per_series, + story_arc_count=args.story_arc_count, + insert_batch_size=args.insert_batch_size, + ) + seed_elapsed_ms = round((time.monotonic() - seed_started) * 1000) + + phase[0] = "confirm" + confirm_started = time.monotonic() + confirmed_arc_count = await confirm_import_story_arcs( + session, + int(job.id), + story_arc_ids=(), + decisions=(), + ) + await session.commit() + confirm_elapsed_ms = round((time.monotonic() - confirm_started) * 1000) + session.expunge_all() + + await session.execute( + update(ImportedStoryArc) + .where(ImportedStoryArc.import_job_id == job.id) + .values(status=ImportedStoryArcStatus.IMPORTED) + ) + await session.execute( + update(ImportJob) + .where(ImportJob.id == job.id) + .values(status=ImportJobStatus.ROLLING_BACK) + ) + await session.commit() + + phase[0] = "rollback" + rollback_started = time.monotonic() + await restore_review_state_after_rollback( + session, + int(job.id), + batch_size=args.operation_batch_size, + ) + await session.commit() + rollback_elapsed_ms = round((time.monotonic() - rollback_started) * 1000) + phase[0] = "report" + + final_series = await session.scalar( + select(func.count()) + .select_from(ImportedSeries) + .where(ImportedSeries.status == ImportSeriesStatus.MATCHED) + ) + final_files = await session.scalar( + select(func.count()) + .select_from(ImportedFile) + .where(ImportedFile.status == ImportedFileStatus.NO_MATCH) + ) + final_arcs = await session.scalar( + select(func.count()) + .select_from(ImportedStoryArc) + .where(ImportedStoryArc.status == ImportedStoryArcStatus.CONFIRMED) + ) + if args.backend == "postgresql": + database_bytes = int( + await session.scalar(text("SELECT pg_database_size(current_database())")) or 0 + ) + else: + database_bytes = db_path.stat().st_size + + await engine.dispose() + return { + "profile": "metadata_only", + "backend": args.backend, + "series_count": args.series_count, + "files_per_series": args.files_per_series, + "represented_file_count": args.series_count * args.files_per_series, + "story_arc_count": args.story_arc_count, + "archive_payload_count": 0, + "provider_call_count": 0, + "filesystem_scan_count": 0, + "insert_batch_size": args.insert_batch_size, + "operation_batch_size": args.operation_batch_size, + "seed_commit_count": seed_commit_count, + "confirmed_arc_count": confirmed_arc_count, + "final_matched_series_count": int(final_series or 0), + "final_no_match_file_count": int(final_files or 0), + "final_confirmed_arc_count": int(final_arcs or 0), + "seed_elapsed_ms": seed_elapsed_ms, + "confirm_elapsed_ms": confirm_elapsed_ms, + "rollback_elapsed_ms": rollback_elapsed_ms, + "total_elapsed_ms": round((time.monotonic() - total_started) * 1000), + "confirm_select_count": select_counts.get("confirm", 0), + "rollback_select_count": select_counts.get("rollback", 0), + "peak_rss_bytes": current_process_peak_rss_bytes(), + "database_bytes": database_bytes, + } + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--backend", choices=("sqlite", "postgresql"), default="sqlite") + parser.add_argument( + "--reset-dedicated-database", + action="store_true", + help="Drop and recreate all tables; only safe for a dedicated benchmark database", + ) + parser.add_argument("--series-count", type=_positive_int, default=50_000) + parser.add_argument("--files-per-series", type=_positive_int, default=4) + parser.add_argument("--story-arc-count", type=_positive_int, default=10_000) + parser.add_argument("--insert-batch-size", type=_positive_int, default=2_000) + parser.add_argument("--operation-batch-size", type=_positive_int, default=500) + args = parser.parse_args() + if args.reset_dedicated_database and args.backend != "postgresql": + parser.error("--reset-dedicated-database requires --backend postgresql") + print(json.dumps(await _run(args), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/benchmark_import_scan.py b/scripts/benchmark_import_scan.py index 969f65a9..232b8ab1 100644 --- a/scripts/benchmark_import_scan.py +++ b/scripts/benchmark_import_scan.py @@ -9,6 +9,7 @@ import argparse import asyncio import json +import platform import tempfile import time import zipfile @@ -18,7 +19,9 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine -from pullbox.core.source_metadata import SourceMetadataExtractor +from pullbox.core import file_safety +from pullbox.core.archive import ArchiveReader +from pullbox.core.import_resources import detect_import_resources from pullbox.models import import_job as _import_job_models # noqa: F401 from pullbox.models import issue as _issue_models # noqa: F401 from pullbox.models import library as _library_models # noqa: F401 @@ -28,6 +31,7 @@ from pullbox.models.import_job import ImportJob, ImportJobStatus, ImportSourceType from pullbox.performance.baseline import current_process_peak_rss_bytes from pullbox.providers.base import IssueSummary, SeriesSearchResult +from pullbox.services import import_scan_helpers from pullbox.services.import_provider_cache import CachedImportMetadataProvider from pullbox.services.import_service import ImportService @@ -118,6 +122,7 @@ def _build_tree( series_count: int, files_per_series: int, trusted_comicinfo: bool, + archive_pages: int = 2, ) -> None: for series_idx in range(series_count): title = f"Series {series_idx:04d}" @@ -128,10 +133,8 @@ def _build_tree( for file_idx in range(1, files_per_series + 1): archive_path = folder / f"{title} #{file_idx:03d}.cbz" with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as archive: - archive.writestr( - f"{title} #{file_idx:03d}.jpg", - b"benchmark-page", - ) + for page in range(1, archive_pages + 1): + archive.writestr(f"{title} #{file_idx:03d} p{page:04d}.jpg", b"benchmark-page") if trusted_comicinfo: issue_provider_id = (series_provider_id * 1000) + file_idx archive.writestr( @@ -156,6 +159,9 @@ async def main() -> None: parser.add_argument("--series-count", type=int, default=200) parser.add_argument("--files-per-series", type=int, default=12) parser.add_argument("--trusted-comicinfo", action="store_true") + parser.add_argument("--archive-pages", type=int, default=32) + parser.add_argument("--inspection-workers", type=int, default=0, choices=range(17)) + parser.add_argument("--inspection-delay-ms", type=float, default=0) args = parser.parse_args() provider = FakeMetadataProvider() @@ -166,40 +172,70 @@ async def main() -> None: event_bus=cast("Any", SimpleNamespace()), ) benchmark_service = cast("Any", service) + benchmark_service._settings = service._settings.model_copy( + update={"import_scan_worker_count": args.inspection_workers} + ) benchmark_service._build_scan_metadata_provider = lambda _session: CachedImportMetadataProvider( provider ) archive_read_count = 0 + archive_member_payload_read_count = 0 + archive_safety_inspection_count = 0 + archive_metadata_evidence_count = 0 archive_entry_issue_hint_count = 0 commit_count = 0 - original_read_archive_comicinfo = SourceMetadataExtractor._read_archive_comicinfo - original_archive_entry_issue_hint_from_path = ( - SourceMetadataExtractor.archive_entry_issue_hint_from_path + original_list_zip = ArchiveReader._list_zip + original_read_zip = ArchiveReader._read_zip + original_inspect_zip_archive_safety = file_safety.inspect_zip_archive_safety + original_archive_entry_issue_hint_from_names = ( + import_scan_helpers.archive_entry_issue_hint_from_names ) - def counting_read_archive_comicinfo(path: Path) -> Any: + def counting_list_zip(reader: ArchiveReader) -> list[str]: nonlocal archive_read_count archive_read_count += 1 - return original_read_archive_comicinfo(path) + return original_list_zip(reader) - def counting_archive_entry_issue_hint_from_path( - path: str | Path, + def counting_read_zip( + reader: ArchiveReader, + name: str, + *, + max_bytes: int | None, + ) -> bytes: + nonlocal archive_member_payload_read_count + archive_member_payload_read_count += 1 + return original_read_zip(reader, name, max_bytes=max_bytes) + + def counting_inspect_zip_archive_safety(*call_args: Any, **call_kwargs: Any) -> Any: + nonlocal archive_metadata_evidence_count, archive_safety_inspection_count + if args.inspection_delay_ms > 0: + time.sleep(args.inspection_delay_ms / 1000) + archive_safety_inspection_count += 1 + report = original_inspect_zip_archive_safety(*call_args, **call_kwargs) + if report is not None and report.comicinfo is not None: + archive_metadata_evidence_count += 1 + return report + + def counting_archive_entry_issue_hint_from_names( + entry_names: list[str], *, expected_series_name: str | None = None, ) -> Any: nonlocal archive_entry_issue_hint_count archive_entry_issue_hint_count += 1 - return original_archive_entry_issue_hint_from_path( - path, + return original_archive_entry_issue_hint_from_names( + entry_names, expected_series_name=expected_series_name, ) - SourceMetadataExtractor._read_archive_comicinfo = staticmethod( # type: ignore[method-assign] - counting_read_archive_comicinfo + ArchiveReader._list_zip = counting_list_zip # type: ignore[method-assign] + ArchiveReader._read_zip = counting_read_zip # type: ignore[method-assign] + file_safety.inspect_zip_archive_safety = ( # type: ignore[assignment] + counting_inspect_zip_archive_safety ) - SourceMetadataExtractor.archive_entry_issue_hint_from_path = staticmethod( # type: ignore[method-assign] - counting_archive_entry_issue_hint_from_path + import_scan_helpers.archive_entry_issue_hint_from_names = ( # type: ignore[assignment] + counting_archive_entry_issue_hint_from_names ) try: with tempfile.TemporaryDirectory(prefix="pullbox-scan-bench-") as tmp: @@ -210,6 +246,7 @@ def counting_archive_entry_issue_hint_from_path( series_count=args.series_count, files_per_series=args.files_per_series, trusted_comicinfo=args.trusted_comicinfo, + archive_pages=args.archive_pages, ) db_path = Path(tmp) / "benchmark.db" @@ -245,8 +282,19 @@ async def counted_commit() -> None: "series_count": args.series_count, "files_per_series": args.files_per_series, "trusted_comicinfo": args.trusted_comicinfo, + "archive_pages": args.archive_pages, + "inspection_workers_requested": args.inspection_workers, + "simulated_inspection_delay_ms": args.inspection_delay_ms, + "inspection_workers_effective": detect_import_resources().inspection_workers( + requested=args.inspection_workers + ), + "platform": platform.platform(), + "python": platform.python_version(), "elapsed_ms": elapsed_ms, "archive_read_count": archive_read_count, + "archive_member_payload_read_count": archive_member_payload_read_count, + "archive_safety_inspection_count": archive_safety_inspection_count, + "archive_metadata_evidence_count": archive_metadata_evidence_count, "archive_entry_issue_hint_count": archive_entry_issue_hint_count, "provider_search_calls": provider.search_calls, "provider_get_series_calls": provider.series_calls, @@ -268,11 +316,13 @@ async def counted_commit() -> None: await engine.dispose() finally: - SourceMetadataExtractor._read_archive_comicinfo = staticmethod( # type: ignore[method-assign] - original_read_archive_comicinfo + ArchiveReader._list_zip = original_list_zip # type: ignore[method-assign] + ArchiveReader._read_zip = original_read_zip # type: ignore[method-assign] + file_safety.inspect_zip_archive_safety = ( # type: ignore[assignment] + original_inspect_zip_archive_safety ) - SourceMetadataExtractor.archive_entry_issue_hint_from_path = staticmethod( # type: ignore[method-assign] - original_archive_entry_issue_hint_from_path + import_scan_helpers.archive_entry_issue_hint_from_names = ( # type: ignore[assignment] + original_archive_entry_issue_hint_from_names ) diff --git a/scripts/benchmark_import_target.py b/scripts/benchmark_import_target.py new file mode 100644 index 00000000..c051a312 --- /dev/null +++ b/scripts/benchmark_import_target.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +"""Run one standardized IU7 import target lane and write bounded JSON.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +SRC_PATH = REPO_ROOT / "src" +if str(SRC_PATH) not in sys.path: + sys.path.insert(0, str(SRC_PATH)) + +from pullbox.performance.baseline import write_report # noqa: E402 +from pullbox.performance.import_target_harness import ( # noqa: E402 + ImportTargetBackend, + ImportTargetCacheState, + ImportTargetConfig, + ImportTargetScaleProfile, + ImportTargetSourceLane, + build_import_target_report, +) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", type=Path, default=REPO_ROOT) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument( + "--backend", + choices=tuple(item.value for item in ImportTargetBackend), + required=True, + ) + parser.add_argument( + "--scale-profile", + choices=tuple(item.value for item in ImportTargetScaleProfile), + required=True, + ) + parser.add_argument( + "--source-lane", + choices=tuple(item.value for item in ImportTargetSourceLane), + required=True, + ) + parser.add_argument( + "--cache-state", + choices=tuple(item.value for item in ImportTargetCacheState), + required=True, + ) + parser.add_argument("--injection-point", default="none") + parser.add_argument( + "--api-url", + action="append", + default=[], + help="Absolute endpoint as Label=https://host/path. Repeatable.", + ) + parser.add_argument("--samples", type=int, default=5) + parser.add_argument("--timeout", type=float, default=1800.0) + parser.add_argument("--environment-label", required=True) + parser.add_argument("--filesystem-label", required=True) + parser.add_argument("--output", type=Path, required=True) + return parser + + +def main() -> int: + args = _build_parser().parse_args() + try: + config = ImportTargetConfig( + repo_root=args.repo_root.resolve(), + seed=args.seed, + backend=ImportTargetBackend(args.backend), + scale_profile=ImportTargetScaleProfile(args.scale_profile), + source_lane=ImportTargetSourceLane(args.source_lane), + cache_state=ImportTargetCacheState(args.cache_state), + injection_point=args.injection_point, + api_urls=tuple(args.api_url), + samples=args.samples, + timeout_seconds=args.timeout, + environment_label=args.environment_label, + filesystem_label=args.filesystem_label, + ) + except ValueError as exc: + _build_parser().error(str(exc)) + + report = build_import_target_report(config) + write_report(report, args.output) + gates = report["gate_evaluation"] + return 0 if isinstance(gates, dict) and gates.get("passed") is True else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/benchmark_mylar3_import.py b/scripts/benchmark_mylar3_import.py index 64cc2075..96c7cf1e 100644 --- a/scripts/benchmark_mylar3_import.py +++ b/scripts/benchmark_mylar3_import.py @@ -16,12 +16,15 @@ from pathlib import Path from types import SimpleNamespace from typing import Any, cast +from unittest.mock import patch import structlog from mylar3_import_fixture import create_scaled_mylar3_fixture from sqlalchemy import func, select from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from pullbox.core import file_safety +from pullbox.core.archive import ArchiveReader from pullbox.core.events import EventBus from pullbox.models import import_job as _import_job_models # noqa: F401 from pullbox.models import issue as _issue_models # noqa: F401 @@ -91,6 +94,32 @@ async def _run_benchmark(args: argparse.Namespace, workspace: Path) -> dict[str, ) session_factory = async_sessionmaker(engine, expire_on_commit=False) error: str | None = None + archive_metrics = { + "safety_inspections": 0, + "member_list_reads": 0, + "member_payload_reads": 0, + } + original_inspect = file_safety.inspect_zip_archive_safety + original_list_zip = ArchiveReader._list_zip + original_read_zip = ArchiveReader._read_zip + + def counting_inspect(*call_args: Any, **call_kwargs: Any) -> Any: + archive_metrics["safety_inspections"] += 1 + return original_inspect(*call_args, **call_kwargs) + + def counting_list_zip(reader: ArchiveReader) -> list[str]: + archive_metrics["member_list_reads"] += 1 + return original_list_zip(reader) + + def counting_read_zip( + reader: ArchiveReader, + name: str, + *, + max_bytes: int | None, + ) -> bytes: + archive_metrics["member_payload_reads"] += 1 + return original_read_zip(reader, name, max_bytes=max_bytes) + try: async with session_factory() as session: job = ImportJob( @@ -102,10 +131,15 @@ async def _run_benchmark(args: argparse.Namespace, workspace: Path) -> dict[str, await session.commit() started_at = time.monotonic() - try: - await service.start_scan(session, job.id) - except Exception as exc: # benchmark reports failures before exiting - error = f"{type(exc).__name__}: {exc}" + with ( + patch.object(file_safety, "inspect_zip_archive_safety", counting_inspect), + patch.object(ArchiveReader, "_list_zip", counting_list_zip), + patch.object(ArchiveReader, "_read_zip", counting_read_zip), + ): + try: + await service.start_scan(session, job.id) + except Exception as exc: # benchmark reports failures before exiting + error = f"{type(exc).__name__}: {exc}" elapsed_ms = round((time.monotonic() - started_at) * 1000) await session.refresh(job) @@ -133,6 +167,10 @@ async def _run_benchmark(args: argparse.Namespace, workspace: Path) -> dict[str, ), "provider_calls": provider.calls, "provider_call_count": len(provider.calls), + "archive_safety_inspection_count": archive_metrics["safety_inspections"], + "archive_member_list_read_count": archive_metrics["member_list_reads"], + "archive_member_payload_read_count": archive_metrics["member_payload_reads"], + "archive_probe_count": sum(archive_metrics.values()), "peak_rss_bytes": current_process_peak_rss_bytes(), "final_status": job.status.value, "error": error, @@ -150,6 +188,12 @@ def _validate_report(report: dict[str, Any]) -> list[str]: failures.append(f"final status was {report['final_status']}, expected review") if report["provider_call_count"] != 0: failures.append(f"metadata provider calls occurred: {report['provider_calls']}") + if report["archive_safety_inspection_count"] != report["expected_file_count"]: + failures.append("each archive must have exactly one safety inspection") + if report["archive_member_list_read_count"] != 0: + failures.append("archive member indexes were reopened after safety inspection") + if report["archive_member_payload_read_count"] != 0: + failures.append("metadata-poor benchmark archives unexpectedly read member payloads") if report["materialized_series_count"] != report["expected_discovered_series_count"]: failures.append("materialized series count did not match the generated fixture") if report["materialized_file_count"] != report["expected_file_count"]: diff --git a/scripts/compare_import_database_parity.py b/scripts/compare_import_database_parity.py new file mode 100644 index 00000000..90b24a87 --- /dev/null +++ b/scripts/compare_import_database_parity.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +"""Compare bounded SQLite and PostgreSQL import target reports.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import cast + +REPO_ROOT = Path(__file__).resolve().parents[1] +SRC_PATH = REPO_ROOT / "src" +if str(SRC_PATH) not in sys.path: + sys.path.insert(0, str(SRC_PATH)) + +from pullbox.performance.baseline import write_report # noqa: E402 +from pullbox.performance.import_target_harness import ( # noqa: E402 + build_import_database_parity_report, +) + + +def _read_report(path: Path) -> dict[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("target report must be a JSON object") + return cast("dict[str, object]", value) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("left", type=Path) + parser.add_argument("right", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + report = build_import_database_parity_report( + _read_report(args.left), + _read_report(args.right), + ) + except (OSError, ValueError, json.JSONDecodeError) as exc: + parser.error(str(exc)) + write_report(report, args.output) + return 0 if report["passed"] is True else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_import_scale_fixture.py b/scripts/generate_import_scale_fixture.py new file mode 100644 index 00000000..c937880e --- /dev/null +++ b/scripts/generate_import_scale_fixture.py @@ -0,0 +1,88 @@ +"""Generate a deterministic Local Comic Vine-backed import scale fixture.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +_PROJECT_ROOT = str(Path(__file__).resolve().parent.parent) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + + +def build_parser() -> argparse.ArgumentParser: + """Build the scale-fixture command-line contract.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("catalog", type=Path, help="Read-only localcv.db path") + parser.add_argument("output", type=Path, help="Output path, which must not exist") + parser.add_argument("--series-count", type=int, default=50_000) + parser.add_argument("--file-count", type=int, default=200_000) + parser.add_argument("--seed", type=int, default=1300) + parser.add_argument("--archive-pages", type=int, default=32) + parser.add_argument( + "--single-page-every", + type=int, + default=0, + help="Include an explicit one-page safety exception every N files (0 disables)", + ) + parser.add_argument( + "--profile", + choices=("balanced", "realistic-skew"), + default="balanced", + ) + parser.add_argument("--max-issues-per-series", type=int, default=250) + parser.add_argument( + "--layout-profile", + choices=("series", "mixed"), + default="series", + help="Use uniform series folders for certification or mixed layouts for stress testing", + ) + parser.add_argument( + "--plan-only", + action="store_true", + help="Select and summarize the workload without writing fixture files", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Plan or generate one exact fixture.""" + from scripts.import_scale_fixtures.generator import ( + FixtureRequest, + generate_import_scale_fixture, + plan_import_scale_fixture, + ) + + args = build_parser().parse_args(argv) + request = FixtureRequest( + catalog_path=args.catalog, + output_path=args.output, + series_count=args.series_count, + file_count=args.file_count, + seed=args.seed, + archive_pages=args.archive_pages, + single_page_every=args.single_page_every, + profile=args.profile, + max_issues_per_series=args.max_issues_per_series, + layout_profile=args.layout_profile, + ) + if args.plan_only: + plan = plan_import_scale_fixture(request) + summary: dict[str, object] = { + "profile": plan.profile, + "seed": plan.seed, + "series_count": len(plan.series), + "file_count": sum(len(series.issues) for series in plan.series), + "max_issues_per_series": plan.max_issues_per_series, + "layout_profile": plan.layout_profile, + } + else: + summary = generate_import_scale_fixture(request) + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_iu9_folder_fixture.py b/scripts/generate_iu9_folder_fixture.py new file mode 100644 index 00000000..3a49746f --- /dev/null +++ b/scripts/generate_iu9_folder_fixture.py @@ -0,0 +1,55 @@ +"""CLI entrypoint for deterministic IU9 folder-import fixtures.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +_PROJECT_ROOT = str(Path(__file__).resolve().parent.parent) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + + +def build_parser() -> argparse.ArgumentParser: + """Build the folder-fixture command-line contract.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path, help="Absent or empty output directory") + parser.add_argument("--seed", type=int, default=1300, help="Deterministic fixture seed") + parser.add_argument( + "--cbr-seed-dir", + type=Path, + help="Optional directory containing fixed RAR3/RAR5 CBR seeds and descriptor", + ) + parser.add_argument("--cbr-rar3-sha256", help="Optional SHA-256 pin for iu9-rar3.cbr") + parser.add_argument("--cbr-rar5-sha256", help="Optional SHA-256 pin for iu9-rar5.cbr") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Generate one fresh folder-import fixture tree.""" + from scripts.iu9_acceptance_fixtures.folder import generate_folder_fixture + + parser = build_parser() + args = parser.parse_args(argv) + pins = { + seed_id: digest + for seed_id, digest in ( + ("rar3", args.cbr_rar3_sha256), + ("rar5", args.cbr_rar5_sha256), + ) + if digest is not None + } + if pins and args.cbr_seed_dir is None: + parser.error("CBR SHA-256 pins require --cbr-seed-dir") + generate_folder_fixture( + args.output, + seed=args.seed, + cbr_seed_dir=args.cbr_seed_dir, + cbr_expected_sha256=pins or None, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_iu9_mylar_fixture.py b/scripts/generate_iu9_mylar_fixture.py new file mode 100644 index 00000000..b802997d --- /dev/null +++ b/scripts/generate_iu9_mylar_fixture.py @@ -0,0 +1,55 @@ +"""CLI entrypoint for deterministic IU9 Mylar-import fixtures.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +_PROJECT_ROOT = str(Path(__file__).resolve().parent.parent) +if _PROJECT_ROOT not in sys.path: + sys.path.insert(0, _PROJECT_ROOT) + + +def build_parser() -> argparse.ArgumentParser: + """Build the Mylar-fixture command-line contract.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path, help="Absent or empty output directory") + parser.add_argument("--seed", type=int, default=1300, help="Deterministic fixture seed") + parser.add_argument( + "--cbr-seed-dir", + type=Path, + help="Optional directory containing fixed RAR3/RAR5 CBR seeds and descriptor", + ) + parser.add_argument("--cbr-rar3-sha256", help="Optional SHA-256 pin for iu9-rar3.cbr") + parser.add_argument("--cbr-rar5-sha256", help="Optional SHA-256 pin for iu9-rar5.cbr") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Generate one fresh Mylar-import fixture tree.""" + from scripts.iu9_acceptance_fixtures.mylar import generate_mylar_fixture + + parser = build_parser() + args = parser.parse_args(argv) + pins = { + seed_id: digest + for seed_id, digest in ( + ("rar3", args.cbr_rar3_sha256), + ("rar5", args.cbr_rar5_sha256), + ) + if digest is not None + } + if pins and args.cbr_seed_dir is None: + parser.error("CBR SHA-256 pins require --cbr-seed-dir") + generate_mylar_fixture( + args.output, + seed=args.seed, + cbr_seed_dir=args.cbr_seed_dir, + cbr_expected_sha256=pins or None, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/import_scale_fixtures/__init__.py b/scripts/import_scale_fixtures/__init__.py new file mode 100644 index 00000000..963b26cd --- /dev/null +++ b/scripts/import_scale_fixtures/__init__.py @@ -0,0 +1 @@ +"""Deterministic, local-only fixtures for import scale certification.""" diff --git a/scripts/import_scale_fixtures/generator.py b/scripts/import_scale_fixtures/generator.py new file mode 100644 index 00000000..ab1e36b7 --- /dev/null +++ b/scripts/import_scale_fixtures/generator.py @@ -0,0 +1,503 @@ +"""Build deterministic import fixtures from a read-only Local Comic Vine catalog.""" + +from __future__ import annotations + +import hashlib +import json +import random +import re +import shutil +import sqlite3 +import tempfile +import unicodedata +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Literal +from urllib.parse import quote + +from scripts.iu9_acceptance_fixtures.shared import create_deterministic_cbz, sha256_file + +FixtureProfile = Literal["balanced", "realistic-skew"] +LayoutProfile = Literal["series", "mixed"] +_INVALID_SEGMENT = re.compile(r'[<>:"/\\|?*\x00-\x1f]') +_WINDOWS_RESERVED = { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{number}" for number in range(1, 10)), + *(f"LPT{number}" for number in range(1, 10)), +} + + +@dataclass(frozen=True, slots=True) +class FixtureRequest: + """One exact fixture-generation request.""" + + catalog_path: Path + output_path: Path + series_count: int + file_count: int + seed: int = 1300 + profile: FixtureProfile = "balanced" + max_issues_per_series: int = 250 + layout_profile: LayoutProfile = "series" + archive_pages: int = 32 + single_page_every: int = 0 + + +@dataclass(frozen=True, slots=True) +class PlannedIssue: + """A genuine Comic Vine issue selected for the fixture.""" + + issue_id: int + volume_id: int + name: str | None + issue_number: str + year: int | None + + +@dataclass(frozen=True, slots=True) +class PlannedSeries: + """A selected Comic Vine volume and its exact fixture issues.""" + + volume_id: int + name: str + start_year: int | None + publisher: str | None + issues: tuple[PlannedIssue, ...] + + +@dataclass(frozen=True, slots=True) +class FixturePlan: + """A deterministic plan that can be inspected before filesystem generation.""" + + seed: int + profile: FixtureProfile + max_issues_per_series: int + layout_profile: LayoutProfile + series: tuple[PlannedSeries, ...] + + +@dataclass(frozen=True, slots=True) +class _CatalogSeries: + volume_id: int + name: str + start_year: int | None + publisher: str | None + issue_count: int + + +def _validate_request(request: FixtureRequest) -> None: + if not 2 <= request.archive_pages <= 1000 or request.single_page_every < 0: + raise ValueError("archive_pages must be 2-1000 and single_page_every nonnegative") + if request.series_count < 1: + raise ValueError("series_count must be positive") + if request.file_count < request.series_count: + raise ValueError("file_count must be at least series_count") + if request.max_issues_per_series < 1: + raise ValueError("max_issues_per_series must be positive") + if request.profile not in {"balanced", "realistic-skew"}: + raise ValueError(f"Unsupported fixture profile: {request.profile}") + if request.layout_profile not in {"series", "mixed"}: + raise ValueError(f"Unsupported layout profile: {request.layout_profile}") + if not request.catalog_path.is_file(): + raise FileNotFoundError(f"Comic Vine catalog does not exist: {request.catalog_path}") + + +def _open_catalog(path: Path) -> sqlite3.Connection: + encoded_path = quote(str(path.expanduser().absolute()), safe="/") + connection = sqlite3.connect(f"file:{encoded_path}?mode=ro", uri=True) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA query_only = ON") + connection.execute("PRAGMA temp_store = MEMORY") + return connection + + +def _catalog_year(value: object) -> int | None: + text = str(value).strip() if value is not None else "" + if len(text) != 4 or not text.isdigit(): + return None + return int(text) + + +def _load_catalog_series(connection: sqlite3.Connection) -> list[_CatalogSeries]: + rows = connection.execute( + """ + SELECT + volume.id AS volume_id, + volume.name AS volume_name, + volume.start_year AS start_year, + publisher.name AS publisher_name, + COUNT(issue.id) AS issue_count + FROM cv_volume AS volume + JOIN cv_issue AS issue ON issue.volume_id = volume.id + LEFT JOIN cv_publisher AS publisher ON publisher.id = volume.publisher_id + WHERE TRIM(COALESCE(volume.name, '')) <> '' + AND TRIM(COALESCE(issue.issue_number, '')) <> '' + GROUP BY volume.id, volume.name, volume.start_year, publisher.name + ORDER BY volume.id + """ + ).fetchall() + return [ + _CatalogSeries( + volume_id=int(row["volume_id"]), + name=str(row["volume_name"]), + start_year=_catalog_year(row["start_year"]), + publisher=str(row["publisher_name"]) if row["publisher_name"] else None, + issue_count=int(row["issue_count"]), + ) + for row in rows + ] + + +def _balanced_assignments( + candidates: list[_CatalogSeries], request: FixtureRequest +) -> list[tuple[_CatalogSeries, int]]: + base = request.file_count // request.series_count + shuffled = candidates.copy() + random.Random(request.seed).shuffle(shuffled) + selected: list[_CatalogSeries] = [] + selected_ids: set[int] = set() + for threshold in range(base, 0, -1): + for candidate in shuffled: + if candidate.volume_id in selected_ids or candidate.issue_count < threshold: + continue + selected.append(candidate) + selected_ids.add(candidate.volume_id) + if len(selected) == request.series_count: + break + if len(selected) == request.series_count: + break + if len(selected) != request.series_count: + raise ValueError( + "Comic Vine catalog capacity is insufficient for the requested series count" + ) + + assignments = [min(base, candidate.issue_count) for candidate in selected] + remaining = request.file_count - sum(assignments) + while remaining: + progressed = False + for index, candidate in enumerate(selected): + capacity = min(candidate.issue_count, request.max_issues_per_series) + if assignments[index] >= capacity: + continue + assignments[index] += 1 + remaining -= 1 + progressed = True + if remaining == 0: + break + if not progressed: + raise ValueError( + "Comic Vine catalog capacity is insufficient for the requested file count" + ) + return list(zip(selected, assignments, strict=True)) + + +def _realistic_skew_assignments( + candidates: list[_CatalogSeries], request: FixtureRequest +) -> list[tuple[_CatalogSeries, int]]: + shuffled = candidates.copy() + random.Random(request.seed).shuffle(shuffled) + if len(shuffled) < request.series_count: + raise ValueError( + "Comic Vine catalog capacity is insufficient for the requested series count" + ) + selected = shuffled[: request.series_count] + unselected = shuffled[request.series_count :] + + def capacity(candidate: _CatalogSeries) -> int: + return min(candidate.issue_count, request.max_issues_per_series) + + while sum(capacity(candidate) for candidate in selected) < request.file_count: + lowest_index = min(range(len(selected)), key=lambda index: capacity(selected[index])) + if not unselected: + break + highest_index = max(range(len(unselected)), key=lambda index: capacity(unselected[index])) + if capacity(unselected[highest_index]) <= capacity(selected[lowest_index]): + break + selected[lowest_index], unselected[highest_index] = ( + unselected[highest_index], + selected[lowest_index], + ) + + capacities = [capacity(candidate) for candidate in selected] + if sum(capacities) < request.file_count: + raise ValueError( + "Comic Vine catalog capacity is insufficient for the requested file count and cap" + ) + assignments = [1] * len(selected) + remaining = request.file_count - len(selected) + available = [item - 1 for item in capacities] + total_available = sum(available) + quotas = [remaining * item / total_available for item in available] + floors = [int(quota) for quota in quotas] + assignments = [current + extra for current, extra in zip(assignments, floors, strict=True)] + residual = remaining - sum(floors) + remainder_order = sorted( + range(len(selected)), + key=lambda index: (quotas[index] - floors[index], -index), + reverse=True, + ) + for index in remainder_order[:residual]: + assignments[index] += 1 + return list(zip(selected, assignments, strict=True)) + + +def _stable_issue_seed(seed: int, volume_id: int) -> int: + digest = hashlib.sha256(f"{seed}:{volume_id}".encode()).digest() + return int.from_bytes(digest[:8], "big") + + +def _issue_year(value: object, fallback: int | None) -> int | None: + if isinstance(value, str) and len(value) >= 4 and value[:4].isdigit(): + return int(value[:4]) + return fallback + + +def _load_selected_issues( + connection: sqlite3.Connection, + assignments: list[tuple[_CatalogSeries, int]], + *, + seed: int, +) -> dict[int, tuple[PlannedIssue, ...]]: + candidates_by_volume: dict[int, list[PlannedIssue]] = defaultdict(list) + series_by_id = {candidate.volume_id: candidate for candidate, _count in assignments} + volume_ids = list(series_by_id) + for offset in range(0, len(volume_ids), 500): + batch = volume_ids[offset : offset + 500] + placeholders = ",".join("?" for _item in batch) + rows = connection.execute( + f""" + SELECT id, volume_id, name, issue_number, cover_date + FROM cv_issue + WHERE volume_id IN ({placeholders}) + AND TRIM(COALESCE(issue_number, '')) <> '' + ORDER BY volume_id, id + """, + batch, + ).fetchall() + for row in rows: + volume_id = int(row["volume_id"]) + candidates_by_volume[volume_id].append( + PlannedIssue( + issue_id=int(row["id"]), + volume_id=volume_id, + name=str(row["name"]) if row["name"] else None, + issue_number=str(row["issue_number"]), + year=_issue_year(row["cover_date"], series_by_id[volume_id].start_year), + ) + ) + + selected: dict[int, tuple[PlannedIssue, ...]] = {} + for candidate, requested_count in assignments: + issues = candidates_by_volume[candidate.volume_id] + random.Random(_stable_issue_seed(seed, candidate.volume_id)).shuffle(issues) + chosen = tuple(sorted(issues[:requested_count], key=lambda issue: issue.issue_id)) + if len(chosen) != requested_count: + raise ValueError( + f"Comic Vine catalog capacity changed for volume {candidate.volume_id}" + ) + selected[candidate.volume_id] = chosen + return selected + + +def plan_import_scale_fixture(request: FixtureRequest) -> FixturePlan: + """Select an exact deterministic workload without modifying the filesystem.""" + _validate_request(request) + with _open_catalog(request.catalog_path) as connection: + candidates = _load_catalog_series(connection) + if request.profile == "balanced": + assignments = _balanced_assignments(candidates, request) + else: + assignments = _realistic_skew_assignments(candidates, request) + issues_by_volume = _load_selected_issues(connection, assignments, seed=request.seed) + series = tuple( + PlannedSeries( + volume_id=candidate.volume_id, + name=candidate.name, + start_year=candidate.start_year, + publisher=candidate.publisher, + issues=issues_by_volume[candidate.volume_id], + ) + for candidate, _count in assignments + ) + return FixturePlan( + seed=request.seed, + profile=request.profile, + max_issues_per_series=request.max_issues_per_series, + layout_profile=request.layout_profile, + series=series, + ) + + +def _xml_safe(value: str | None) -> str | None: + if value is None: + return None + return "".join( + character + for character in value + if character in "\t\n\r" + or "\x20" <= character <= "\ud7ff" + or "\ue000" <= character <= "\ufffd" + ) + + +def _truncate_utf8(value: str, max_bytes: int) -> str: + encoded = value.encode("utf-8") + if len(encoded) <= max_bytes: + return value + encoded = encoded[:max_bytes] + while encoded: + try: + return encoded.decode("utf-8").rstrip(" .") + except UnicodeDecodeError: + encoded = encoded[:-1] + return "Unknown" + + +def _safe_segment(value: str | None, *, max_bytes: int = 100) -> str: + normalized = unicodedata.normalize("NFC", _xml_safe(value) or "Unknown") + cleaned = _INVALID_SEGMENT.sub("-", normalized) + cleaned = " ".join(cleaned.split()).strip(" .") or "Unknown" + if cleaned.casefold().split(".", maxsplit=1)[0].upper() in _WINDOWS_RESERVED: + cleaned = f"_{cleaned}" + return _truncate_utf8(cleaned, max_bytes) + + +def _series_directory( + series: PlannedSeries, + index: int, + *, + layout_profile: LayoutProfile, +) -> Path: + year = str(series.start_year) if series.start_year is not None else "Unknown Year" + identity_suffix = f" ({year}) [cv-{series.volume_id}]" + name_budget = max(1, 140 - len(identity_suffix.encode("utf-8"))) + series_name = _safe_segment(series.name, max_bytes=name_budget) + series_segment = f"{series_name}{identity_suffix}" + if layout_profile == "series": + return Path(series_segment) + layout = index % 10 + if layout < 7: + return Path(series_segment) + if layout < 9: + return Path("Publishers") / _safe_segment(series.publisher) / series_segment + initial = _safe_segment(series.name[:1].upper() or "#", max_bytes=8) + return Path("Collections") / initial / series_segment + + +def _issue_filename(issue: PlannedIssue) -> str: + number = _safe_segment(issue.issue_number, max_bytes=24) + title = _safe_segment(issue.name or "Untitled", max_bytes=72) + return f"Issue {number} - {title} [cv-issue-{issue.issue_id}].cbz" + + +def _selection_digest(plan: FixturePlan) -> str: + digest = hashlib.sha256() + for series in plan.series: + digest.update(f"v:{series.volume_id}\n".encode()) + for issue in series.issues: + digest.update(f"i:{issue.issue_id}:{issue.issue_number}\n".encode()) + return digest.hexdigest() + + +def generate_import_scale_fixture(request: FixtureRequest) -> dict[str, object]: + """Generate an immutable-style source tree and streaming evidence manifest.""" + output = request.output_path.expanduser().absolute() + if output.exists(): + raise FileExistsError(f"Fixture output must not already exist: {output}") + plan = plan_import_scale_fixture(request) + output.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=f".{output.name}.building-", dir=output.parent)) + manifest_path = staging / "fixture-manifest.jsonl" + path_samples: list[str] = [] + issue_distribution: dict[str, int] = defaultdict(int) + content_digest = hashlib.sha256() + logical_bytes = 0 + ordinal = 0 + try: + with manifest_path.open("w", encoding="utf-8", newline="\n") as manifest: + for series_index, series in enumerate(plan.series): + directory = _series_directory( + series, + series_index, + layout_profile=request.layout_profile, + ) + issue_distribution[str(len(series.issues))] += 1 + for issue in series.issues: + ordinal += 1 + pages = ( + 1 + if request.single_page_every and ordinal % request.single_page_every == 0 + else request.archive_pages + ) + relative_path = directory / _issue_filename(issue) + relative_text = relative_path.as_posix() + archive_path = staging / "source" / relative_path + create_deterministic_cbz( + archive_path, + seed=request.seed, + case_id=f"cv-{series.volume_id}-{issue.issue_id}", + series=_xml_safe(series.name) or "Unknown", + number=_xml_safe(issue.issue_number) or "Unknown", + title=_xml_safe(issue.name), + year=issue.year, + publisher=_xml_safe(series.publisher), + comicvine_series_id=series.volume_id, + comicvine_issue_id=issue.issue_id, + page_count=pages, + ) + archive_digest = sha256_file(archive_path) + archive_size = archive_path.stat().st_size + logical_bytes += archive_size + row = { + "archive_sha256": archive_digest, + "archive_size": archive_size, + "page_count": pages, + "expected_safety_review": pages == 1, + "issue_id": issue.issue_id, + "issue_number": _xml_safe(issue.issue_number), + "issue_title": _xml_safe(issue.name), + "publisher": _xml_safe(series.publisher), + "relative_path": relative_text, + "series_name": _xml_safe(series.name), + "start_year": series.start_year, + "volume_id": series.volume_id, + } + manifest.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + content_digest.update(f"{relative_text}\0{archive_digest}\n".encode()) + if len(path_samples) < 10: + path_samples.append(relative_text) + summary: dict[str, object] = { + "schema_version": 1, + "fixture_kind": "comicvine-import-scale", + "profile": request.profile, + "seed": request.seed, + "series_count": len(plan.series), + "file_count": sum(len(series.issues) for series in plan.series), + "max_issues_per_series": request.max_issues_per_series, + "layout_profile": request.layout_profile, + "archive_pages": request.archive_pages, + "single_page_every": request.single_page_every, + "single_page_files": ordinal // request.single_page_every + if request.single_page_every + else 0, + "issue_count_distribution": dict( + sorted(issue_distribution.items(), key=lambda row: int(row[0])) + ), + "logical_bytes": logical_bytes, + "selection_sha256": _selection_digest(plan), + "content_sha256": content_digest.hexdigest(), + "path_samples": path_samples, + } + (staging / "fixture-summary.json").write_text( + json.dumps(summary, indent=2, ensure_ascii=False, sort_keys=True) + "\n", + encoding="utf-8", + ) + staging.rename(output) + return summary + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise diff --git a/scripts/import_scale_fixtures/mylar_multipage.py b/scripts/import_scale_fixtures/mylar_multipage.py new file mode 100644 index 00000000..e3d0d492 --- /dev/null +++ b/scripts/import_scale_fixtures/mylar_multipage.py @@ -0,0 +1,187 @@ +"""Create a separate multipage copy of the synthetic Mylar scale fixture.""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sqlite3 +import tempfile +import time +import zipfile +from contextlib import closing +from pathlib import Path, PurePosixPath + +from scripts.iu9_acceptance_fixtures.shared import sha256_file + + +def _relative_path(folder: str, name: str, root: PurePosixPath) -> Path: + relative = PurePosixPath(folder).relative_to(root) / name + if relative.is_absolute() or ".." in relative.parts or "\\" in str(relative): + raise ValueError("Fixture source path must stay inside its recorded root") + return Path(*relative.parts) + + +def _copy_archive(original: Path, destination: Path, pages: int) -> None: + if original.stat().st_size > 1_000_000: + raise ValueError("Only tiny synthetic one-page archives may be upgraded") + with zipfile.ZipFile(original) as archive: + names = archive.namelist() + images = [name for name in names if name.endswith(".png")] + if len(images) != 1 or set(names) != {images[0], "ComicInfo.xml"}: + raise ValueError("Expected synthetic PNG plus ComicInfo.xml only") + if any(info.file_size > 1_000_000 for info in archive.infolist()): + raise ValueError("Synthetic member exceeds fixture size limit") + metadata, image = archive.read("ComicInfo.xml"), archive.read(images[0]) + destination.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(destination, "x", compression=zipfile.ZIP_DEFLATED) as archive: + for name, data in [ + ("ComicInfo.xml", metadata), + *[(f"pages/{number:03d}.png", image) for number in range(1, pages + 1)], + ]: + info = zipfile.ZipInfo(name, date_time=(2020, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, data) + + +def upgrade_fixture( + source: Path, + output: Path, + *, + container_root: str, + pages: int = 32, + single_page_every: int = 2000, +) -> dict[str, object]: + """Preserve source data while generating a new synthetic benchmark fixture.""" + started = time.monotonic() + source, output = source.resolve(), output.absolute() + if output.exists(): + raise FileExistsError("Output must not already exist") + if output.is_relative_to(source): + raise ValueError("Output must be separate from the original fixture") + if not 2 <= pages <= 1000 or single_page_every < 0: + raise ValueError("Invalid page count or safety exception interval") + report = json.loads((source / "generation-report.json").read_text()) + if report.get("fixture_kind") != "mylar-import-from-cv-stress-manifest": + raise ValueError("Source is not the supported synthetic Mylar fixture") + old_root, new_root = ( + PurePosixPath(report["recorded_source_root"]), + PurePosixPath(container_root), + ) + if not new_root.is_absolute() or ".." in new_root.parts: + raise ValueError("Container root must be an absolute safe path") + before = sha256_file(source / "mylar.db") + output.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=".mylar-multipage-", dir=output.parent)) + count = 0 + total_bytes = 0 + try: + with ( + closing( + sqlite3.connect(f"{(source / 'mylar.db').as_uri()}?mode=ro", uri=True) + ) as original, + closing(sqlite3.connect(staging / "mylar.db")) as db, + ): + original.backup(db) + rows = original.execute( + "SELECT i.IssueID, c.ComicLocation, i.Location FROM issues i " + "JOIN comics c ON c.ComicID=i.ComicID ORDER BY i.IssueID" + ) + with (staging / "fixture-manifest.jsonl").open("w") as manifest: + for issue_id, folder, name in rows: + relative = _relative_path(folder, name, old_root) + source_path = source / "source" / relative + if source_path.resolve() != source_path or not source_path.is_file(): + raise ValueError("Fixture paths must be regular files without symlinks") + count += 1 + page_count = ( + 1 if single_page_every and count % single_page_every == 0 else pages + ) + target = staging / "source" / relative + _copy_archive(source_path, target, page_count) + size = target.stat().st_size + total_bytes += size + db.execute( + "UPDATE issues SET ComicSize=? WHERE IssueID=?", (str(size), issue_id) + ) + manifest.write( + json.dumps( + { + "issue_id": issue_id, + "relative_path": relative.as_posix(), + "archive_sha256": sha256_file(target), + "archive_size": size, + "page_count": page_count, + "expected_safety_review": page_count == 1, + }, + ensure_ascii=False, + sort_keys=True, + ) + + "\n" + ) + for rowid, folder in original.execute("SELECT rowid, ComicLocation FROM comics"): + relative = _relative_path(folder, "", old_root) + db.execute( + "UPDATE comics SET ComicLocation=? WHERE rowid=?", + ((new_root / relative.as_posix()).as_posix(), rowid), + ) + if count != report["issue_count"]: + raise ValueError("Fixture issue count differs from the generation report") + integrity = db.execute("PRAGMA integrity_check").fetchone()[0] + if integrity != "ok": + raise ValueError("Generated database failed integrity check") + db.commit() + if sha256_file(source / "mylar.db") != before: + raise ValueError("Source database changed during generation") + result: dict[str, object] = { + "fixture_kind": "mylar-import-multipage", + "source_fixture": str(source), + "recorded_source_root": container_root, + "series_count": report["series_count"], + "issue_count": count, + "archive_pages": pages, + "single_page_every": single_page_every, + "single_page_files": count // single_page_every if single_page_every else 0, + "archive_bytes": total_bytes, + "sqlite_integrity_check": integrity, + "source_database_sha256": before, + "database_sha256": sha256_file(staging / "mylar.db"), + "elapsed_seconds": round(time.monotonic() - started, 3), + "notes": [ + "Generated image members, not real comic payloads or disk-throughput evidence.", + "Original fixture and all original archive metadata are preserved.", + "Single-page exceptions are expected to require safety review.", + ], + } + (staging / "generation-report.json").write_text(json.dumps(result, indent=2) + "\n") + staging.rename(output) + return result + except BaseException: + shutil.rmtree(staging) + raise + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--container-root", required=True) + parser.add_argument("--pages", type=int, default=32) + parser.add_argument("--single-page-every", type=int, default=2000) + args = parser.parse_args() + print( + json.dumps( + upgrade_fixture( + args.source, + args.output, + container_root=args.container_root, + pages=args.pages, + single_page_every=args.single_page_every, + ), + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/install_ci_tool.sh b/scripts/install_ci_tool.sh index 76bcef6d..09c5fe60 100755 --- a/scripts/install_ci_tool.sh +++ b/scripts/install_ci_tool.sh @@ -3,7 +3,7 @@ set -euo pipefail if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then - echo "Usage: $0 [install-dir]" >&2 + echo "Usage: $0 [install-dir]" >&2 exit 1 fi @@ -53,6 +53,22 @@ case "${tool}" in archive="${binary}_${version}_${os}_${archive_arch}.tar.gz" version_args="version" ;; + grype) + version="0.110.0" + binary="grype" + checksum_file="grype_${version}_checksums.txt" + base_url="https://github.com/anchore/grype/releases/download/v${version}" + case "${arch}" in + x86_64|amd64) archive_arch="amd64" ;; + arm64|aarch64) archive_arch="arm64" ;; + *) + echo "Unsupported grype architecture: ${arch}" >&2 + exit 1 + ;; + esac + archive="${binary}_${version}_${os}_${archive_arch}.tar.gz" + version_args="version" + ;; *) echo "Unsupported tool: ${tool}" >&2 exit 1 diff --git a/scripts/iu9_acceptance_fixtures/__init__.py b/scripts/iu9_acceptance_fixtures/__init__.py new file mode 100644 index 00000000..c70e8829 --- /dev/null +++ b/scripts/iu9_acceptance_fixtures/__init__.py @@ -0,0 +1 @@ +"""Deterministic fixture generators for the IU9 live acceptance matrix.""" diff --git a/scripts/iu9_acceptance_fixtures/folder.py b/scripts/iu9_acceptance_fixtures/folder.py new file mode 100644 index 00000000..7275aad3 --- /dev/null +++ b/scripts/iu9_acceptance_fixtures/folder.py @@ -0,0 +1,526 @@ +"""Deterministic real-world folder-import fixture generator.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +from typing import TYPE_CHECKING + +from .shared import ( + consume_cbr_seed_set, + create_deterministic_cbz, + create_deterministic_zip, + deterministic_jpeg, + prepare_fixture_root, + snapshot_tree, + write_bytes, + write_manifest, + write_text, +) + +if TYPE_CHECKING: + from pathlib import Path + +_SCHEMA_VERSION = 1 + + +def _relative(root: Path, path: Path) -> str: + return path.relative_to(root).as_posix() + + +def _synthetic_series_id(series: str) -> int: + digest = hashlib.sha256(series.casefold().encode("utf-8")).digest() + return 500_000 + (int.from_bytes(digest[:4], "big") % 400_000) + + +def _case( + case_id: str, + *, + paths: list[str], + expected_outcome: str, + tags: list[str], + issue_number: str | None = None, + archive_format: str | None = None, + note: str | None = None, +) -> dict[str, object]: + row: dict[str, object] = { + "id": case_id, + "paths": paths, + "expected_outcome": expected_outcome, + "tags": tags, + } + if issue_number is not None: + row["issue_number"] = issue_number + if archive_format is not None: + row["archive_format"] = archive_format + if note is not None: + row["note"] = note + return row + + +def generate_folder_fixture( + output_root: Path, + *, + seed: int = 1300, + cbr_seed_dir: Path | None = None, + cbr_expected_sha256: dict[str, str] | None = None, +) -> Path: + """Generate resettable folder trees and a relative acceptance manifest.""" + root = prepare_fixture_root(output_root) + source_a = root / "roots" / "source-a" + source_b = root / "roots" / "source-b" + source_a.mkdir(parents=True) + source_b.mkdir(parents=True) + cases: list[dict[str, object]] = [] + + def add_cbz( + case_id: str, + relative_path: str, + *, + series: str, + number: str, + title: str | None = None, + year: int = 2026, + publisher: str = "Fixture House", + expected_outcome: str = "success", + tags: list[str] | None = None, + metadata_number: str | None = None, + comicvine_series_id: int | None = None, + comicvine_issue_id: int | None = None, + include_comicinfo: bool = True, + identity_source: str = "comicinfo", + ) -> Path: + path = root / relative_path + resolved_series_id = comicvine_series_id or _synthetic_series_id(series) + resolved_issue_id = comicvine_issue_id or 700_000 + len(cases) + create_deterministic_cbz( + path, + seed=seed, + case_id=case_id, + series=series, + number=metadata_number or number, + title=title, + year=year, + publisher=publisher, + comicvine_series_id=resolved_series_id if include_comicinfo else None, + comicvine_issue_id=resolved_issue_id if include_comicinfo else None, + include_comicinfo=include_comicinfo, + ) + case = _case( + case_id, + paths=[relative_path], + expected_outcome=expected_outcome, + tags=tags or [], + issue_number=number, + archive_format="cbz", + ) + if metadata_number is not None: + case["comicinfo_number"] = metadata_number + case["metadata_profile"] = "canonical-comicinfo" if include_comicinfo else "pages-only" + case["identity_source"] = identity_source + if include_comicinfo: + case["comicvine_series_id"] = resolved_series_id + case["comicvine_issue_id"] = resolved_issue_id + cases.append(case) + return path + + issue_only = add_cbz( + "issue_only_filename", + "roots/source-a/Absolute Batman/Issue 01.cbz", + series="Absolute Batman", + number="1", + title="The Zoo", + year=2024, + publisher="DC Comics", + expected_outcome="review", + tags=[ + "series-folder", + "filename-without-series", + "minimal-layout", + "metadata-poor", + ], + include_comicinfo=False, + identity_source="folder-and-filename", + ) + add_cbz( + "publisher_series_layout", + "roots/source-a/DC Comics/Absolute Batman/Issue 02.cbz", + series="Absolute Batman", + number="2", + title="The Bat", + year=2024, + publisher="DC Comics", + expected_outcome="review", + tags=[ + "publisher-folder", + "series-folder", + "filename-without-series", + "metadata-poor", + ], + include_comicinfo=False, + identity_source="publisher-folder-and-filename", + ) + add_cbz( + "issue_title_filename", + ("roots/source-a/Batman (2011)/Batman The Court of Owls, Part One Issue 001.cbz"), + series="Batman", + number="1", + title="The Court of Owls, Part One", + year=2011, + publisher="DC Comics", + expected_outcome="review", + tags=["issue-title", "issue-token", "series-year-folder", "metadata-poor"], + include_comicinfo=False, + identity_source="filename", + ) + + number_cases = ( + ("number_zero", "Issue 0.cbz", "0"), + ("number_dot_five", "Issue 0.5.cbz", "0.5"), + ("number_leading_dot_five", "Issue .5.cbz", ".5"), + ("number_half", "Issue One Half.cbz", "1/2"), + ("number_suffix", "Issue 1A.cbz", "1A"), + ("number_ten_thousand", "Issue 10000.cbz", "10000"), + ("number_one_million", "Issue 1000000.cbz", "1000000"), + ) + for case_id, filename, number in number_cases: + add_cbz( + case_id, + f"roots/source-a/Number Lab (2026)/{filename}", + series="Number Lab", + number=number, + title=f"Number {number}", + tags=["issue-number-edge", "exact-text"], + ) + + add_cbz( + "unicode_punctuation", + "roots/source-a/Étoiles & L\u2019ombre (2025)/Étoiles \u2014 Issue 001 (日本語).cbz", + series="Étoiles & L\u2019ombre", + number="1", + title="L\u2019été \u2014 日本語", + year=2025, + tags=["unicode", "punctuation", "ampersand", "apostrophe"], + ) + add_cbz( + "nested_generic_containers", + "roots/source-a/Comics/Archive/By Publisher/Indie/Deep Series/Issue 003.cbz", + series="Deep Series", + number="3", + tags=["deep-layout", "generic-containers", "filename-without-series"], + ) + add_cbz( + "uppercase_extension", + "roots/source-a/Upper Case (2022)/Upper Case 001.CBZ", + series="Upper Case", + number="1", + year=2022, + tags=["uppercase-extension"], + ) + add_cbz( + "conflicting_metadata", + "roots/source-a/Conflict Series (2023)/Conflict Series Issue 004.cbz", + series="Conflict Series", + number="4", + metadata_number="5", + year=2023, + expected_outcome="review", + tags=["metadata-conflict", "filename-versus-comicinfo"], + ) + + loose_one = add_cbz( + "loose_mixed_series", + "roots/source-b/Loose Imports/Alpha 001.cbz", + series="Alpha", + number="1", + year=2019, + tags=["loose-files", "mixed-series"], + ) + loose_two = root / "roots/source-b/Loose Imports/Beta 007.cbz" + create_deterministic_cbz( + loose_two, + seed=seed, + case_id="loose_mixed_series_beta", + series="Beta", + number="7", + title="Seven", + year=2020, + publisher="Fixture House", + comicvine_series_id=_synthetic_series_id("Beta"), + comicvine_issue_id=799_007, + ) + cases[-1]["paths"] = [_relative(root, loose_one), _relative(root, loose_two)] + + duplicate_source = add_cbz( + "duplicate_identical", + "roots/source-b/Duplicates/Copy A/Duplicate Series 001.cbz", + series="Duplicate Series", + number="1", + expected_outcome="review", + tags=["duplicate-content", "identical-bytes"], + ) + duplicate_copy = root / "roots/source-b/Duplicates/Copy B/Duplicate Series 001.cbz" + duplicate_copy.parent.mkdir(parents=True) + shutil.copyfile(duplicate_source, duplicate_copy) + duplicate_copy.chmod(0o644) + cases[-1]["paths"] = [_relative(root, duplicate_source), _relative(root, duplicate_copy)] + + add_cbz( + "duplicate_different", + "roots/source-b/Duplicates/Variant/Duplicate Series 001.cbz", + series="Duplicate Series", + number="1", + title="Different Payload", + expected_outcome="review", + tags=["duplicate-identity", "different-bytes"], + ) + hardlink = root / "roots/source-b/Duplicates/Hardlink/Duplicate Series 001.cbz" + hardlink.parent.mkdir(parents=True) + os.link(duplicate_source, hardlink) + cases.append( + _case( + "hardlink_duplicate", + paths=[_relative(root, duplicate_source), _relative(root, hardlink)], + expected_outcome="review", + tags=["hardlink", "duplicate-content"], + issue_number="1", + archive_format="cbz", + ) + ) + + disguised = root / "roots/source-b/Archive Oddities/ZIP Payload Issue 001.cbr" + create_deterministic_cbz( + disguised, + seed=seed, + case_id="mislabeled_zip_cbr", + series="Archive Oddities", + number="1", + title="ZIP Wearing a CBR Extension", + year=2026, + publisher="Fixture House", + comicvine_series_id=_synthetic_series_id("Archive Oddities"), + comicvine_issue_id=799_101, + ) + cases.append( + _case( + "mislabeled_zip_cbr", + paths=[_relative(root, disguised)], + expected_outcome="success", + tags=["mislabeled-archive", "zip-as-cbr"], + issue_number="1", + archive_format="zip-mislabeled-cbr", + ) + ) + corrupt = write_bytes( + root / "roots/source-b/Archive Oddities/Corrupt Issue 002.cbz", + b"IU9 corrupt archive\n", + ) + cases.append( + _case( + "corrupt_cbz", + paths=[_relative(root, corrupt)], + expected_outcome="blocked", + tags=["corrupt-archive", "fail-closed"], + issue_number="2", + archive_format="unreadable", + ) + ) + empty = create_deterministic_zip( + root / "roots/source-b/Archive Oddities/Empty Issue 003.cbz", + {}, + ) + cases.append( + _case( + "empty_cbz", + paths=[_relative(root, empty)], + expected_outcome="blocked", + tags=["empty-archive", "no-pages"], + issue_number="3", + archive_format="cbz", + ) + ) + + arc_one = root / "roots/source-b/Story Arcs/Fixture Crisis/01 - Alpha 001.cbz" + arc_two = root / "roots/source-b/Story Arcs/Fixture Crisis/02 - Beta 007.cbz" + create_deterministic_cbz( + arc_one, + seed=seed, + case_id="story_arc_alpha", + series="Alpha", + number="1", + title="The Beginning", + year=2019, + publisher="Fixture House", + comicvine_series_id=_synthetic_series_id("Alpha"), + comicvine_issue_id=799_201, + ) + create_deterministic_cbz( + arc_two, + seed=seed, + case_id="story_arc_beta", + series="Beta", + number="7", + title="The Crossover", + year=2020, + publisher="Fixture House", + comicvine_series_id=_synthetic_series_id("Beta"), + comicvine_issue_id=799_207, + ) + cases.append( + _case( + "story_arc_reading_order", + paths=[_relative(root, arc_one), _relative(root, arc_two)], + expected_outcome="review", + tags=["story-arc", "mixed-series", "reading-order-prefix"], + note="Folder shape is arc evidence, not proof; user review remains authoritative.", + ) + ) + + hidden = write_bytes(root / "roots/source-b/Mixed Siblings/.DS_Store", b"synthetic\n") + notes = write_text( + root / "roots/source-b/Mixed Siblings/README.txt", + "Non-comic sibling that the scanner must ignore.\n", + ) + cases.append( + _case( + "hidden_noncomic", + paths=[_relative(root, hidden), _relative(root, notes)], + expected_outcome="success", + tags=["ignored-file", "hidden-file", "non-comic"], + ) + ) + + links = root / "roots/source-b/Links" + links.mkdir(parents=True) + safe_link = links / "Issue 01 linked.cbz" + safe_link.symlink_to(os.path.relpath(issue_only, links)) + broken_link = links / "Missing Issue 999.cbz" + broken_link.symlink_to("../does-not-exist/Missing Issue 999.cbz") + cases.extend( + ( + _case( + "safe_symlink", + paths=[_relative(root, safe_link)], + expected_outcome="review", + tags=["symlink", "link-inside-source-roots"], + ), + _case( + "broken_symlink", + paths=[_relative(root, broken_link)], + expected_outcome="blocked", + tags=["symlink", "broken-link", "fail-closed"], + ), + ) + ) + + long_title = "Long Series " + ("Very " * 32) + "End" + add_cbz( + "long_near_limit_name", + f"roots/source-b/Long Paths/{long_title}/Issue 001.cbz", + series=long_title, + number="1", + expected_outcome="review", + tags=["long-path", "long-series-name", "naming-limit"], + ) + + trusted_archive = add_cbz( + "trusted_comicvine_identity", + ("roots/source-a/Trusted Identity Series (2024)/Trusted Identity Series 001 (2024).cbz"), + series="Trusted Identity Series", + number="1", + title="Trusted Identity", + year=2024, + publisher="Fixture House", + tags=["canonical-comicinfo", "canonical-series-sidecar", "provider-free-identity"], + comicvine_series_id=123456, + comicvine_issue_id=700100, + identity_source="series-sidecar-and-comicinfo", + ) + sidecar_dir = trusted_archive.parent + series_json = { + "comicid": 123456, + "name": "Trusted Identity Series", + "year": 2024, + "publisher": "Fixture House", + "comicvine": { + "id": 123456, + "url": "https://comicvine.gamespot.com/volume/4050-123456/", + }, + } + sidecars = [ + write_text( + sidecar_dir / "series.json", + json.dumps(series_json, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + ), + write_text( + sidecar_dir / "cvinfo", + ("comicid: 123456\nurl: https://comicvine.gamespot.com/volume/4050-123456/\n"), + ), + write_bytes(sidecar_dir / "cover.jpg", deterministic_jpeg()), + write_bytes(sidecar_dir / "folder.jpg", deterministic_jpeg()), + ] + cases.append( + _case( + "series_sidecar_metadata", + paths=[_relative(root, path) for path in sidecars], + expected_outcome="success", + tags=["series-json", "comicvine-link", "cover-sidecars"], + ) + ) + + cbr_manifest: dict[str, object] = { + "status": "not_provided", + "required_filenames": ["iu9-rar3.cbr", "iu9-rar5.cbr"], + } + if cbr_seed_dir is not None: + cbr_destination = source_b / "Genuine CBR Seeds (2026)" + cbr_evidence = consume_cbr_seed_set( + cbr_seed_dir, + cbr_destination, + expected_sha256=cbr_expected_sha256, + ) + cbr_manifest = { + "status": "provided", + "seeds": [item.to_manifest(root) for item in cbr_evidence], + } + for issue_number, item in enumerate(cbr_evidence, start=1): + cases.append( + _case( + f"genuine_{item.archive_format}_cbr", + paths=[_relative(root, item.destination)], + expected_outcome="review", + tags=[ + "genuine-cbr", + item.archive_format, + "external-cc0-seed", + "metadata-poor-no-comicinfo", + ], + issue_number=str(issue_number), + archive_format=item.archive_format, + note="Filename is parseable; provider identity requires review or matching.", + ) + ) + + manifest: dict[str, object] = { + "schema_version": _SCHEMA_VERSION, + "fixture_kind": "iu9-folder-import", + "seed": seed, + "roots": [ + { + "id": "source-a", + "source_relative": "roots/source-a", + "recommended_mode": "managed-copy-or-in-place", + }, + { + "id": "source-b", + "source_relative": "roots/source-b", + "recommended_mode": "managed-copy-or-in-place", + }, + ], + "cbr_seed_set": cbr_manifest, + "cases": cases, + "tree": snapshot_tree(root), + } + return write_manifest(root, manifest) diff --git a/scripts/iu9_acceptance_fixtures/mylar.py b/scripts/iu9_acceptance_fixtures/mylar.py new file mode 100644 index 00000000..c53e20a6 --- /dev/null +++ b/scripts/iu9_acceptance_fixtures/mylar.py @@ -0,0 +1,1057 @@ +"""Deterministic Mylar database and multi-root acceptance fixture generator.""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +from typing import TYPE_CHECKING + +from .shared import ( + CbrSeedEvidence, + consume_cbr_seed_set, + create_deterministic_cbz, + deterministic_jpeg, + prepare_fixture_root, + snapshot_tree, + write_bytes, + write_manifest, + write_text, +) + +if TYPE_CHECKING: + from collections.abc import Sequence + from pathlib import Path + +_SCHEMA_VERSION = 1 +_STORY_ARC_COLUMNS = ( + "StoryArcID", + "ComicName", + "IssueNumber", + "SeriesYear", + "IssueYEAR", + "StoryArc", + "TotalIssues", + "Status", + "inCacheDir", + "Location", + "IssueArcID", + "ReadingOrder", + "IssueID", + "ComicID", + "ReleaseDate", + "IssueDate", + "Publisher", + "IssuePublisher", + "IssueName", + "CV_ArcID", + "Int_IssueNumber", + "DynamicComicName", + "Volume", + "Manual", + "DateAdded", + "DigitalDate", + "Type", + "Aliases", + "ArcImage", +) + + +def _relative(root: Path, path: Path) -> str: + return path.relative_to(root).as_posix() + + +def _create_base_tables(connection: sqlite3.Connection) -> None: + connection.execute( + """ + CREATE TABLE comics ( + ComicID TEXT, + ComicName TEXT, + ComicYear TEXT, + ComicPublisher TEXT, + ComicLocation TEXT, + Status TEXT, + Total INTEGER, + ComicImage TEXT + ) + """ + ) + connection.execute( + """ + CREATE TABLE issues ( + IssueID TEXT, + ComicName TEXT, + IssueName TEXT, + Issue_Number TEXT, + ComicID TEXT, + Location TEXT, + IssueDate TEXT, + Int_IssueNumber INTEGER + ) + """ + ) + connection.execute( + """ + CREATE TABLE annuals ( + IssueID TEXT, + Issue_Number TEXT, + IssueName TEXT, + IssueDate TEXT, + ComicID TEXT, + Location TEXT, + Int_IssueNumber INTEGER, + ComicName TEXT, + ReleaseComicID TEXT, + ReleaseComicName TEXT + ) + """ + ) + + +def _create_full_optional_tables(connection: sqlite3.Connection) -> None: + connection.execute( + """ + CREATE TABLE storyarcs ( + StoryArcID TEXT, ComicName TEXT, IssueNumber TEXT, + SeriesYear TEXT, IssueYEAR TEXT, StoryArc TEXT, + TotalIssues TEXT, Status TEXT, inCacheDir TEXT, + Location TEXT, IssueArcID TEXT, ReadingOrder INTEGER, + IssueID TEXT, ComicID TEXT, ReleaseDate TEXT, + IssueDate TEXT, Publisher TEXT, IssuePublisher TEXT, + IssueName TEXT, CV_ArcID TEXT, Int_IssueNumber INTEGER, + DynamicComicName TEXT, Volume TEXT, Manual TEXT, + DateAdded TEXT, DigitalDate TEXT, Type TEXT, + Aliases TEXT, ArcImage TEXT + ) + """ + ) + connection.execute( + """ + CREATE TABLE readlist ( + IssueID TEXT, ComicName TEXT, Issue_Number TEXT, + Status TEXT, DateAdded TEXT, Location TEXT, + inCacheDir TEXT, SeriesYear TEXT, ComicID TEXT, + StatusChange TEXT + ) + """ + ) + + +def _create_database( + path: Path, + *, + comics: Sequence[dict[str, object]], + issues: Sequence[dict[str, object]], + annuals: Sequence[dict[str, object]], + optional_mode: str, + story_arcs: Sequence[dict[str, object]] = (), + readlist: Sequence[dict[str, object]] = (), +) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + connection = sqlite3.connect(path) + try: + connection.execute("PRAGMA page_size = 4096") + connection.execute("PRAGMA journal_mode = DELETE") + _create_base_tables(connection) + connection.executemany( + "INSERT INTO comics VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + [ + ( + row.get("ComicID"), + row.get("ComicName"), + row.get("ComicYear"), + row.get("ComicPublisher"), + row.get("ComicLocation"), + row.get("Status", "Active"), + row.get("Total", 0), + row.get("ComicImage"), + ) + for row in comics + ], + ) + connection.executemany( + "INSERT INTO issues VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + [ + ( + row.get("IssueID"), + row.get("ComicName"), + row.get("IssueName"), + row.get("Issue_Number"), + row.get("ComicID"), + row.get("Location"), + row.get("IssueDate"), + row.get("Int_IssueNumber"), + ) + for row in issues + ], + ) + connection.executemany( + "INSERT INTO annuals VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [ + ( + row.get("IssueID"), + row.get("Issue_Number"), + row.get("IssueName"), + row.get("IssueDate"), + row.get("ComicID"), + row.get("Location"), + row.get("Int_IssueNumber"), + row.get("ComicName"), + row.get("ReleaseComicID"), + row.get("ReleaseComicName"), + ) + for row in annuals + ], + ) + if optional_mode == "full": + _create_full_optional_tables(connection) + placeholders = ", ".join("?" for _column in _STORY_ARC_COLUMNS) + connection.executemany( + f"INSERT INTO storyarcs VALUES ({placeholders})", + [tuple(row.get(column) for column in _STORY_ARC_COLUMNS) for row in story_arcs], + ) + connection.executemany( + "INSERT INTO readlist VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [ + ( + row.get("IssueID"), + row.get("ComicName"), + row.get("Issue_Number"), + row.get("Status"), + row.get("DateAdded"), + row.get("Location"), + row.get("inCacheDir"), + row.get("SeriesYear"), + row.get("ComicID"), + row.get("StatusChange"), + ) + for row in readlist + ], + ) + elif optional_mode == "legacy": + connection.execute( + """ + CREATE TABLE storyarcs ( + StoryArcID TEXT, + StoryArc TEXT, + ReadingOrder TEXT, + IssueNumber TEXT + ) + """ + ) + connection.execute( + "INSERT INTO storyarcs VALUES (?, ?, ?, ?)", + ("legacy-arc", "Legacy Fixture Arc", "1", "0.5"), + ) + connection.execute( + """ + CREATE TABLE readlist ( + IssueID TEXT, ComicName TEXT, Issue_Number TEXT, + Status TEXT, DateAdded TEXT, Location TEXT, + inCacheDir TEXT, SeriesYear TEXT, ComicID TEXT, + StatusChange TEXT + ) + """ + ) + connection.execute( + "INSERT INTO readlist (IssueID, ComicName, Issue_Number) VALUES (?, ?, ?)", + ("legacy-read-1", "Number Lab", "0.5"), + ) + elif optional_mode != "absent": + raise ValueError(f"Unsupported Mylar optional-table mode: {optional_mode}") + connection.commit() + connection.execute("VACUUM") + finally: + connection.close() + path.chmod(0o644) + return path + + +def generate_mylar_fixture( + output_root: Path, + *, + seed: int = 1300, + cbr_seed_dir: Path | None = None, + cbr_expected_sha256: dict[str, str] | None = None, +) -> Path: + """Generate Mylar databases, mounted roots, sidecars, and case evidence.""" + root = prepare_fixture_root(output_root) + library_a = root / "roots" / "library-a" + library_b = root / "roots" / "library-b" + story_arc_root = root / "roots" / "story-arcs" + for directory in (library_a, library_b, story_arc_root): + directory.mkdir(parents=True) + + def add_archive( + actual_root: Path, + relative_path: str, + *, + case_id: str, + series: str, + number: str, + title: str, + year: int, + publisher: str, + comicvine_series_id: int, + comicvine_issue_id: int, + ) -> Path: + path = actual_root / relative_path + return create_deterministic_cbz( + path, + seed=seed, + case_id=case_id, + series=series, + number=number, + title=title, + year=year, + publisher=publisher, + comicvine_series_id=comicvine_series_id, + comicvine_issue_id=comicvine_issue_id, + ) + + identity_dir = "DC Comics/Identity Series (2020)" + identity_one = add_archive( + library_a, + f"{identity_dir}/Identity Series 001 (2020).cbz", + case_id="mylar_identity_1", + series="Identity Series", + number="1", + title="First Identity", + year=2020, + publisher="DC Comics", + comicvine_series_id=810001, + comicvine_issue_id=810001001, + ) + identity_two = add_archive( + library_a, + f"{identity_dir}/Identity Series 002 (2020).cbz", + case_id="mylar_identity_2", + series="Identity Series", + number="2", + title="Second Identity", + year=2020, + publisher="DC Comics", + comicvine_series_id=810001, + comicvine_issue_id=810001002, + ) + + mapped_dir = "Image/Mapped Series (2021)" + mapped_one = add_archive( + library_b, + f"{mapped_dir}/Mapped Series 001 (2021).cbz", + case_id="mylar_mapped_1", + series="Mapped Series", + number="1", + title="Mapped", + year=2021, + publisher="Image Comics", + comicvine_series_id=810002, + comicvine_issue_id=810002001, + ) + add_archive( + library_b, + "No Location Series (2017)/No Location Series 000 (2017).cbz", + case_id="mylar_no_location", + series="No Location Series", + number="0", + title="Found by Absolute Issue Path", + year=2017, + publisher="Fixture House", + comicvine_series_id=810003, + comicvine_issue_id=810003000, + ) + + split_dir_a = "Fixture House/Split Series (2022)" + split_one = add_archive( + library_a, + f"{split_dir_a}/Split Series 001 (2022).cbz", + case_id="mylar_split_1", + series="Split Series", + number="1", + title="Root A", + year=2022, + publisher="Fixture House", + comicvine_series_id=810004, + comicvine_issue_id=810004001, + ) + add_archive( + library_b, + "Split Overflow/Split Series 002 (2022).cbz", + case_id="mylar_split_2", + series="Split Series", + number="2", + title="Root B", + year=2022, + publisher="Fixture House", + comicvine_series_id=810004, + comicvine_issue_id=810004002, + ) + + number_dir = "Fixture House/Number Lab (2026)" + number_values = ("0", ".5", "0.5", "1/2", "1A", "10000", "1000000") + number_files: dict[str, Path] = {} + filename_number = { + "0": "0", + ".5": "dot5", + "0.5": "0.5", + "1/2": "one-half", + "1A": "1A", + "10000": "10000", + "1000000": "1000000", + } + for index, number in enumerate(number_values, start=1): + number_files[number] = add_archive( + library_a, + f"{number_dir}/Number Lab Issue {filename_number[number]}.cbz", + case_id=f"mylar_number_{index}", + series="Number Lab", + number=number, + title=f"Number {number}", + year=2026, + publisher="Fixture House", + comicvine_series_id=810005, + comicvine_issue_id=810005000 + index, + ) + + annual_dir = "Marvel/Annual Owner (2023)" + annual_regular = add_archive( + library_a, + f"{annual_dir}/Annual Owner 001 (2023).cbz", + case_id="mylar_annual_regular", + series="Annual Owner", + number="1", + title="Regular", + year=2023, + publisher="Marvel", + comicvine_series_id=810009, + comicvine_issue_id=810009001, + ) + annual_file = add_archive( + library_a, + f"{annual_dir}/Annual Owner Annual 001 (2024).cbz", + case_id="mylar_annual", + series="Annual Owner Annual", + number="1", + title="Annual Event", + year=2024, + publisher="Marvel", + comicvine_series_id=910009, + comicvine_issue_id=910009001, + ) + + cbr_manifest: dict[str, object] = { + "status": "not_provided", + "required_filenames": ["iu9-rar3.cbr", "iu9-rar5.cbr"], + } + cbr_evidence: tuple[CbrSeedEvidence, ...] = () + if cbr_seed_dir is not None: + cbr_evidence = consume_cbr_seed_set( + cbr_seed_dir, + library_b / "Fixture House" / "Mapped CBR Series (2025)", + expected_sha256=cbr_expected_sha256, + destination_filenames={ + "rar3": "Mapped CBR Series 001 (2025).cbr", + "rar5": "Mapped CBR Series 002 (2025).cbr", + }, + ) + cbr_manifest = { + "status": "provided", + "seeds": [item.to_manifest(root) for item in cbr_evidence], + } + + series_sidecar: dict[str, object] = { + "comicid": 810001, + "name": "Identity Series", + "year": 2020, + "comicvine": { + "id": 810001, + "url": "https://comicvine.gamespot.com/volume/4050-810001/", + }, + } + write_text( + library_a / identity_dir / "series.json", + json.dumps(series_sidecar, indent=2, sort_keys=True) + "\n", + ) + write_text( + library_a / identity_dir / "cvinfo", + "comicid: 810001\nurl: https://comicvine.gamespot.com/volume/4050-810001/\n", + ) + write_bytes(library_a / identity_dir / "cover.jpg", deterministic_jpeg()) + + comics: list[dict[str, object]] = [ + { + "ComicID": "CV-810001", + "ComicName": "Identity Series", + "ComicYear": "2020", + "ComicPublisher": "DC Comics", + "ComicLocation": "/iu9/mylar-a/DC Comics/Identity Series (2020)", + "Total": 2, + }, + { + "ComicID": "810002", + "ComicName": "Mapped Series", + "ComicYear": "2021", + "ComicPublisher": "Image Comics", + "ComicLocation": "/legacy/comics-b/Image/Mapped Series (2021)", + "Total": 1, + }, + { + "ComicID": "CV-810003", + "ComicName": "No Location Series", + "ComicYear": "2017", + "ComicPublisher": "Fixture House", + "ComicLocation": None, + "Total": 1, + }, + { + "ComicID": "CV-810004", + "ComicName": "Split Series", + "ComicYear": "2022", + "ComicPublisher": "Fixture House", + "ComicLocation": "/iu9/mylar-a/Fixture House/Split Series (2022)", + "Total": 2, + }, + { + "ComicID": "CV-810005", + "ComicName": "Number Lab", + "ComicYear": "2026", + "ComicPublisher": "Fixture House", + "ComicLocation": "/iu9/mylar-a/Fixture House/Number Lab (2026)", + "Total": len(number_values), + }, + { + "ComicID": "CV-810006", + "ComicName": "Missing Root", + "ComicYear": "2018", + "ComicPublisher": "Fixture House", + "ComicLocation": "/offline/library/Missing Root (2018)", + "Total": 1, + }, + { + "ComicID": "CV-810001", + "ComicName": "Duplicate Identity", + "ComicYear": "2020", + "ComicPublisher": "Fixture House", + "ComicLocation": "/iu9/mylar-b/Duplicate Identity (2020)", + "Total": 0, + }, + { + "ComicID": "CV-not-a-number", + "ComicName": "Malformed Identity", + "ComicYear": "unknown", + "ComicPublisher": "Fixture House", + "ComicLocation": None, + "Total": 0, + }, + { + "ComicID": "CV-810009", + "ComicName": "Annual Owner", + "ComicYear": "2023", + "ComicPublisher": "Marvel", + "ComicLocation": "/iu9/mylar-a/Marvel/Annual Owner (2023)", + "Total": 1, + }, + ] + if cbr_evidence: + comics.append( + { + "ComicID": "CV-810010", + "ComicName": "Mapped CBR Series", + "ComicYear": "2025", + "ComicPublisher": "Fixture House", + "ComicLocation": "/legacy/comics-b/Fixture House/Mapped CBR Series (2025)", + "Total": len(cbr_evidence), + } + ) + + def issue_row( + issue_id: int | str, + comic_id: str, + comic_name: str, + number: str, + location: str, + *, + title: str, + date: str, + int_number: int, + ) -> dict[str, object]: + return { + "IssueID": str(issue_id), + "ComicID": comic_id, + "ComicName": comic_name, + "IssueName": title, + "Issue_Number": number, + "Location": location, + "IssueDate": date, + "Int_IssueNumber": int_number, + } + + issues = [ + issue_row( + 8_100_010_01, + "810001", + "Identity Series", + "1", + identity_one.name, + title="First Identity", + date="2020-01-01", + int_number=1000, + ), + issue_row( + 8_100_010_02, + "810001", + "Identity Series", + "2", + identity_two.name, + title="Second Identity", + date="2020-02-01", + int_number=2000, + ), + issue_row( + 8_100_020_01, + "810002", + "Mapped Series", + "1", + mapped_one.name, + title="Mapped", + date="2021-01-01", + int_number=1000, + ), + issue_row( + 8_100_030_00, + "810003", + "No Location Series", + "0", + "/iu9/mylar-b/No Location Series (2017)/No Location Series 000 (2017).cbz", + title="Found by Absolute Issue Path", + date="2017-01-01", + int_number=0, + ), + issue_row( + 8_100_040_01, + "810004", + "Split Series", + "1", + split_one.name, + title="Root A", + date="2022-01-01", + int_number=1000, + ), + issue_row( + 8_100_040_02, + "810004", + "Split Series", + "2", + "/iu9/mylar-b/Split Overflow/Split Series 002 (2022).cbz", + title="Root B", + date="2022-02-01", + int_number=2000, + ), + ] + issue_int_values = { + "0": 0, + ".5": 500, + "0.5": 500, + "1/2": 500, + "1A": 1001, + "10000": 10_000_000, + "1000000": 1_000_000_000, + } + for index, number in enumerate(number_values, start=1): + issues.append( + issue_row( + 8_100_050_00 + index, + "810005", + "Number Lab", + number, + number_files[number].name, + title=f"Number {number}", + date="2026-01-01", + int_number=issue_int_values[number], + ) + ) + issues.extend( + ( + issue_row( + 8_100_060_01, + "810006", + "Missing Root", + "1", + "Missing Root 001 (2018).cbz", + title="Offline", + date="2018-01-01", + int_number=1000, + ), + issue_row( + "not-an-issue-id", + "not-a-comic-id", + "Malformed Identity", + "???", + "../outside-root.cbz", + title="Malformed", + date="not-a-date", + int_number=0, + ), + issue_row( + 8_100_090_01, + "810009", + "Annual Owner", + "1", + annual_regular.name, + title="Regular", + date="2023-01-01", + int_number=1000, + ), + ) + ) + for index, item in enumerate(cbr_evidence, start=1): + issues.append( + issue_row( + 810010000 + index, + "810010", + "Mapped CBR Series", + str(index), + item.destination.name, + title=f"Genuine {item.archive_format.upper()} Source", + date=f"2025-{index:02d}-01", + int_number=index * 1000, + ) + ) + annuals = [ + { + "IssueID": "910009001", + "Issue_Number": "1", + "IssueName": "Annual Event", + "IssueDate": "2024-06-01", + "ComicID": "810009", + "Location": annual_file.name, + "Int_IssueNumber": 1000, + "ComicName": "Annual Owner", + "ReleaseComicID": "CV-910009", + "ReleaseComicName": "Annual Owner Annual", + } + ] + + arc_dir = story_arc_root / "Fixture Crisis" + arc_dir.mkdir(parents=True) + arc_identity = arc_dir / "01 - Identity Series 001.cbz" + arc_mapped = arc_dir / "02 - Mapped Series 001.cbz" + shutil.copyfile(identity_one, arc_identity) + shutil.copyfile(mapped_one, arc_mapped) + arc_identity.chmod(0o644) + arc_mapped.chmod(0o644) + story_arcs = [ + { + "StoryArcID": "fixture-arc-1", + "ComicName": "Identity Series", + "IssueNumber": "1", + "SeriesYear": "2020", + "IssueYEAR": "2020", + "StoryArc": "Fixture Crisis", + "TotalIssues": "3", + "Status": "Downloaded", + "Location": "/iu9/story-arcs/Fixture Crisis/01 - Identity Series 001.cbz", + "IssueArcID": "fixture-arc-entry-1", + "ReadingOrder": 1, + "IssueID": "810001001", + "ComicID": "810001", + "ReleaseDate": "2020-01-01", + "IssueDate": "2020-01-01", + "Publisher": "DC Comics", + "IssuePublisher": "DC Comics", + "IssueName": "First Identity", + "CV_ArcID": "4045-99001", + "Int_IssueNumber": 1000, + "Manual": "added", + "DateAdded": "2026-01-01", + "DigitalDate": "2020-01-01", + "Type": "Comic", + }, + { + "StoryArcID": "fixture-arc-1", + "ComicName": "Mapped Series", + "IssueNumber": "1", + "SeriesYear": "2021", + "IssueYEAR": "2021", + "StoryArc": "Fixture Crisis", + "TotalIssues": "3", + "Status": "Downloaded", + "Location": "/iu9/story-arcs/Fixture Crisis/02 - Mapped Series 001.cbz", + "IssueArcID": "fixture-arc-entry-2", + "ReadingOrder": 2, + "IssueID": "810002001", + "ComicID": "810002", + "IssueName": "Mapped", + "CV_ArcID": "4045-99001", + }, + { + "StoryArcID": "fixture-arc-1", + "ComicName": "Unresolved Series", + "IssueNumber": "1/2", + "SeriesYear": "2019", + "StoryArc": "Fixture Crisis", + "TotalIssues": "3", + "Status": "Wanted", + "Location": "/iu9/story-arcs/Fixture Crisis/03 - Missing 001.cbz", + "IssueArcID": "fixture-arc-entry-missing", + "ReadingOrder": 3, + "IssueName": "Missing Chapter", + "CV_ArcID": "4045-99001", + "Manual": "added", + }, + { + "StoryArcID": "fixture-arc-2", + "ComicName": "Number Lab", + "IssueNumber": "1000000", + "SeriesYear": "2026", + "StoryArc": "Duplicate Order Arc", + "Status": "Downloaded", + "IssueArcID": "fixture-arc-2-entry-1", + "ReadingOrder": 7, + "IssueID": "810005007", + "ComicID": "810005", + "CV_ArcID": "4045-99002", + }, + { + "StoryArcID": "fixture-arc-2", + "ComicName": "Number Lab", + "IssueNumber": "0.5", + "SeriesYear": "2026", + "StoryArc": "Duplicate Order Arc", + "Status": "Downloaded", + "IssueArcID": "fixture-arc-2-entry-2", + "ReadingOrder": 7, + "IssueID": "810005003", + "ComicID": "810005", + "CV_ArcID": "4045-99002", + }, + ] + readlist: list[dict[str, object]] = [ + { + "IssueID": "810001001", + "ComicName": "Identity Series", + "Issue_Number": "1", + "Status": "Downloaded", + "DateAdded": "2026-01-01", + "Location": ( + "/iu9/mylar-a/DC Comics/Identity Series (2020)/Identity Series 001 (2020).cbz" + ), + "SeriesYear": "2020", + "ComicID": "810001", + }, + { + "IssueID": "missing-readlist-entry", + "ComicName": "Unresolved Series", + "Issue_Number": "1000000", + "Status": "Wanted", + "DateAdded": "2026-01-01", + "Location": "/offline/readlist/Unresolved Series 1000000.cbz", + "SeriesYear": "2026", + }, + ] + + database = _create_database( + root / "mylar.db", + comics=comics, + issues=issues, + annuals=annuals, + optional_mode="full", + story_arcs=story_arcs, + readlist=readlist, + ) + variant_comics = comics[:1] + variant_issues = issues[:2] + absent = _create_database( + root / "variants" / "optional-tables-absent.db", + comics=variant_comics, + issues=variant_issues, + annuals=[], + optional_mode="absent", + ) + empty = _create_database( + root / "variants" / "optional-tables-empty.db", + comics=variant_comics, + issues=variant_issues, + annuals=[], + optional_mode="full", + ) + legacy = _create_database( + root / "variants" / "legacy-storyarcs.db", + comics=variant_comics, + issues=variant_issues, + annuals=[], + optional_mode="legacy", + ) + invalid_db = write_text( + root / "variants" / "not-a-sqlite-database.db", + "This is a deliberate invalid-database acceptance case.\n", + ) + + config = """[General] +READ2FILENAME = 1 + +[StoryArc] +STORYARCDIR = 1 +STORYARC_LOCATION = /iu9/story-arcs +COPY2ARCDIR = 1 +ARC_FOLDERFORMAT = $arc ($spanyears) +ARC_FILEOPS = copy +ARC_FILEOPS_SOFTLINK_RELATIVE = 0 +UPCOMING_STORYARCS = 1 +SEARCH_STORYARCS = 1 +""" + write_text(root / "config.ini", config) + + cases: list[dict[str, object]] = [ + { + "id": "identity_root", + "database": _relative(root, database), + "series": "Identity Series", + "expected_outcome": "success", + "tags": ["identity-first", "exact-container-path"], + }, + { + "id": "explicit_path_mapping", + "database": _relative(root, database), + "series": "Mapped Series", + "expected_outcome": "success", + "tags": ["path-map", "legacy-prefix", "relative-issue-path"], + }, + { + "id": "missing_comic_location_absolute_issue", + "database": _relative(root, database), + "series": "No Location Series", + "expected_outcome": "review", + "tags": ["missing-comiclocation", "absolute-issue-path"], + }, + { + "id": "split_series", + "database": _relative(root, database), + "series": "Split Series", + "expected_outcome": "review", + "tags": ["multiple-roots", "relative-and-absolute-issue-paths"], + }, + { + "id": "missing_root", + "database": _relative(root, database), + "series": "Missing Root", + "expected_outcome": "blocked", + "tags": ["offline-root", "missing-file"], + }, + { + "id": "duplicate_comic_id", + "database": _relative(root, database), + "series": "Duplicate Identity", + "expected_outcome": "review", + "tags": ["duplicate-comic-id"], + }, + { + "id": "malformed_comic_id", + "database": _relative(root, database), + "series": "Malformed Identity", + "expected_outcome": "review", + "tags": ["malformed-id", "malformed-date", "root-escape-location"], + }, + { + "id": "annual_release_identity", + "database": _relative(root, database), + "series": "Annual Owner", + "expected_outcome": "success", + "tags": ["annual", "release-comic-id", "separate-release-series"], + }, + { + "id": "weird_issue_numbers", + "database": _relative(root, database), + "series": "Number Lab", + "expected_outcome": "review", + "issue_numbers": list(number_values), + "tags": ["decimal", "fraction", "suffix", "large-number"], + }, + { + "id": "story_arc_full", + "database": _relative(root, database), + "expected_outcome": "review", + "tags": ["storyarcs", "existing-placement", "missing-member", "duplicate-order"], + }, + { + "id": "readlist_full", + "database": _relative(root, database), + "expected_outcome": "review", + "tags": ["readlist", "existing-member", "missing-member"], + }, + { + "id": "optional_tables_absent", + "database": _relative(root, absent), + "expected_outcome": "success", + "tags": ["no-storyarcs-table", "no-readlist-table"], + }, + { + "id": "optional_tables_empty", + "database": _relative(root, empty), + "expected_outcome": "success", + "tags": ["empty-storyarcs", "empty-readlist"], + }, + { + "id": "legacy_storyarc_table", + "database": _relative(root, legacy), + "expected_outcome": "review", + "tags": ["legacy-storyarcs-columns", "legacy-readlist"], + }, + { + "id": "invalid_database", + "database": _relative(root, invalid_db), + "expected_outcome": "blocked", + "tags": ["not-sqlite", "fail-closed"], + }, + ] + if cbr_evidence: + cases.append( + { + "id": "mapped_genuine_cbr_conversion", + "database": _relative(root, database), + "series": "Mapped CBR Series", + "paths": [_relative(root, item.destination) for item in cbr_evidence], + "expected_outcome": "success", + "expected_materialization": "cbz", + "tags": [ + "explicit-path-mapping", + "managed-copy", + "genuine-cbr", + "cbr-to-cbz", + "identity-from-mylar-database", + ], + } + ) + manifest: dict[str, object] = { + "schema_version": _SCHEMA_VERSION, + "fixture_kind": "iu9-mylar-import", + "seed": seed, + "runtime_mounts": [ + { + "root_id": "mylar-a", + "source_relative": "roots/library-a", + "container_path": "/iu9/mylar-a", + }, + { + "root_id": "mylar-b", + "source_relative": "roots/library-b", + "container_path": "/iu9/mylar-b", + }, + { + "root_id": "story-arcs", + "source_relative": "roots/story-arcs", + "container_path": "/iu9/story-arcs", + }, + ], + "suggested_path_maps": [ + { + "source_prefix": "/legacy/comics-b", + "target_prefix": "/iu9/mylar-b", + } + ], + "cbr_seed_set": cbr_manifest, + "archive_identity_source": "mylar-database-with-canonical-comicinfo-where-available", + "cases": cases, + "tree": snapshot_tree(root), + } + return write_manifest(root, manifest) diff --git a/scripts/iu9_acceptance_fixtures/shared.py b/scripts/iu9_acceptance_fixtures/shared.py new file mode 100644 index 00000000..25b20454 --- /dev/null +++ b/scripts/iu9_acceptance_fixtures/shared.py @@ -0,0 +1,442 @@ +"""Shared deterministic archives, seed validation, and manifest evidence.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +import struct +import zipfile +import zlib +from dataclasses import asdict, dataclass +from pathlib import Path, PurePosixPath +from typing import cast +from xml.etree.ElementTree import Element, SubElement, tostring + +RAR3_SIGNATURE = b"Rar!\x1a\x07\x00" +RAR5_SIGNATURE = b"Rar!\x1a\x07\x01\x00" +CBR_SEED_DESCRIPTOR = "cbr-seeds.json" +CBR_SEED_FILENAMES = { + "rar3": "iu9-rar3.cbr", + "rar5": "iu9-rar5.cbr", +} +CBR_FIXTURE_FILENAMES = { + "rar3": "Genuine CBR Seeds 001 (2026).cbr", + "rar5": "Genuine CBR Seeds 002 (2026).cbr", +} +_FIXED_ZIP_TIME = (2020, 1, 1, 0, 0, 0) +_HEX_DIGITS = frozenset("0123456789abcdef") + + +class CbrSeedValidationError(ValueError): + """Raised when an externally supplied genuine-CBR seed is not trustworthy.""" + + +@dataclass(frozen=True, slots=True) +class CbrSeedEvidence: + """Validated provenance and destination evidence for one CBR seed.""" + + seed_id: str + archive_format: str + source_filename: str + destination: Path + sha256: str + source_url: str + license: str + expected_members: tuple[str, ...] + + def to_manifest(self, fixture_root: Path) -> dict[str, object]: + """Return path-safe JSON evidence relative to the fixture root.""" + values = asdict(self) + values["destination"] = self.destination.relative_to(fixture_root).as_posix() + values["expected_members"] = list(self.expected_members) + return values + + +def prepare_fixture_root(root: Path) -> Path: + """Create a fresh fixture root without deleting existing content.""" + root = root.expanduser().absolute() + if root.exists(): + if not root.is_dir(): + raise FileExistsError(f"Fixture output is not a directory: {root}") + if any(root.iterdir()): + raise FileExistsError(f"Fixture output must be empty: {root}") + else: + root.mkdir(parents=True) + root.chmod(0o755) + return root + + +def write_bytes(path: Path, payload: bytes, *, mode: int = 0o644) -> Path: + """Write deterministic fixture bytes with stable permissions.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + path.chmod(mode) + return path + + +def write_text(path: Path, text: str, *, mode: int = 0o644) -> Path: + """Write deterministic UTF-8 fixture text with stable permissions.""" + return write_bytes(path, text.encode("utf-8"), mode=mode) + + +def sha256_file(path: Path) -> str: + """Return the SHA-256 digest for a regular file.""" + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def deterministic_png(*, seed: int, identity: str) -> bytes: + """Build a valid deterministic 2x2 RGB PNG from a seed and identity.""" + color = hashlib.sha256(f"{seed}:{identity}".encode()).digest()[:3] + signature = b"\x89PNG\r\n\x1a\n" + + def chunk(kind: bytes, payload: bytes) -> bytes: + body = kind + payload + crc = struct.pack(">I", zlib.crc32(body) & 0xFFFFFFFF) + return struct.pack(">I", len(payload)) + body + crc + + image_row = b"\x00" + (color * 2) + pixels = image_row * 2 + return b"".join( + ( + signature, + chunk(b"IHDR", struct.pack(">IIBBBBB", 2, 2, 8, 2, 0, 0, 0)), + chunk(b"IDAT", zlib.compress(pixels, level=9)), + chunk(b"IEND", b""), + ) + ) + + +def deterministic_jpeg() -> bytes: + """Return a tiny valid 1x1 JPEG for cover sidecars.""" + return bytes.fromhex( + "ffd8ffe000104a46494600010100000100010000ffdb004300" + "01010101010101010101010101010101010101010101010101" + "01010101010101010101010101010101010101010101010101" + "010101010101ffc0000b080001000101011100ffc40014000100" + "000000000000000000000000000000ffc4001410010000000000" + "0000000000000000000000ffda0008010100003f00d2cfffd9" + ) + + +def comic_info_xml( + *, + series: str, + number: str, + title: str | None = None, + year: int | None = None, + publisher: str | None = None, + comicvine_series_id: int | None = None, + comicvine_issue_id: int | None = None, +) -> bytes: + """Build deterministic ComicInfo.xml without deferred Metron metadata.""" + root = Element("ComicInfo") + SubElement(root, "Series").text = series + SubElement(root, "Number").text = number + if title is not None: + SubElement(root, "Title").text = title + if year is not None: + SubElement(root, "Year").text = str(year) + if publisher is not None: + SubElement(root, "Publisher").text = publisher + identity_tags: list[str] = [] + if comicvine_series_id is not None: + identity_tags.append(f"[cv_vol_id:{comicvine_series_id}]") + if comicvine_issue_id is not None: + identity_tags.append(f"[cv_issue_id:{comicvine_issue_id}]") + if identity_tags: + SubElement(root, "Notes").text = " ".join(identity_tags) + xml = cast("bytes", tostring(root, encoding="utf-8", xml_declaration=True)) + return xml + b"\n" + + +def _zip_info(name: str) -> zipfile.ZipInfo: + info = zipfile.ZipInfo(name, date_time=_FIXED_ZIP_TIME) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + return info + + +def create_deterministic_zip(path: Path, members: dict[str, bytes]) -> Path: + """Create a deterministic ZIP-family archive from safe member names.""" + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: + for name, payload in members.items(): + member = PurePosixPath(name) + if member.is_absolute() or ".." in member.parts: + raise ValueError(f"Unsafe deterministic ZIP member: {name}") + archive.writestr(_zip_info(name), payload) + path.chmod(0o644) + return path + + +def create_deterministic_cbz( + path: Path, + *, + seed: int, + case_id: str, + series: str, + number: str, + title: str | None = None, + year: int | None = None, + publisher: str | None = None, + comicvine_series_id: int | None = None, + comicvine_issue_id: int | None = None, + page_count: int = 3, + include_comicinfo: bool = True, +) -> Path: + """Create a valid deterministic CBZ with PNG pages and ComicInfo.xml.""" + if page_count < 1: + raise ValueError("A valid CBZ fixture requires at least one page") + members: dict[str, bytes] = {} + if include_comicinfo: + members["ComicInfo.xml"] = comic_info_xml( + series=series, + number=number, + title=title, + year=year, + publisher=publisher, + comicvine_series_id=comicvine_series_id, + comicvine_issue_id=comicvine_issue_id, + ) + members.update( + { + f"pages/{page:03d}.png": deterministic_png( + seed=seed, + identity=f"{case_id}:page:{page}", + ) + for page in range(1, page_count + 1) + } + ) + return create_deterministic_zip(path, members) + + +def load_json(path: Path) -> dict[str, object]: + """Read a JSON object used by the fixture contract.""" + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"Expected a JSON object: {path}") + return cast("dict[str, object]", value) + + +def _descriptor_rows(descriptor: dict[str, object]) -> dict[str, dict[str, object]]: + if descriptor.get("schema_version") != 1: + raise CbrSeedValidationError("Unsupported CBR seed descriptor schema") + raw_rows = descriptor.get("seeds") + if not isinstance(raw_rows, list): + raise CbrSeedValidationError("CBR seed descriptor must contain a seed list") + rows: dict[str, dict[str, object]] = {} + for raw_row in raw_rows: + if not isinstance(raw_row, dict) or not isinstance(raw_row.get("id"), str): + raise CbrSeedValidationError("CBR seed descriptor contains an invalid row") + row = cast("dict[str, object]", raw_row) + seed_id = cast("str", row["id"]) + if seed_id in rows: + raise CbrSeedValidationError(f"Duplicate CBR seed id: {seed_id}") + rows[seed_id] = row + if set(rows) != set(CBR_SEED_FILENAMES): + raise CbrSeedValidationError("CBR seed descriptor must define rar3 and rar5") + return rows + + +def _safe_expected_members(value: object, *, seed_id: str) -> tuple[str, ...]: + if not isinstance(value, list) or not value: + raise CbrSeedValidationError(f"{seed_id} must declare expected archive members") + members: list[str] = [] + for raw_member in value: + if not isinstance(raw_member, str): + raise CbrSeedValidationError(f"{seed_id} has an invalid expected member") + member = PurePosixPath(raw_member) + if member.is_absolute() or ".." in member.parts: + raise CbrSeedValidationError(f"{seed_id} has an unsafe expected member") + members.append(raw_member) + return tuple(members) + + +def _expected_digest( + row: dict[str, object], + *, + seed_id: str, + expected_sha256: dict[str, str] | None, +) -> str | None: + caller_digest = None if expected_sha256 is None else expected_sha256.get(seed_id) + descriptor_digest = row.get("sha256") + if descriptor_digest is not None and not isinstance(descriptor_digest, str): + raise CbrSeedValidationError(f"{seed_id} SHA-256 must be text") + if caller_digest is not None and descriptor_digest not in {None, caller_digest}: + raise CbrSeedValidationError(f"{seed_id} SHA-256 pins disagree") + digest = caller_digest or descriptor_digest + if digest is not None and ( + len(digest) != 64 or any(character not in _HEX_DIGITS for character in digest) + ): + raise CbrSeedValidationError(f"{seed_id} SHA-256 pin is invalid") + return digest + + +def consume_cbr_seed_set( + seed_dir: Path, + destination: Path, + *, + expected_sha256: dict[str, str] | None = None, + destination_filenames: dict[str, str] | None = None, +) -> tuple[CbrSeedEvidence, ...]: + """Validate and copy fixed RAR3/RAR5 CBR seeds without downloading them.""" + descriptor_path = seed_dir / CBR_SEED_DESCRIPTOR + if not descriptor_path.is_file() or descriptor_path.is_symlink(): + raise CbrSeedValidationError(f"Missing regular {CBR_SEED_DESCRIPTOR}") + rows = _descriptor_rows(load_json(descriptor_path)) + validated: list[tuple[str, Path, str, str, str, tuple[str, ...]]] = [] + for seed_id in ("rar3", "rar5"): + row = rows[seed_id] + expected_filename = CBR_SEED_FILENAMES[seed_id] + if row.get("filename") != expected_filename: + raise CbrSeedValidationError(f"{seed_id} must use fixed filename {expected_filename}") + if row.get("archive_format") != seed_id: + raise CbrSeedValidationError(f"{seed_id} archive family does not match its id") + source = seed_dir / expected_filename + if source.is_symlink() or not source.is_file(): + raise CbrSeedValidationError(f"{seed_id} seed must be a regular file") + with source.open("rb") as seed_file: + payload_prefix = seed_file.read(len(RAR5_SIGNATURE)) + expected_signature = RAR3_SIGNATURE if seed_id == "rar3" else RAR5_SIGNATURE + if not payload_prefix.startswith(expected_signature): + raise CbrSeedValidationError(f"{seed_id.upper()} signature is invalid") + actual_digest = sha256_file(source) + pinned_digest = _expected_digest( + row, + seed_id=seed_id, + expected_sha256=expected_sha256, + ) + if pinned_digest is not None and actual_digest != pinned_digest: + raise CbrSeedValidationError(f"{seed_id} SHA-256 does not match its pin") + source_url = row.get("source_url") + license_name = row.get("license") + if not isinstance(source_url, str) or not source_url.startswith("https://"): + raise CbrSeedValidationError(f"{seed_id} must declare an HTTPS source URL") + if not isinstance(license_name, str) or not license_name.strip(): + raise CbrSeedValidationError(f"{seed_id} must declare a license") + expected_members = _safe_expected_members(row.get("expected_members"), seed_id=seed_id) + target_name = ( + CBR_FIXTURE_FILENAMES[seed_id] + if destination_filenames is None + else destination_filenames.get(seed_id) + ) + if not isinstance(target_name, str): + raise CbrSeedValidationError(f"Missing destination filename for {seed_id}") + target_path = PurePosixPath(target_name) + if ( + target_path.is_absolute() + or len(target_path.parts) != 1 + or target_path.suffix.casefold() != ".cbr" + ): + raise CbrSeedValidationError(f"Unsafe destination filename for {seed_id}") + validated.append( + (seed_id, source, actual_digest, source_url, license_name, expected_members) + ) + + destination.mkdir(parents=True, exist_ok=True) + evidence: list[CbrSeedEvidence] = [] + for seed_id, source, digest, source_url, license_name, expected_members in validated: + target_name = ( + CBR_FIXTURE_FILENAMES[seed_id] + if destination_filenames is None + else destination_filenames[seed_id] + ) + target = destination / target_name + shutil.copyfile(source, target) + target.chmod(0o644) + evidence.append( + CbrSeedEvidence( + seed_id=seed_id, + archive_format=seed_id, + source_filename=source.name, + destination=target, + sha256=digest, + source_url=source_url, + license=license_name, + expected_members=expected_members, + ) + ) + return tuple(evidence) + + +def _archive_evidence(path: Path) -> tuple[str | None, list[str]]: + try: + with zipfile.ZipFile(path) as archive: + return "zip", archive.namelist() + except (OSError, zipfile.BadZipFile): + with path.open("rb") as archive_file: + prefix = archive_file.read(len(RAR5_SIGNATURE)) + if prefix.startswith(RAR5_SIGNATURE): + return "rar5", [] + if prefix.startswith(RAR3_SIGNATURE): + return "rar3", [] + if path.suffix.casefold() in {".cbz", ".cbr", ".zip", ".rar"}: + return "unreadable", [] + return None, [] + + +def snapshot_tree(root: Path) -> list[dict[str, object]]: + """Capture deterministic, relative evidence for generated files and links.""" + paths = sorted( + (path for path in root.rglob("*") if path.is_file() or path.is_symlink()), + key=lambda path: path.relative_to(root).as_posix(), + ) + inode_paths: dict[tuple[int, int], list[str]] = {} + for path in paths: + metadata = path.lstat() + if stat.S_ISREG(metadata.st_mode): + inode_paths.setdefault((metadata.st_dev, metadata.st_ino), []).append( + path.relative_to(root).as_posix() + ) + hardlink_labels = { + path_name: f"hardlink-{index:03d}" + for index, names in enumerate( + sorted((names for names in inode_paths.values() if len(names) > 1), key=min), + start=1, + ) + for path_name in names + } + + rows: list[dict[str, object]] = [] + for path in paths: + relative = path.relative_to(root).as_posix() + metadata = path.lstat() + if stat.S_ISLNK(metadata.st_mode): + rows.append( + { + "path": relative, + "type": "symlink", + "link_target": os.readlink(path), + } + ) + continue + archive_format, members = _archive_evidence(path) + row: dict[str, object] = { + "path": relative, + "type": "file", + "size": metadata.st_size, + "mode": stat.S_IMODE(metadata.st_mode), + "sha256": sha256_file(path), + } + if relative in hardlink_labels: + row["hardlink_group"] = hardlink_labels[relative] + if archive_format is not None: + row["archive_format"] = archive_format + row["archive_members"] = members + rows.append(row) + return rows + + +def write_manifest(root: Path, manifest: dict[str, object]) -> Path: + """Write canonical JSON without timestamps or absolute host paths.""" + path = root / "manifest.json" + payload = json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + return write_text(path, payload) diff --git a/scripts/mylar3_import_fixture.py b/scripts/mylar3_import_fixture.py index 2ee35571..b62c0dba 100644 --- a/scripts/mylar3_import_fixture.py +++ b/scripts/mylar3_import_fixture.py @@ -31,6 +31,7 @@ def create_minimal_cbz(path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: archive.writestr("001.jpg", b"pullbox-mylar-fixture") + archive.writestr("002.jpg", b"pullbox-mylar-fixture") def create_mylar3_db( diff --git a/scripts/pre_commit_pytest.sh b/scripts/pre_commit_pytest.sh new file mode 100755 index 00000000..d3d1b21e --- /dev/null +++ b/scripts/pre_commit_pytest.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +if [[ -x ".venv/bin/pytest" ]]; then + exec ".venv/bin/pytest" tests/unit/ -x -q -m "not slow" --no-header +fi + +exec pytest tests/unit/ -x -q -m "not slow" --no-header diff --git a/scripts/run_dependency_audit.py b/scripts/run_dependency_audit.py new file mode 100644 index 00000000..5886514a --- /dev/null +++ b/scripts/run_dependency_audit.py @@ -0,0 +1,198 @@ +"""Run the blocking dependency audit with narrowly reviewed exceptions.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import tomllib +from datetime import UTC, date, datetime +from pathlib import Path +from typing import Any + +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +PROJECT = Path(__file__).resolve().parents[1] / "pyproject.toml" +NLTK_ADVISORY_IDS = {"PYSEC-2026-3740", "GHSA-8mgp-746c-j5xp", "CVE-2026-81726"} +NLTK_EXCEPTION_EXPIRES = date(2026, 10, 3) + + +def _dependencies(report: object) -> dict[str, dict[str, Any]]: + if not isinstance(report, dict) or not isinstance(report.get("dependencies"), list): + raise ValueError("Missing dependency audit inventory") + dependencies: dict[str, dict[str, Any]] = {} + for dep in report["dependencies"]: + if ( + not isinstance(dep, dict) + or "skip_reason" in dep + or not isinstance(dep.get("name"), str) + or not dep["name"] + or not isinstance(dep.get("version"), str) + or not dep["version"] + or not isinstance(dep.get("vulns"), list) + ): + raise ValueError("Incomplete or malformed dependency audit record") + name = canonicalize_name(dep["name"]) + if name in dependencies: + raise ValueError(f"Duplicate dependency audit record: {name}") + for vuln in dep["vulns"]: + if ( + not isinstance(vuln, dict) + or not isinstance(vuln.get("id"), str) + or not vuln["id"] + or not isinstance(vuln.get("aliases"), list) + or not all(isinstance(alias, str) and alias for alias in vuln["aliases"]) + or not isinstance(vuln.get("fix_versions"), list) + or not all(isinstance(version, str) for version in vuln["fix_versions"]) + ): + raise ValueError(f"Malformed vulnerability record for {name}") + dependencies[name] = dep + if not dependencies: + raise ValueError("Empty dependency audit inventory") + return dependencies + + +def evaluate_report( + report: object, + scanner_status: int, + runtime_dependencies: set[str], + today: date, +) -> int: + """Return a failing status unless the complete audit satisfies policy.""" + try: + dependencies = _dependencies(report) + findings = sum(len(dep["vulns"]) for dep in dependencies.values()) + if scanner_status != (1 if findings else 0): + raise ValueError( + f"Scanner failed or returned inconsistent evidence (exit {scanner_status})" + ) + except ValueError as exc: + print(f"BLOCKED: {exc}") + return 1 + + development_only = not {"safety", "nltk"}.intersection(runtime_dependencies) + reviewed_tool = dependencies.get("safety", {}).get("version") == "3.8.1" + blocked = 0 + accepted = 0 + for name, dep in dependencies.items(): + for vuln in dep["vulns"]: + if ( + name == "nltk" + and dep["version"] == "3.10.3" + and vuln["id"] in NLTK_ADVISORY_IDS + and set(vuln["aliases"]) <= NLTK_ADVISORY_IDS + and not vuln["fix_versions"] + and reviewed_tool + and development_only + and today < NLTK_EXCEPTION_EXPIRES + ): + accepted += 1 + print( + f"ACCEPTED (temporary risk): nltk==3.10.3 {vuln['id']}; " + f"Safety 3.8.1 development-only edit_distance use; " + f"expires {NLTK_EXCEPTION_EXPIRES} 00:00 UTC" + ) + else: + blocked += 1 + print(f"BLOCKED: {name}=={dep['version']} {vuln['id']}") + if name == "nltk" and vuln["id"] in NLTK_ADVISORY_IDS: + print( + "NLTK exception expired or scope changed; upgrade or obtain a new review." + ) + print(f"Dependency audit: {blocked} blocking finding(s), {accepted} temporary exception(s).") + return int(blocked > 0) + + +def runtime_dependencies(project: Path) -> set[str]: + """Read every dependency group that can ship in the runtime.""" + with project.open("rb") as stream: + config = tomllib.load(stream)["project"] + requirements = list(config["dependencies"]) + for group, extra in config.get("optional-dependencies", {}).items(): + if group not in {"dev", "e2e"}: + requirements.extend(extra) + return {canonicalize_name(Requirement(value).name) for value in requirements} + + +def run_audit(requirements: Path, report_path: Path, *, today: date | None = None) -> int: + """Collect and evaluate the real scanner report, preserving evidence.""" + try: + report_path.unlink(missing_ok=True) + result = subprocess.run( + [ + sys.executable, + "-m", + "pip_audit", + "--strict", + # pip freeze already includes transitives; audit every exact installed version. + "--no-deps", + "--disable-pip", + "--vulnerability-service", + "pypi", + "--desc", + "on", + "--aliases", + "on", + "--format", + "json", + "--output", + "stdout", + "-r", + str(requirements), + # Preserve the pre-existing Pygments exception; do not globally ignore NLTK. + "--ignore-vuln", + "CVE-2026-4539", + ], + check=False, + capture_output=True, + text=True, + ) + report_path.write_text(result.stdout, encoding="utf-8") + print(result.stderr, end="", file=sys.stderr) + report = json.loads(result.stdout) + dependencies = _dependencies(report) + expected = [ + Requirement(line) + for line in requirements.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + if not expected: + raise ValueError("Empty requirements export") + for requirement in expected: + name = canonicalize_name(requirement.name) + pins = list(requirement.specifier) + if ( + len(pins) != 1 + or pins[0].operator not in {"==", "==="} + or "*" in pins[0].version + or requirement.url + or requirement.marker + ): + raise ValueError(f"Expected an exact installed-version pin for {name}") + if name not in dependencies or not requirement.specifier.contains( + dependencies[name]["version"], prereleases=True + ): + raise ValueError(f"Incomplete or version-mismatched audit inventory: {name}") + return evaluate_report( + report, + result.returncode, + runtime_dependencies(PROJECT), + today or datetime.now(UTC).date(), + ) + except (OSError, ValueError, KeyError, TypeError) as exc: + print(f"BLOCKED: dependency audit could not complete: {exc}", file=sys.stderr) + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("-r", "--requirements", type=Path, required=True) + parser.add_argument("--report", type=Path, default=Path("dependency-audit-report.json")) + args = parser.parse_args() + return run_audit(args.requirements, args.report) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_secret_scans.sh b/scripts/run_secret_scans.sh new file mode 100644 index 00000000..0961f3fd --- /dev/null +++ b/scripts/run_secret_scans.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + echo "Usage: $0 [base-ref=origin/develop]" >&2 + exit 1 +fi + +scanner="$1" +base_ref="${2:-origin/develop}" +# Resolve before scanning; an unavailable PR base must not silently skip history. +base_sha="$(git rev-parse --verify --end-of-options "${base_ref}^{commit}")" + +"${scanner}" dir . --no-banner --redact --timeout=300 +"${scanner}" git . --log-opts="--no-merges ${base_sha}..HEAD" --no-banner --redact --timeout=300 diff --git a/scripts/security_check.sh b/scripts/security_check.sh index 196b0179..8dea99be 100755 --- a/scripts/security_check.sh +++ b/scripts/security_check.sh @@ -4,7 +4,6 @@ set -euo pipefail venv_bin="${VENV_BIN:-.venv/bin}" pip_bin="${venv_bin}/pip" -pip_audit_bin="${venv_bin}/pip-audit" safety_bin="${venv_bin}/safety" bandit_bin="${venv_bin}/bandit" requirements_file="$(mktemp /tmp/pullbox-requirements-audit.XXXXXX.txt)" @@ -21,8 +20,7 @@ echo "═══ Pullbox Security Check ═══" echo "" echo "── pip-audit (blocking) ──" "${pip_bin}" freeze --exclude-editable | awk '!/^pullbox==/' > "${requirements_file}" -if ! "${pip_audit_bin}" --strict --desc on -r "${requirements_file}" \ - --ignore-vuln CVE-2026-4539; then +if ! "${venv_bin}/python" scripts/run_dependency_audit.py -r "${requirements_file}"; then status=1 fi echo "" diff --git a/scripts/verify_container_security_runtime.py b/scripts/verify_container_security_runtime.py index 656c6d25..6d4ebc8d 100644 --- a/scripts/verify_container_security_runtime.py +++ b/scripts/verify_container_security_runtime.py @@ -3,6 +3,7 @@ from __future__ import annotations import pyexpat +from importlib.util import find_spec MINIMUM_EXPAT_VERSION = (2, 8, 1) @@ -17,6 +18,10 @@ def verify_expat_version(version: tuple[int, int, int]) -> None: def main() -> None: """Run all container runtime security assertions.""" + for package in ("safety", "nltk"): + if find_spec(package) is not None: + raise SystemExit(f"Development-only package {package} must not ship in production") + print("Container excludes development-only Safety/NLTK dependencies") verify_expat_version(pyexpat.version_info) print(f"Container Expat runtime verified: {pyexpat.EXPAT_VERSION}") diff --git a/src/pullbox/__init__.py b/src/pullbox/__init__.py index 4acb83d3..712524b2 100644 --- a/src/pullbox/__init__.py +++ b/src/pullbox/__init__.py @@ -2,7 +2,7 @@ from datetime import UTC, datetime -__version__ = "1.2.1" +__version__ = "1.3.0" # Set once at process start; used by System > About for uptime calculation. STARTED_AT: datetime = datetime.now(UTC) diff --git a/src/pullbox/__main__.py b/src/pullbox/__main__.py index 69519367..f1fc6a8b 100644 --- a/src/pullbox/__main__.py +++ b/src/pullbox/__main__.py @@ -17,6 +17,10 @@ validate_https_runtime_settings, ) +# Leave time for in-flight requests to finish while ensuring long-lived SSE +# connections cannot consume Docker's default 10-second stop grace period. +GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS = 5 + def _resolve_db_path(db_url: str) -> Path | None: """Extract the SQLite file path from a database URL. @@ -61,6 +65,7 @@ def main() -> None: host=settings.bind_address, port=settings.port, factory=True, + timeout_graceful_shutdown=GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS, **uvicorn_ssl_kwargs(https_settings), ) diff --git a/src/pullbox/api/v1/catalog.py b/src/pullbox/api/v1/catalog.py new file mode 100644 index 00000000..7f23c501 --- /dev/null +++ b/src/pullbox/api/v1/catalog.py @@ -0,0 +1,41 @@ +"""Instance-local catalog controls. No metadata proxy routes.""" + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +from pullbox.api.deps import AuthenticatedUser, InteractiveOperatorUser +from pullbox.services.catalog.service import CatalogStatus + +router = APIRouter(prefix="/catalog", tags=["catalog"]) + + +@router.get("") +async def catalog_status(user: AuthenticatedUser) -> CatalogStatus: + from pullbox.services.catalog.service import get_catalog_service + + return get_catalog_service().status() + + +@router.post("/sync", status_code=202) +async def sync_catalog(user: InteractiveOperatorUser) -> dict[str, str]: + from pullbox.core.scheduler import get_scheduler + + status = get_scheduler().run_task_now("catalog_update") + if status is None: + raise HTTPException(503, "The catalog task is unavailable. Restart Pullbox and retry.") + return {"status": status} + + +class CatalogPreferences(BaseModel): + automatic_updates: bool + + +@router.patch("/preferences") +async def catalog_preferences( + body: CatalogPreferences, user: InteractiveOperatorUser +) -> CatalogStatus: + from pullbox.services.catalog.service import get_catalog_service + + service = get_catalog_service() + await service.set_automatic_updates(body.automatic_updates) + return service.status() diff --git a/src/pullbox/api/v1/config.py b/src/pullbox/api/v1/config.py index a08d4a67..5f55e597 100644 --- a/src/pullbox/api/v1/config.py +++ b/src/pullbox/api/v1/config.py @@ -25,9 +25,48 @@ from pullbox.schemas.config import ( ConfigResponse, ConfigUpdate, + LibraryRootCreate, + LibraryRootPolicyClear, + LibraryRootPolicyPreviewRequest, + LibraryRootPolicyPreviewResponse, + LibraryRootPolicyState, + LibraryRootPolicyUpdate, + LibraryRootPreviewResponse, + LibraryRootRebindConfirmRequest, + LibraryRootRebindPreviewRequest, + LibraryRootRebindPreviewResponse, + LibraryRootRemovalConfirm, + LibraryRootRemovalPreview, + LibraryRootState, + LibraryRootUpdate, NamingPreview, NamingPreviewEntry, NamingPreviewGrouped, + NamingSettingsPreview, + NamingSettingsPreviewRequest, + NamingSettingsState, + NamingSettingsUpdate, +) +from pullbox.services.library_root_management import ( + create_library_root, + list_library_roots, + preview_library_root, + preview_library_root_rebind, + preview_library_root_removal, + rebind_library_root, + remove_library_root, + update_library_root, +) +from pullbox.services.library_root_policy_service import ( + clear_library_root_policy, + get_library_root_policy_state, + preview_library_root_policy, + update_library_root_policy, +) +from pullbox.services.naming_settings import ( + get_naming_settings, + preview_naming_settings, + save_naming_settings, ) logger = structlog.get_logger(__name__) @@ -35,6 +74,37 @@ router = APIRouter(prefix="/config", tags=["config"], include_in_schema=False) +@router.get("/naming", response_model=NamingSettingsState) +async def naming_settings( + _user: InteractiveOperatorUser, + session: DbSession, + library_root_id: int | None = Query(None, gt=0), +) -> NamingSettingsState: + """Read global defaults or effective naming for a single library.""" + return await get_naming_settings(session, library_root_id) + + +@router.put("/naming", response_model=NamingSettingsState) +async def update_naming_settings( + payload: NamingSettingsUpdate, + _user: InteractiveOperatorUser, + session: DbSession, +) -> NamingSettingsState: + """Save one naming scope without renaming existing files.""" + state = await save_naming_settings(session, payload) + logger.info("naming_settings_updated", library_root_id=state.library_root_id) + return state + + +@router.post("/naming/preview", response_model=NamingSettingsPreview) +async def preview_scoped_naming_settings( + payload: NamingSettingsPreviewRequest, + _user: InteractiveOperatorUser, +) -> NamingSettingsPreview: + """Preview all naming fields with the proposed character cleanup settings.""" + return preview_naming_settings(payload.policy) + + def _validate_library_permission_setting(key: str, value: str) -> None: """Validate library permission config before it reaches import workflows.""" from pullbox.core.exceptions import ValidationError @@ -146,6 +216,22 @@ async def update_config( runtime_managed_keys = {"logs_dir", "backup_dir"} runtime_managed_https = https_runtime_config_values() + from pullbox.services.story_arc_file_defaults import ( + STORY_ARC_FILE_DEFAULT_KEYS, + validate_story_arc_file_defaults, + ) + from pullbox.services.story_arc_placement_integration import StoryArcPlacementIntegrationError + + # Validate the complete group before any update or runtime side effect. + if body.values.keys() & set(STORY_ARC_FILE_DEFAULT_KEYS): + effective_arc_files = await _effective_config_values( + session, body.values, STORY_ARC_FILE_DEFAULT_KEYS + ) + try: + await validate_story_arc_file_defaults(session, effective_arc_files) + except StoryArcPlacementIntegrationError as exc: + raise ValidationError(str(exc)) from exc + actually_changed: set[str] = set() old_values: dict[str, str] = {} @@ -629,6 +715,22 @@ async def update_config( # ── ComicVine API Key ──────────────────────────────────────────────── +@router.post("/story-arc-files/preview") +async def preview_story_arc_file_defaults( + body: ConfigUpdate, + _user: InteractiveOperatorUser, +) -> dict[str, str]: + """Render sample naming with the real placement renderer; never touch disk.""" + from pullbox.core.exceptions import ValidationError + from pullbox.services.story_arc_file_defaults import parse_story_arc_file_defaults + from pullbox.services.story_arc_placement_integration import StoryArcPlacementIntegrationError + + try: + return {"path": parse_story_arc_file_defaults(body.values).naming_preview()} + except StoryArcPlacementIntegrationError as exc: + raise ValidationError(str(exc)) from exc + + @router.post("/comicvine/test") async def test_comicvine_key( _user: InteractiveOperatorUser, @@ -681,6 +783,223 @@ async def save_comicvine_key( } +# ── Library Root Management ──────────────────────────────────────── + + +@router.get("/library-roots", response_model=list[LibraryRootState]) +async def get_library_roots( + _user: InteractiveOperatorUser, + session: DbSession, +) -> list[LibraryRootState]: + """List configured roots with live read/write/capacity state.""" + states = await list_library_roots(session) + return [LibraryRootState.model_validate(state) for state in states] + + +@router.post( + "/library-roots/preview", + response_model=LibraryRootPreviewResponse, +) +async def preview_new_library_root( + body: LibraryRootCreate, + _user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootPreviewResponse: + """Validate a proposed root without persisting it.""" + preview = await preview_library_root(session, **body.model_dump()) + return LibraryRootPreviewResponse.model_validate(preview) + + +@router.post( + "/library-roots", + response_model=LibraryRootState, + status_code=201, +) +async def post_library_root( + body: LibraryRootCreate, + _user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootState: + """Add an existing persistent container directory as a root.""" + state = await create_library_root(session, **body.model_dump()) + logger.info("library_root_created", library_root_id=state["id"]) + return LibraryRootState.model_validate(state) + + +@router.patch( + "/library-roots/{library_root_id}", + response_model=LibraryRootState, +) +async def patch_library_root( + library_root_id: int, + body: LibraryRootUpdate, + _user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootState: + """Update root roles/default state while preserving immutable path identity.""" + state = await update_library_root( + session, + library_root_id, + body.model_dump(exclude_unset=True), + ) + logger.info("library_root_updated", library_root_id=library_root_id) + return LibraryRootState.model_validate(state) + + +@router.post( + "/library-roots/{library_root_id}/rebind/preview", + response_model=LibraryRootRebindPreviewResponse, +) +async def preview_existing_library_root_rebind( + library_root_id: int, + body: LibraryRootRebindPreviewRequest, + user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootRebindPreviewResponse: + """Preview a path rebind and its persisted association impact without writing.""" + preview = await preview_library_root_rebind( + session, + library_root_id, + replacement_path=body.replacement_path, + actor_id=user.id, + ) + return LibraryRootRebindPreviewResponse.model_validate(preview) + + +@router.post( + "/library-roots/{library_root_id}/rebind", + response_model=LibraryRootState, +) +async def confirm_existing_library_root_rebind( + library_root_id: int, + body: LibraryRootRebindConfirmRequest, + user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootState: + """Apply one explicitly confirmed, signed and drift-checked root path rebind.""" + state = await rebind_library_root( + session, + library_root_id, + replacement_path=body.replacement_path, + preview_token=body.preview_token, + actor_id=user.id, + ) + logger.info("library_root_rebound", library_root_id=library_root_id) + return LibraryRootState.model_validate(state) + + +@router.post( + "/library-roots/{library_root_id}/remove/preview", response_model=LibraryRootRemovalPreview +) +async def preview_root_removal( + library_root_id: int, + user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootRemovalPreview: + preview = await preview_library_root_removal(session, library_root_id, actor_id=user.id) + return LibraryRootRemovalPreview.model_validate(preview) + + +@router.delete("/library-roots/{library_root_id}", status_code=204) +async def delete_library_root( + library_root_id: int, + body: LibraryRootRemovalConfirm, + user: InteractiveOperatorUser, + session: DbSession, +) -> None: + await remove_library_root( + session, library_root_id, actor_id=user.id, preview_token=body.preview_token + ) + logger.info("library_root_removed", library_root_id=library_root_id, actor_id=user.id) + + +# ── Per-root Naming Policy ────────────────────────────────────────── + + +@router.get( + "/library-roots/{library_root_id}/naming-policy", + response_model=LibraryRootPolicyState, +) +async def get_root_naming_policy( + library_root_id: int, + _user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootPolicyState: + """Return a library root's effective policy and inheritance scope.""" + state = await get_library_root_policy_state(session, library_root_id) + return LibraryRootPolicyState.model_validate(state) + + +@router.put( + "/library-roots/{library_root_id}/naming-policy", + response_model=LibraryRootPolicyState, +) +async def put_root_naming_policy( + library_root_id: int, + body: LibraryRootPolicyUpdate, + _user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootPolicyState: + """Create or update one root's explicit policy with optimistic locking.""" + state = await update_library_root_policy( + session, + library_root_id, + expected_revision=body.expected_revision, + definition=body.policy.model_dump(), + ) + logger.info( + "library_root_policy_updated", + library_root_id=library_root_id, + revision=state["revision"], + source="manual", + ) + return LibraryRootPolicyState.model_validate(state) + + +@router.delete( + "/library-roots/{library_root_id}/naming-policy", + response_model=LibraryRootPolicyState, +) +async def delete_root_naming_policy( + library_root_id: int, + body: LibraryRootPolicyClear, + _user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootPolicyState: + """Clear an explicit policy so the root inherits global defaults again.""" + state = await clear_library_root_policy( + session, + library_root_id, + expected_revision=body.expected_revision, + ) + logger.info( + "library_root_policy_cleared", + library_root_id=library_root_id, + source="manual", + ) + return LibraryRootPolicyState.model_validate(state) + + +@router.post( + "/library-roots/{library_root_id}/naming-policy/preview", + response_model=LibraryRootPolicyPreviewResponse, +) +async def preview_root_naming_policy( + library_root_id: int, + body: LibraryRootPolicyPreviewRequest, + _user: InteractiveOperatorUser, + session: DbSession, +) -> LibraryRootPolicyPreviewResponse: + """Preview a complete proposal without writing a root policy.""" + preview = await preview_library_root_policy( + session, + library_root_id, + definition=body.policy.model_dump(), + examples=[example.model_dump() for example in body.examples], + ) + return LibraryRootPolicyPreviewResponse.model_validate(preview) + + # ── Naming Preview ─────────────────────────────────────────────────── _SAMPLE_DATA = [ diff --git a/src/pullbox/api/v1/covers.py b/src/pullbox/api/v1/covers.py index a540d2ca..0f438bc9 100644 --- a/src/pullbox/api/v1/covers.py +++ b/src/pullbox/api/v1/covers.py @@ -20,7 +20,14 @@ from pullbox.core.library_root_resolution import resolve_path_inside_roots from pullbox.models.issue import Issue from pullbox.models.series import Series -from pullbox.services.cover_cache_service import cache_series_cover, resolve_series_cover_file +from pullbox.models.story_arc import StoryArc +from pullbox.services.cover_cache_service import ( + cache_series_cover, + cache_story_arc_cover, + resolve_series_cover_file, + resolve_story_arc_cover_file, +) +from pullbox.services.cover_url_service import story_arc_provider_cover_url logger = structlog.get_logger(__name__) @@ -113,6 +120,36 @@ async def get_series_cover( return Response(status_code=404) +# ── Story Arc Cover ─────────────────────────────────────────────── + + +@router.get("/story-arcs/{story_arc_id}/cover") +async def get_story_arc_cover( + story_arc_id: int, + _user: AuthenticatedUser, + session: DbSession, +) -> Response: + """Serve cached provider artwork for one Story Arc.""" + story_arc = await session.get(StoryArc, story_arc_id) + if story_arc is None: + raise NotFoundError("Story Arc", story_arc_id) + cover = await resolve_story_arc_cover_file(session, story_arc) + if cover: + if not story_arc.cover_path: + story_arc.cover_path = f"/api/v1/story-arcs/{story_arc.id}/cover" + await session.commit() + return _serve_image(cover) + provider_url = story_arc_provider_cover_url(story_arc) + if provider_url: + if not story_arc.cover_url: + story_arc.cover_url = provider_url + cover = await cache_story_arc_cover(session, story_arc) + if cover: + await session.commit() + return _serve_image(cover) + return Response(status_code=404) + + # ── Issue Cover ─────────────────────────────────────────────────── diff --git a/src/pullbox/api/v1/downloads.py b/src/pullbox/api/v1/downloads.py index 4b8d294e..0c63bcf2 100644 --- a/src/pullbox/api/v1/downloads.py +++ b/src/pullbox/api/v1/downloads.py @@ -7,7 +7,7 @@ from pullbox.api.deps import AuthenticatedUser, DbSession from pullbox.core.acquisition import AcquisitionProtocol -from pullbox.core.exceptions import NotFoundError +from pullbox.core.exceptions import NotFoundError, ProviderError from pullbox.models.blocklist import BlocklistReason from pullbox.models.direct_acquisition import DirectAcquisitionAttempt from pullbox.models.download import DownloadClientType, DownloadHistory, DownloadState @@ -15,6 +15,7 @@ from pullbox.providers.base import DownloadClient, ProviderRegistry from pullbox.providers.download.qbittorrent import QBittorrentError from pullbox.providers.indexer.newznab import NewznabError +from pullbox.providers.indexer.prowlarr import ProwlarrError from pullbox.schemas.blocklist import BlocklistEntryResponse from pullbox.schemas.download import ( DirectSourceAlternative, @@ -853,7 +854,7 @@ async def retry_download( status_code=409, detail=f"Unsupported acquisition protocol: {protocol.value}", ) - except (NewznabError, QBittorrentError) as exc: + except (NewznabError, ProwlarrError, QBittorrentError, ProviderError) as exc: logger.warning( "download_retry_client_rejected", download_id=download.id, diff --git a/src/pullbox/api/v1/filesystem.py b/src/pullbox/api/v1/filesystem.py index ff067230..1352aac8 100644 --- a/src/pullbox/api/v1/filesystem.py +++ b/src/pullbox/api/v1/filesystem.py @@ -10,27 +10,11 @@ from pullbox.api.deps import DbSession, InteractiveOperatorUser from pullbox.core.file_safety import get_allowed_extensions +from pullbox.core.filesystem_policy import is_sensitive_path logger = structlog.get_logger(__name__) router = APIRouter(prefix="/filesystem", tags=["filesystem"], include_in_schema=False) -# Directories that should never be browsable via the API. -# We resolve() each prefix so symlinks (e.g. macOS /etc → /private/etc) are caught. -_BLOCKED_DIRS: tuple[str, ...] = ( - "/etc", - "/proc", - "/sys", - "/dev", - "/run", - "/boot", - "/root", - "/var/log", - "/var/run", -) -_BLOCKED_PREFIXES: frozenset[str] = frozenset( - {p for d in _BLOCKED_DIRS for p in (d, str(Path(d).resolve()))} -) - _MAX_PATH_LENGTH = 4096 @@ -168,7 +152,6 @@ def _validate_browsable_path(path: str, allowed_roots: Sequence[Path] | None = N # before any listing is returned. # codeql[py/path-injection] resolved = Path(sanitized).resolve() - resolved_str = str(resolved) # Defense in depth: reject if ".." survived resolution if ".." in resolved.parts: @@ -176,12 +159,9 @@ def _validate_browsable_path(path: str, allowed_roots: Sequence[Path] | None = N return fallback # Block sensitive system directories - for prefix in _BLOCKED_PREFIXES: - if resolved_str == prefix or resolved_str.startswith(f"{prefix}/"): - logger.warning( - "filesystem_path_blocked", requested_path=path, reason="sensitive_directory" - ) - return fallback + if is_sensitive_path(resolved): + logger.warning("filesystem_path_blocked", requested_path=path, reason="sensitive_directory") + return fallback # Fallback if path doesn't exist # ``resolved`` has passed the browser safety checks above; this probe only diff --git a/src/pullbox/api/v1/import_completed_cleanup.py b/src/pullbox/api/v1/import_completed_cleanup.py new file mode 100644 index 00000000..0be8755a --- /dev/null +++ b/src/pullbox/api/v1/import_completed_cleanup.py @@ -0,0 +1,141 @@ +"""Authenticated completed-import cleanup endpoints.""" + +from __future__ import annotations + +from fastapi import APIRouter, Request + +from pullbox.api.deps import DbSession, InteractiveOperatorUser # noqa: TC001 +from pullbox.schemas.import_completed_cleanup import ( + CleanLibraryImportCreateRequest, + CleanLibraryImportPreviewRead, + CleanLibraryImportResultRead, + CompletedImportCleanupApplyRequest, + CompletedImportCleanupPreviewRead, + CompletedImportCleanupResultRead, +) +from pullbox.services.audit_service import source_ip_from_request +from pullbox.services.import_completed_cleanup import ( + CompletedImportCleanupAction, + apply_completed_import_cleanup, + preview_completed_import_cleanup, +) +from pullbox.services.import_library_adoption import ( + create_clean_library_import, + preview_clean_library_import, +) +from pullbox.tasks.import_task import trigger_import_execute + +router = APIRouter(prefix="/import", tags=["import"]) + + +@router.get( + "/{job_id}/clean-library/preview", + response_model=CleanLibraryImportPreviewRead, +) +async def preview_clean_library_import_route( + job_id: int, + target_root_id: int, + _user: InteractiveOperatorUser, + session: DbSession, +) -> CleanLibraryImportPreviewRead: + """Preview a source-preserving clean-library build.""" + preview = await preview_clean_library_import( + session, + job_id, + target_root_id=target_root_id, + actor_id=_user.id, + ) + return CleanLibraryImportPreviewRead.model_validate(preview, from_attributes=True) + + +@router.post( + "/{job_id}/clean-library", + response_model=CleanLibraryImportResultRead, +) +async def create_clean_library_import_route( + job_id: int, + body: CleanLibraryImportCreateRequest, + _user: InteractiveOperatorUser, + session: DbSession, +) -> CleanLibraryImportResultRead: + """Start a verified managed-copy import from referenced files.""" + result = await create_clean_library_import( + session, + job_id, + target_root_id=body.target_root_id, + actor_id=_user.id, + preview_token=body.preview_token, + ) + await session.commit() + trigger_import_execute(result.job_id) + return CleanLibraryImportResultRead.model_validate(result, from_attributes=True) + + +@router.get( + "/{job_id}/cleanup/{action}/preview", + response_model=CompletedImportCleanupPreviewRead, +) +async def preview_completed_import_cleanup_route( + job_id: int, + action: CompletedImportCleanupAction, + _user: InteractiveOperatorUser, + session: DbSession, +) -> CompletedImportCleanupPreviewRead: + """Preview an exact completed-job recovery scope.""" + preview = await preview_completed_import_cleanup( + session, + job_id, + action, + actor_id=_user.id, + ) + return CompletedImportCleanupPreviewRead.model_validate(preview, from_attributes=True) + + +@router.post( + "/{job_id}/cleanup/{action}", + response_model=CompletedImportCleanupResultRead, +) +async def apply_completed_import_cleanup_route( + job_id: int, + action: CompletedImportCleanupAction, + body: CompletedImportCleanupApplyRequest, + request: Request, + _user: InteractiveOperatorUser, + session: DbSession, +) -> CompletedImportCleanupResultRead: + """Apply a signed recovery scope and resume only the affected work.""" + from pullbox.composition.services import build_import_service + + result = await apply_completed_import_cleanup( + session, + job_id, + action, + actor_id=_user.id, + actor_username=_user.username, + source_ip=source_ip_from_request(request), + preview_token=body.preview_token, + ) + + if action is CompletedImportCleanupAction.RETRY_SOURCE_INSPECTION: + # Persist the signed cleanup transition before slow archive inspection. + # The recheck then starts without an existing SQLite writer lock. + await session.commit() + service = await build_import_service(session) + _job, retrying_count = await service.retry_failed_series( + session, + job_id, + file_ids=list(result.retry_file_ids), + ) + result_read = CompletedImportCleanupResultRead.model_validate( + result, + from_attributes=True, + ).model_copy(update={"requires_import_retry": retrying_count > 0}) + await session.commit() + if retrying_count > 0: + trigger_import_execute(job_id) + return result_read + + await session.commit() + if result.requires_import_retry: + trigger_import_execute(job_id) + return CompletedImportCleanupResultRead.model_validate(result, from_attributes=True) diff --git a/src/pullbox/api/v1/import_job_archive.py b/src/pullbox/api/v1/import_job_archive.py new file mode 100644 index 00000000..5f68b70e --- /dev/null +++ b/src/pullbox/api/v1/import_job_archive.py @@ -0,0 +1,46 @@ +"""Authenticated import history archive endpoints.""" + +from __future__ import annotations + +from fastapi import APIRouter + +from pullbox.api.deps import DbSession, InteractiveOperatorUser # noqa: TC001 +from pullbox.schemas.import_job_archive import ImportJobArchiveResponse +from pullbox.services.import_job_archive import set_import_job_archived + +router = APIRouter(prefix="/import", tags=["import"]) + + +async def _set_archive_state( + session: DbSession, + job_id: int, + *, + archived: bool, +) -> ImportJobArchiveResponse: + job = await set_import_job_archived(session, job_id, archived=archived) + await session.commit() + return ImportJobArchiveResponse( + job_id=job.id, + archived=job.archived_at is not None, + archived_at=job.archived_at, + ) + + +@router.post("/{job_id}/archive", response_model=ImportJobArchiveResponse) +async def archive_import_job( + job_id: int, + _user: InteractiveOperatorUser, + session: DbSession, +) -> ImportJobArchiveResponse: + """Archive one finished import without deleting it.""" + return await _set_archive_state(session, job_id, archived=True) + + +@router.post("/{job_id}/restore", response_model=ImportJobArchiveResponse) +async def restore_import_job( + job_id: int, + _user: InteractiveOperatorUser, + session: DbSession, +) -> ImportJobArchiveResponse: + """Restore one archived import to the normal history view.""" + return await _set_archive_state(session, job_id, archived=False) diff --git a/src/pullbox/api/v1/import_job_control_actions.py b/src/pullbox/api/v1/import_job_control_actions.py index 042b943a..ba499116 100644 --- a/src/pullbox/api/v1/import_job_control_actions.py +++ b/src/pullbox/api/v1/import_job_control_actions.py @@ -18,14 +18,29 @@ from pullbox.schemas.import_job import ( ConfirmImportRequest, ImportJobCreate, + ImportJobDeleteResponse, ImportJobRead, ImportPreviewResponse, RetryFailedResponse, RetryImportResponse, + RetryStoryArcPlacementsResponse, +) +from pullbox.services.import_managed_copy_preflight import ManagedCopyPreflightError +from pullbox.services.story_arc_sync_queue import ( + discard_unpublished_import_story_arc_sync_work, ) logger = structlog.get_logger(__name__) +_ROLLBACK_PENDING_DELETE_MESSAGE = ( + "Rollback is still stopping an in-progress Story Arc placement. " + "The import remains in history; delete it again after rollback finishes." +) +_ROLLBACK_INCOMPLETE_DELETE_MESSAGE = ( + "Rollback is incomplete and some data requires manual recovery. " + "The import remains in history so its recovery evidence is preserved." +) + _CLEARABLE_HISTORY_STATUSES = ( ImportJobStatus.COMPLETED, ImportJobStatus.FAILED, @@ -106,6 +121,11 @@ async def confirm_import_response( """Confirm selected series for import and trigger the execute task.""" try: job = await service.confirm_import(session, job_id, body) + except ManagedCopyPreflightError as exc: + # The service has restored REVIEW state and attached a sanitized live + # capacity snapshot. Persist that evidence before returning the block. + await session.commit() + raise HTTPException(status_code=409, detail=exc.message) from exc except ValidationError as exc: raise HTTPException(status_code=409, detail=exc.message) from exc @@ -120,10 +140,20 @@ async def confirm_import_response( async def clear_import_history_response(session: Any) -> dict[str, int]: """Delete terminal import history records while leaving active jobs intact.""" jobs_result = await session.execute( - sa_select(ImportJob).where(ImportJob.status.in_(_CLEARABLE_HISTORY_STATUSES)) + sa_select(ImportJob).where( + ImportJob.status.in_(_CLEARABLE_HISTORY_STATUSES), + ImportJob.archived_at.is_(None), + ) ) jobs = list(jobs_result.scalars().all()) + try: + await discard_unpublished_import_story_arc_sync_work( + session, + tuple(int(job.id) for job in jobs), + ) + except ValidationError as exc: + raise HTTPException(status_code=409, detail=exc.message) from exc for job in jobs: await session.delete(job) @@ -137,7 +167,7 @@ async def cancel_import_job_response( session: Any, job_id: int, purge_import_runtime_state: Any, -) -> None: +) -> ImportJobDeleteResponse | None: """Cancel an active import job or delete a finished one from history.""" try: action = await service.cancel_job(session, job_id) @@ -145,11 +175,16 @@ async def cancel_import_job_response( raise HTTPException(status_code=409, detail=exc.message) from exc await session.commit() - purge_import_runtime_state(job_id) - logger.info( - "import_job_deleted" if action == "deleted" else "import_job_cancelled", - job_id=job_id, - ) + if action == "deleted": + purge_import_runtime_state(job_id) + logger.info("import_job_deleted", job_id=job_id) + return None + if action == "rollback_incomplete": + raise HTTPException(status_code=409, detail=_ROLLBACK_INCOMPLETE_DELETE_MESSAGE) + if action != "rollback_pending": + raise RuntimeError(f"Unsupported import deletion result: {action!r}") + logger.info("import_job_delete_waiting_for_story_arc_rollback", job_id=job_id) + return ImportJobDeleteResponse(message=_ROLLBACK_PENDING_DELETE_MESSAGE) async def pause_import_job_response( @@ -240,11 +275,35 @@ async def retry_failed_series_response( await session.commit() logger.info("import_retry_failed", job_id=job_id, retrying_count=count) - trigger_import_execute(job_id) + if count > 0: + trigger_import_execute(job_id) return RetryFailedResponse(job_id=job.id, retrying_count=count) +async def retry_story_arc_placements_response( + service: Any, + *, + session: Any, + job_id: int, + trigger_story_arc_sync: Any, +) -> RetryStoryArcPlacementsResponse: + """Requeue terminal import placement work and nudge its durable worker.""" + try: + job, count = await service.retry_story_arc_placements(session, job_id) + except ValidationError as exc: + raise HTTPException(status_code=409, detail=exc.message) from exc + + await session.commit() + logger.info( + "import_story_arc_placements_retry_requested", + job_id=job_id, + retrying_count=count, + ) + trigger_story_arc_sync() + return RetryStoryArcPlacementsResponse(job_id=job.id, retrying_count=count) + + async def allow_safety_blocked_file_once_and_retry_response( service: Any, *, diff --git a/src/pullbox/api/v1/import_job_review_actions.py b/src/pullbox/api/v1/import_job_review_actions.py index 256f3f60..c4f8f920 100644 --- a/src/pullbox/api/v1/import_job_review_actions.py +++ b/src/pullbox/api/v1/import_job_review_actions.py @@ -28,7 +28,15 @@ ImportSelectionBulkUpdateResponse, SeriesSelectionBulkUpdateRequest, SeriesSelectionUpdateRequest, + StoryArcPolicyConfirmationRequest, + StoryArcPolicyConfirmationResponse, + StoryArcReviewDecisionRequest, + StoryArcReviewDecisionResponse, ) +from pullbox.services.import_story_arc_policy_confirmation import ( + confirm_import_story_arc_policy, +) +from pullbox.services.story_arc_placement_integration import StoryArcPlacementPolicyInput @dataclass(frozen=True) @@ -91,20 +99,33 @@ async def list_conflicts_response( service: Any, session: Any, job_id: int, + *, + page: int, + page_size: int, ) -> ConflictGroupsResponse: - """Return all conflict groups for an import job.""" - groups = await service.get_conflict_groups(session, job_id) + """Return one bounded conflict-group page for an import job.""" + result = await service.get_conflict_groups_page( + session, + job_id, + page=page, + page_size=page_size, + ) return ConflictGroupsResponse( job_id=job_id, groups=[ FileConflictGroup( + kind=g["kind"], conflict_group_id=g["conflict_group_id"], matched_issue_id=g["matched_issue_id"], + series_id=g.get("series_id"), + diagnostics=dict(g.get("diagnostics") or {}), files=[ImportedFileRead.model_validate(f) for f in g["files"]], ) - for g in groups + for g in result.items ], - total=len(groups), + total=result.total, + page=result.page, + page_size=result.page_size, ) @@ -169,6 +190,66 @@ async def update_series_selection_response( return ImportedSeriesRead.model_validate(imported_series) +async def update_story_arc_decision_response( + service: Any, + session: Any, + job_id: int, + imported_story_arc_id: int, + body: StoryArcReviewDecisionRequest, +) -> StoryArcReviewDecisionResponse: + """Persist one explicit staged story-arc review decision.""" + try: + staged_arc = await service.update_story_arc_decision( + session, + job_id, + imported_story_arc_id, + action=body.action, + proposed_story_arc_id=body.proposed_story_arc_id, + ) + except ValidationError as exc: + _raise_validation_http(exc) + return StoryArcReviewDecisionResponse( + imported_story_arc_id=int(staged_arc.id), + status=staged_arc.status.value, + selected_for_import=bool(staged_arc.selected_for_import), + proposed_story_arc_id=staged_arc.proposed_story_arc_id, + ) + + +async def confirm_story_arc_policy_response( + session: Any, + job_id: int, + imported_story_arc_id: int, + body: StoryArcPolicyConfirmationRequest, +) -> StoryArcPolicyConfirmationResponse: + """Validate and persist one explicitly confirmed staged arc policy.""" + payload = body.placement_policy + try: + result = await confirm_import_story_arc_policy( + session, + job_id=job_id, + imported_story_arc_id=imported_story_arc_id, + expected_policy_digest=body.expected_policy_digest, + explicit_confirmation=body.confirm_policy, + materialize_filesystem=body.materialize_filesystem, + monitored=body.monitored, + search_missing=body.search_missing, + include_upcoming=body.include_upcoming, + placement_policy=StoryArcPlacementPolicyInput( + mode=payload.mode, + target_library_root_id=payload.target_library_root_id, + destination_root=payload.destination_root, + folder_template=payload.folder_template, + file_template=payload.file_template, + symlink_style=payload.symlink_style, + synchronize=payload.synchronize, + ), + ) + except ValidationError as exc: + _raise_validation_http(exc) + return StoryArcPolicyConfirmationResponse.model_validate(result, from_attributes=True) + + async def get_selection_state_response( service: Any, session: Any, diff --git a/src/pullbox/api/v1/import_jobs.py b/src/pullbox/api/v1/import_jobs.py index 963a58ec..047d8537 100644 --- a/src/pullbox/api/v1/import_jobs.py +++ b/src/pullbox/api/v1/import_jobs.py @@ -3,11 +3,13 @@ from __future__ import annotations import asyncio +from dataclasses import replace +from pathlib import Path from typing import TYPE_CHECKING, Any, Literal import structlog -from fastapi import APIRouter, Query, Request -from fastapi.responses import PlainTextResponse, StreamingResponse # noqa: TC002 +from fastapi import APIRouter, Query, Request, Response +from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse from sqlalchemy.exc import OperationalError from pullbox.api.deps import AuthenticatedStreamUser, AuthenticatedUser, DbSession # noqa: TC001 @@ -25,6 +27,7 @@ resume_import_job_response, retry_failed_series_response, retry_import_job_response, + retry_story_arc_placements_response, rollback_import_job_response, ) from pullbox.api.v1.import_job_logs import ( @@ -37,6 +40,7 @@ allow_safety_blocked_file_once_response, bulk_update_file_selection_response, bulk_update_series_selection_response, + confirm_story_arc_policy_response, get_selection_state_response, list_conflicts_response, list_import_files_response, @@ -50,6 +54,7 @@ unmatch_series_match_response, update_file_selection_response, update_series_selection_response, + update_story_arc_decision_response, ) from pullbox.api.v1.import_job_streams import ( ensure_import_job_exists_for_stream as _ensure_import_job_exists_for_stream, @@ -62,7 +67,11 @@ from pullbox.api.v1.import_job_streams import ( load_initial_import_progress_sse as _load_initial_import_progress_sse, ) -from pullbox.core.exceptions import ValidationError +from pullbox.core.exceptions import MylarReadError, ValidationError +from pullbox.core.library_file_ownership import ( + ReferencedFileValidationError, + resolve_referenced_source_root, +) from pullbox.core.sqlite_lock import ( SQLITE_LOCK_RETRY_ATTEMPTS, is_sqlite_locked_error, @@ -85,6 +94,7 @@ ImportedFilesResponse, ImportedSeriesRead, ImportJobCreate, + ImportJobDeleteResponse, ImportJobLogsResponse, ImportJobRead, ImportPreviewResponse, @@ -93,9 +103,30 @@ ImportSelectionBulkUpdateResponse, RetryFailedResponse, RetryImportResponse, + RetryStoryArcPlacementsResponse, SeriesSelectionBulkUpdateRequest, SeriesSelectionUpdateRequest, + StoryArcPolicyConfirmationRequest, + StoryArcPolicyConfirmationResponse, + StoryArcReviewDecisionRequest, + StoryArcReviewDecisionResponse, +) +from pullbox.schemas.import_layout import LayoutAnalysisResponse, LayoutPreviewRequest +from pullbox.schemas.import_mylar3_path_preflight import ( + MylarPathPreviewRequest, + MylarPathPreviewResponse, +) +from pullbox.schemas.import_story_arc_preflight import ( + StoryArcPreflightRequest, + StoryArcPreflightResponse, ) +from pullbox.services import import_mylar3_path_reports as path_reports +from pullbox.services.import_layout_analysis import ImportLayoutAnalyzer +from pullbox.services.import_mylar3_path_preflight import ( + Mylar3PathPreflightAnalyzer, + add_report_unavailable_attention, +) +from pullbox.services.import_story_arc_preflight import StoryArcPreflightAnalyzer from pullbox.tasks.import_task import ( purge_import_runtime_state, trigger_import_execute, @@ -197,6 +228,127 @@ async def create_import_job( ) +@router.post("/layout-preview", response_model=LayoutAnalysisResponse) +async def preview_import_layout( + _user: AuthenticatedUser, + session: DbSession, + body: LayoutPreviewRequest, +) -> LayoutAnalysisResponse: + """Analyze a source layout without creating a job or mutating files.""" + analyzer = ImportLayoutAnalyzer() + try: + result = await analyzer.analyze( + body.source_path, + spec=body.layout.to_core(), + ) + except (OSError, ValueError) as exc: + raise ValidationError( + "The layout preview source is no longer available or readable." + ) from exc + try: + await resolve_referenced_source_root(session, Path(body.source_path), None) + except ReferencedFileValidationError: + warnings = list(result.warnings) + if "source_outside_library_root" not in warnings: + warnings.append("source_outside_library_root") + result = replace( + result, + can_keep_in_place=False, + warnings=warnings, + ) + return LayoutAnalysisResponse.model_validate(result) + + +@router.post("/mylar-path-preview", response_model=MylarPathPreviewResponse) +async def preview_import_mylar_paths( + _user: AuthenticatedUser, + session: DbSession, + body: MylarPathPreviewRequest, +) -> MylarPathPreviewResponse: + """Analyze source paths read-only and retain a private report without creating a job.""" + analyzer = Mylar3PathPreflightAnalyzer() + try: + result = await analyzer.analyze( + session, + body.source_path, + auto_detect=body.auto_detect, + mappings=body.mappings, + file_handling_mode=body.file_handling_mode, + ) + except (OSError, ValueError) as exc: + raise ValidationError( + "The Mylar path preview source is no longer available or readable." + ) from exc + try: + result.report_id = await asyncio.to_thread( + path_reports.save_report, result, body.source_path + ) + except (OSError, ValueError): + logger.warning("mylar_preflight_report_unavailable", exc_info=True) + result.warnings.append("report_unavailable") + result = add_report_unavailable_attention(result, path_reports.report_directory()) + logger.info( + "mylar_path_preflight_completed", + report_id=result.report_id, + resolution=result.resolution.model_dump(), + warnings=result.warnings, + can_confirm=result.can_confirm, + can_continue_with_unresolved=result.can_continue_with_unresolved, + ) + return result.model_copy(update={"exceptions": result.exceptions[: path_reports.PAGE_SIZE]}) + + +@router.get("/mylar-path-reports/{report_id}/export") +async def export_mylar_path_report(_user: AuthenticatedUser, report_id: str) -> JSONResponse: + try: + report = await asyncio.to_thread(path_reports.load_report, report_id) + except (OSError, ValueError): + return JSONResponse( + {"detail": "Report unavailable or expired. Analyze Mylar paths again."}, status_code=404 + ) + return JSONResponse( + report, + headers={ + "Content-Disposition": 'attachment; filename="mylar-path-preflight.json"', + "Cache-Control": "no-store", + }, + ) + + +@router.get("/mylar-path-reports/{report_id}") +async def get_mylar_path_report( + _user: AuthenticatedUser, + report_id: str, + page: int = Query(1, ge=1), + search: str = Query("", max_length=200), +) -> JSONResponse: + try: + result = await asyncio.to_thread(path_reports.report_page, report_id, page, search) + except (OSError, ValueError): + return JSONResponse( + {"detail": "Report unavailable or expired. Analyze Mylar paths again."}, status_code=404 + ) + return JSONResponse(result, headers={"Cache-Control": "no-store"}) + + +@router.post("/story-arc-preview", response_model=StoryArcPreflightResponse) +async def preview_import_story_arcs( + _user: AuthenticatedUser, + body: StoryArcPreflightRequest, +) -> StoryArcPreflightResponse: + """Inspect local Story Arc evidence without creating a job or changing files.""" + analyzer = StoryArcPreflightAnalyzer() + try: + return await analyzer.analyze( + body.source_path, + source_type=body.source_type, + ) + except (MylarReadError, OSError, ValueError) as exc: + raise ValidationError( + "The Story Arc preview source is no longer available or readable." + ) from exc + + # ── Post-Import Recovery (orphaned routes before /{job_id}) ────────── @@ -333,20 +485,32 @@ async def clear_import_history( return await clear_import_history_response(session) -@router.delete("/{job_id}", status_code=204) +@router.delete( + "/{job_id}", + status_code=204, + responses={ + 202: { + "model": ImportJobDeleteResponse, + "description": "Deletion is waiting for cooperative Story Arc rollback.", + } + }, +) async def cancel_import_job( job_id: int, _user: AuthenticatedUser, session: DbSession, -) -> None: +) -> Response: """Cancel an active import job or delete a finished one from history.""" service = _make_import_service() - await cancel_import_job_response( + pending = await cancel_import_job_response( service, session=session, job_id=job_id, purge_import_runtime_state=purge_import_runtime_state, ) + if pending is not None: + return JSONResponse(status_code=202, content=pending.model_dump(mode="json")) + return Response(status_code=204) @router.post("/{job_id}/pause", response_model=ImportJobRead) @@ -425,6 +589,26 @@ async def retry_failed_series( ) +@router.post( + "/{job_id}/story-arc-placements/retry", + status_code=202, + response_model=RetryStoryArcPlacementsResponse, +) +async def retry_import_story_arc_placements( + job_id: int, + _user: AuthenticatedUser, + session: DbSession, +) -> RetryStoryArcPlacementsResponse: + """Retry only failed/cancelled placement work for a stalled import.""" + service = _make_import_service() + return await retry_story_arc_placements_response( + service, + session=session, + job_id=job_id, + trigger_story_arc_sync=service.schedule_story_arc_sync, + ) + + @router.post( "/{job_id}/files/{file_id}/safety/allow-once-and-retry", status_code=202, @@ -497,10 +681,18 @@ async def list_conflicts( job_id: int, _user: AuthenticatedUser, session: DbSession, + page: int = Query(1, ge=1), + page_size: int = Query(25, ge=1, le=100), ) -> ConflictGroupsResponse: - """List all conflict groups for an import job.""" + """List one bounded page of conflict groups for an import job.""" service = _make_import_service() - return await list_conflicts_response(service, session, job_id) + return await list_conflicts_response( + service, + session, + job_id, + page=page, + page_size=page_size, + ) @router.put( @@ -577,6 +769,52 @@ async def update_series_selection( return response +@router.put( + "/{job_id}/story-arcs/{imported_story_arc_id}/decision", + response_model=StoryArcReviewDecisionResponse, +) +async def update_story_arc_decision( + job_id: int, + imported_story_arc_id: int, + _user: AuthenticatedUser, + session: DbSession, + body: StoryArcReviewDecisionRequest, +) -> StoryArcReviewDecisionResponse: + """Persist an explicit Step 3 select/skip decision for one staged story arc.""" + service = _make_import_service() + response = await update_story_arc_decision_response( + service, + session, + job_id, + imported_story_arc_id, + body, + ) + await session.commit() + return response + + +@router.put( + "/{job_id}/story-arcs/{imported_story_arc_id}/policy-confirmation", + response_model=StoryArcPolicyConfirmationResponse, +) +async def confirm_story_arc_policy( + job_id: int, + imported_story_arc_id: int, + _user: AuthenticatedUser, + session: DbSession, + body: StoryArcPolicyConfirmationRequest, +) -> StoryArcPolicyConfirmationResponse: + """Explicitly confirm one complete Step 3 story-arc policy.""" + response = await confirm_story_arc_policy_response( + session, + job_id, + imported_story_arc_id, + body, + ) + await session.commit() + return response + + @router.get( "/{job_id}/selection-state", response_model=ImportReviewSelectionState, diff --git a/src/pullbox/api/v1/import_safety_bulk.py b/src/pullbox/api/v1/import_safety_bulk.py new file mode 100644 index 00000000..5b1ce400 --- /dev/null +++ b/src/pullbox/api/v1/import_safety_bulk.py @@ -0,0 +1,89 @@ +"""Category-scoped bulk review endpoints for structured import safety blocks.""" + +from __future__ import annotations + +from fastapi import APIRouter, Request + +from pullbox.api.deps import DbSession, InteractiveOperatorUser # noqa: TC001 +from pullbox.core.exceptions import ValidationError +from pullbox.schemas.import_safety_bulk import ( + ImportSafetyBulkAllowRequest, + ImportSafetyBulkPreviewRead, + ImportSafetyBulkResultRead, +) +from pullbox.services.audit_service import source_ip_from_request +from pullbox.services.import_safety_bulk_review import ( + IMPORT_SAFETY_BULK_CONFIRMATION, + ImportSafetyBulkInterruptedError, + allow_import_safety_category_once, + preview_import_safety_category, +) +from pullbox.services.import_safety_diagnostics import ImportSafetyCategory # noqa: TC001 +from pullbox.tasks.import_task import trigger_import_safety_bulk_rematch + +router = APIRouter(prefix="/import", tags=["import"]) + + +def _preview_response(preview: object) -> ImportSafetyBulkPreviewRead: + payload = ImportSafetyBulkPreviewRead.model_validate(preview, from_attributes=True) + return payload.model_copy( + update={ + "requires_confirmation": payload.overrideable, + "confirmation_text": ( + IMPORT_SAFETY_BULK_CONFIRMATION if payload.overrideable else None + ), + } + ) + + +@router.get( + "/{job_id}/safety/categories/{category}/preview", + response_model=ImportSafetyBulkPreviewRead, +) +async def preview_import_safety_bulk_action( + job_id: int, + category: ImportSafetyCategory, + _user: InteractiveOperatorUser, + session: DbSession, +) -> ImportSafetyBulkPreviewRead: + """Preview one existing structured safety category in one import job.""" + preview = await preview_import_safety_category( + session, + job_id, + category, + actor_id=_user.id, + ) + return _preview_response(preview) + + +@router.post( + "/{job_id}/safety/categories/{category}/allow-once", + response_model=ImportSafetyBulkResultRead, +) +async def allow_import_safety_category_once_route( + job_id: int, + category: ImportSafetyCategory, + body: ImportSafetyBulkAllowRequest, + request: Request, + _user: InteractiveOperatorUser, + session: DbSession, +) -> ImportSafetyBulkResultRead: + """Apply a signed and explicitly confirmed category preview once.""" + try: + result = await allow_import_safety_category_once( + session, + job_id, + category, + actor_id=_user.id, + actor_username=_user.username, + source_ip=source_ip_from_request(request), + preview_token=body.preview_token, + ) + except ImportSafetyBulkInterruptedError as exc: + trigger_import_safety_bulk_rematch(job_id) + raise ValidationError( + "The safety bulk action stopped because the import job changed state." + ) from exc + + trigger_import_safety_bulk_rematch(job_id) + return ImportSafetyBulkResultRead.model_validate(result, from_attributes=True) diff --git a/src/pullbox/api/v1/intervention.py b/src/pullbox/api/v1/intervention.py index f6c17606..0c5ec499 100644 --- a/src/pullbox/api/v1/intervention.py +++ b/src/pullbox/api/v1/intervention.py @@ -247,7 +247,7 @@ async def approve_pending_match( if pm is None: raise NotFoundError("PendingMatch", pending_id) - if is_direct_pending_match(pm): + if is_direct_pending_match(pm) or (pm.match_details or {}).get("source_kind") == "dc": svc = InterventionService() else: from pullbox.composition.services import build_domain_download_service @@ -286,6 +286,7 @@ async def approve_pending_match( issue_id=approved.issue_id, title=approved.title, status=str(approved.state), + source_kind="dc" if (pm.match_details or {}).get("source_kind") == "dc" else "indexer", ) diff --git a/src/pullbox/api/v1/issues.py b/src/pullbox/api/v1/issues.py index b6500899..93f97fa4 100644 --- a/src/pullbox/api/v1/issues.py +++ b/src/pullbox/api/v1/issues.py @@ -2,7 +2,7 @@ import time from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path import structlog @@ -12,14 +12,19 @@ from sqlalchemy.orm import joinedload, selectinload from pullbox.api.deps import AuthenticatedUser, DbSession -from pullbox.composition.airdcpp import get_airdcpp_supervisor_registry +from pullbox.composition.airdcpp import ( + get_airdcpp_supervisor_registry, + load_airdcpp_search_clients, +) from pullbox.config import get_settings from pullbox.core.exceptions import ConfigurationError, NotFoundError, ValidationError from pullbox.core.file_ops import register_library_file from pullbox.core.file_safety import classify_resource_safety_exception +from pullbox.core.library_root_resolution import preferred_managed_root_id from pullbox.models.client import DownloadClientConfig from pullbox.models.direct_acquisition import DirectAcquisitionAttempt from pullbox.models.download import DownloadClientType, DownloadState +from pullbox.models.indexer import IndexerConfig from pullbox.models.issue import Issue, IssueStatus, IssueType from pullbox.models.library import LibraryFile, MatchConfidence from pullbox.models.search_log import SearchLog, SearchType @@ -46,6 +51,7 @@ SearchResultItem, ) from pullbox.services.airdcpp_acquisition import AirDcppQueueAcquisitionService +from pullbox.services.airdcpp_automatic_search import attach_automatic_airdcpp_search from pullbox.services.airdcpp_route_tokens import get_airdcpp_route_token_store from pullbox.services.direct_acquisition_planner_service import ( DirectAcquisitionPlanningError, @@ -155,6 +161,7 @@ def _enrich_issue(issue: Issue) -> dict[str, object]: """Add computed fields (series_title, has_file) to an issue.""" mapper = inspect(type(issue)) data: dict[str, object] = {c.key: getattr(issue, c.key) for c in mapper.columns} + data["issue_number_text"] = issue.effective_issue_number_text data["series_title"] = issue.series.title if issue.series else None data["has_file"] = issue.library_file is not None return data @@ -418,10 +425,17 @@ async def _run_issue_search( raise NotFoundError("Issue", issue_id) issue_ctx = _build_issue_context(target) + has_automatic_dc = False + dc_registry = get_airdcpp_supervisor_registry() if include_download_clients else None + if dc_registry is not None: + has_automatic_dc = bool( + await load_airdcpp_search_clients(session, dc_registry, automatic=True) + ) runtime = await build_search_runtime( session, include_download_clients=include_download_clients, include_direct_providers=True, + allow_empty_registry=has_automatic_dc, ) # Release the read transaction before slow indexer/network work. Search log # persistence happens later in a short write transaction owned by the caller. @@ -553,7 +567,9 @@ def _build_issue_search_log( direct_outcome = outcome.direct_outcome direct_matched = len(direct_outcome.matched) if direct_outcome else 0 direct_rejected = len(direct_outcome.rejected) if direct_outcome else 0 - details["validated_count"] = len(bundle.matched_items) + direct_matched + dc_outcome = outcome.dc_outcome + dc_matched = len(dc_outcome.matched) if dc_outcome else 0 + details["validated_count"] = len(outcome.matched) + direct_matched + dc_matched details["direct_results_count"] = direct_matched + direct_rejected details["direct_providers_searched"] = ( direct_outcome.providers_searched if direct_outcome else 0 @@ -567,7 +583,7 @@ def _build_issue_search_log( } for failure in direct_outcome.failures ] - results_found = len(outcome.raw_results) + direct_matched + direct_rejected + results_found = outcome.results_found_count best_confidence = ( outcome.best_validation.confidence.value if outcome.best_validation is not None @@ -583,12 +599,7 @@ def _build_issue_search_log( if action_status: details["action_status"] = action_status if results_rejected is None: - direct_rejected = ( - len(bundle.outcome.direct_outcome.rejected) - if bundle.outcome and bundle.outcome.direct_outcome - else 0 - ) - results_rejected = len(bundle.rejected_items) + direct_rejected + results_rejected = outcome.results_rejected_count if outcome else 0 return SearchLog( issue_id=bundle.target.issue_id, @@ -755,8 +766,19 @@ async def grab_release( raise ProviderError("download", "No download clients configured") - download_svc, indexer_configs = built - if body.indexer_id is not None and body.indexer_id not in indexer_configs: + download_svc, _health_configs = built + # Aggregated Prowlarr sources intentionally do not have per-indexer health entries. + if ( + body.indexer_id is not None + and await session.scalar( + select(IndexerConfig.id).where( + IndexerConfig.id == body.indexer_id, + IndexerConfig.enabled.is_(True), + IndexerConfig.manager_available.is_(True), + ) + ) + is None + ): raise ValidationError( "The originating indexer is no longer available. Run the search again." ) @@ -992,6 +1014,16 @@ async def download_issue( issue_id, include_download_clients=True, ) + if bundle.runtime is not None and bundle.outcome is not None: + started_at = time.monotonic() + outcome = await attach_automatic_airdcpp_search( + session, bundle.outcome, validator_kwargs=bundle.runtime.validator_kwargs + ) + bundle = replace( + bundle, + outcome=outcome, + search_time_ms=bundle.search_time_ms + int((time.monotonic() - started_at) * 1000), + ) # Searching a skipped issue implicitly marks it as wanted. Do this after # the search setup transaction has been released so slow indexer calls do # not hold a write transaction open. @@ -1057,13 +1089,10 @@ async def download_issue( source_priority=bundle.runtime.source_priority, ) - direct_outcome = bundle.outcome.direct_outcome - total_results = len(bundle.outcome.raw_results) + ( - len(direct_outcome.matched) + len(direct_outcome.rejected) if direct_outcome else 0 - ) search_log.results_grabbed = routed.grabbed search_log.results_queued = routed.queued - search_log.results_rejected = max(0, total_results - routed.grabbed - routed.queued) + search_log.results_rejected = bundle.outcome.results_rejected_count + search_log.best_confidence = routed.best_confidence details = dict(search_log.details or {}) details["action_status"] = routed.action_status if routed.notices: @@ -1074,7 +1103,7 @@ async def download_issue( if routed.grabbed: payload: dict[str, object] = { "issue_id": issue_id, - "status": "downloading", + "status": "retry_pending" if routed.action_status == "retry_pending" else "downloading", "release_title": routed.release_title or selection.release.title, "source_kind": routed.source_kind, } @@ -1082,6 +1111,10 @@ async def download_issue( payload["download_id"] = routed.download_id if routed.acquisition_id is not None: payload["acquisition_id"] = routed.acquisition_id + if routed.action_status == "retry_pending": + payload["message"] = ( + "AirDC++ queue confirmation is pending; Pullbox will reconcile it automatically." + ) return payload if routed.queued or routed.action_status == "pending_exists": @@ -1098,7 +1131,19 @@ async def download_issue( ), } - return {"issue_id": issue_id, "status": "no_results"} + messages = { + "source_unavailable": "Matches found, but downloads could not be queued.", + "already_downloading": "This issue already has an active download.", + "already_owned": ( + "This issue already has a library file; use Find Alternative to replace it." + ), + } + return { + "issue_id": issue_id, + "status": routed.action_status, + "message": messages.get(routed.action_status, "No eligible match was queued."), + "notices": list(routed.notices), + } @router.post( @@ -1131,7 +1176,7 @@ async def import_file_for_issue( issue=prepared.issue, confidence=MatchConfidence.MANUAL, move_to_library=True, - library_root_id=prepared.issue.series.library_root_id, + library_root_id=preferred_managed_root_id(prepared.issue.series), loaded_issue=prepared.issue, ingest_policy=prepared.ingest_policy, allow_resource_safety_exception=body.allow_resource_safety_exception, diff --git a/src/pullbox/api/v1/library.py b/src/pullbox/api/v1/library.py index 813f9cc7..71a1153b 100644 --- a/src/pullbox/api/v1/library.py +++ b/src/pullbox/api/v1/library.py @@ -22,7 +22,12 @@ from pullbox.core.exceptions import ValidationError from pullbox.core.library_root_resolution import resolve_path_inside_roots from pullbox.models.config import SystemConfig -from pullbox.models.library import LibraryFile, LibraryRoot, MatchConfidence +from pullbox.models.library import ( + LibraryFile, + LibraryFileStorageMode, + LibraryRoot, + MatchConfidence, +) from pullbox.models.series import Series from pullbox.schemas.library import ( LibraryBrowserActionFlags, @@ -189,6 +194,7 @@ def _build_library_actions( kind: str, file_format: str | None, tracking_scope: str = "tracked_file", + referenced_file_count: int = 0, ) -> LibraryBrowserActionFlags: normalized_format = (file_format or "").strip().lower() if kind == "root": @@ -204,6 +210,9 @@ def _build_library_actions( }: return LibraryBrowserActionFlags(can_properties=False) + if referenced_file_count: + return LibraryBrowserActionFlags(can_properties=True, can_delete=True) + return LibraryBrowserActionFlags( can_properties=True, can_rename=True, @@ -486,6 +495,15 @@ async def _validate_library_browser_convert( raise ValidationError("Only files can be converted from the browser.") if not await _tracked_library_file_exists(session, source): raise ValidationError("This path is not tracked by Pullbox's library catalog.") + storage_mode = ( + await session.execute( + select(LibraryFile.storage_mode).where(LibraryFile.file_path == str(source)) + ) + ).scalar_one_or_none() + if storage_mode == LibraryFileStorageMode.REFERENCED: + raise ValidationError( + "Referenced library files cannot be converted. They must stay unchanged on disk." + ) source_format = source.suffix.lower().lstrip(".") if source_format not in _CONVERTIBLE_FILE_FORMATS: @@ -583,6 +601,7 @@ async def library_browser_entry( kind=kind, file_format=file_format, tracking_scope=tracking_scope, + referenced_file_count=delete_context.referenced_file_count, ), delete_context=LibraryBrowserDeleteContext.model_validate(asdict(delete_context)), rename_context=rename_context, @@ -623,6 +642,10 @@ async def _validate_library_browser_rename( if kind == "root": raise ValidationError("Library roots cannot be renamed from the browser.") _require_mutable_library_catalog_entry(tracking_scope) + if delete_context.referenced_file_count: + raise ValidationError( + "Referenced library files cannot be renamed. They must stay unchanged on disk." + ) if rename_context.stale_reference: raise ValidationError( rename_context.message @@ -750,6 +773,8 @@ async def delete_library_browser_entry( "source_path": outcome.source_path, "deleted_via_trash": outcome.deleted_via_trash, "result_path": outcome.result_path, + "managed_files_deleted": outcome.managed_files_deleted, + "referenced_files_detached": outcome.referenced_files_detached, } diff --git a/src/pullbox/api/v1/reader.py b/src/pullbox/api/v1/reader.py index 07fb7366..9fe92f3e 100644 --- a/src/pullbox/api/v1/reader.py +++ b/src/pullbox/api/v1/reader.py @@ -15,6 +15,7 @@ ReaderWantToReadChanged, get_event_bus, ) +from pullbox.core.issue_numbers import format_issue_number from pullbox.core.page_sources import PageSourceError, PageSourceErrorCode, ReaderResourceLimits from pullbox.core.page_sources.capabilities import inspect_reader_capabilities from pullbox.schemas.reader import ( @@ -386,7 +387,7 @@ def _adjacent_response( issue_id = adjacent.issue_id return ReaderAdjacentIssueResponse( issue_id=issue_id, - issue_label=f"#{adjacent.issue_number:g}", + issue_label=f"#{format_issue_number(adjacent.issue_number)}", title=adjacent.title, manifest_url=f"/api/v1/reader/issues/{issue_id}/manifest", issue_detail_url=f"/issues/{issue_id}", diff --git a/src/pullbox/api/v1/router.py b/src/pullbox/api/v1/router.py index 3b60063d..4aae8f42 100644 --- a/src/pullbox/api/v1/router.py +++ b/src/pullbox/api/v1/router.py @@ -6,6 +6,7 @@ from pullbox.api.v1.audit import router as audit_router from pullbox.api.v1.auth import router as auth_router from pullbox.api.v1.blocklist import router as blocklist_router +from pullbox.api.v1.catalog import router as catalog_router from pullbox.api.v1.clients import router as clients_router from pullbox.api.v1.config import router as config_router from pullbox.api.v1.covers import router as covers_router @@ -15,7 +16,10 @@ from pullbox.api.v1.downloads import router as downloads_router from pullbox.api.v1.filesystem import router as filesystem_router from pullbox.api.v1.health import router as health_router +from pullbox.api.v1.import_completed_cleanup import router as import_completed_cleanup_router +from pullbox.api.v1.import_job_archive import router as import_job_archive_router from pullbox.api.v1.import_jobs import router as import_jobs_router +from pullbox.api.v1.import_safety_bulk import router as import_safety_bulk_router from pullbox.api.v1.indexers import router as indexers_router from pullbox.api.v1.intervention import router as intervention_router from pullbox.api.v1.issues import router as issues_router @@ -23,6 +27,8 @@ from pullbox.api.v1.reader import router as reader_router from pullbox.api.v1.search import router as search_router from pullbox.api.v1.series import router as series_router +from pullbox.api.v1.story_arc_placements import router as story_arc_placements_router +from pullbox.api.v1.story_arcs import router as story_arcs_router from pullbox.api.v1.suggestions import router as suggestions_router from pullbox.api.v1.system import router as system_router from pullbox.api.v1.whats_new import router as whats_new_router @@ -31,10 +37,13 @@ v1_router = APIRouter(prefix="/api/v1") v1_router.include_router(activity_router) +v1_router.include_router(catalog_router) v1_router.include_router(audit_router) v1_router.include_router(blocklist_router) v1_router.include_router(auth_router) v1_router.include_router(series_router) +v1_router.include_router(story_arcs_router) +v1_router.include_router(story_arc_placements_router) v1_router.include_router(issues_router) v1_router.include_router(library_router) v1_router.include_router(reader_router) @@ -50,6 +59,9 @@ v1_router.include_router(filesystem_router) v1_router.include_router(health_router) v1_router.include_router(import_jobs_router) +v1_router.include_router(import_completed_cleanup_router) +v1_router.include_router(import_job_archive_router) +v1_router.include_router(import_safety_bulk_router) v1_router.include_router(intervention_router) v1_router.include_router(suggestions_router) v1_router.include_router(system_router) diff --git a/src/pullbox/api/v1/series.py b/src/pullbox/api/v1/series.py index 6876f726..bc23e6ef 100644 --- a/src/pullbox/api/v1/series.py +++ b/src/pullbox/api/v1/series.py @@ -6,7 +6,7 @@ from typing import Any, Literal import structlog -from fastapi import APIRouter, Query, Request, Response +from fastapi import APIRouter, HTTPException, Query, Request, Response from sqlalchemy import case, func, inspect, select, update from sqlalchemy.orm import contains_eager @@ -16,7 +16,7 @@ from pullbox.core.library_policy import load_search_on_add_default from pullbox.models.issue import Issue, IssueStatus from pullbox.models.search_log import SearchLog, SearchType -from pullbox.models.series import Series, SeriesStatus +from pullbox.models.series import IssueCatalogState, Series, SeriesStatus from pullbox.schemas.issue import IssueListResponse from pullbox.schemas.pagination import PaginatedResponse from pullbox.schemas.series import ( @@ -110,6 +110,7 @@ def _enrich_series_list( "publisher_name": series.publisher.name if series.publisher else None, "path": series.path, "library_root_id": series.library_root_id, + "preferred_library_root_id": series.preferred_library_root_id, "owned_count": ( int(owned_count) if owned_count is not None @@ -335,6 +336,8 @@ async def build_series_delete_context( return SeriesDeleteContextResponse( series_count=context.series_count, linked_file_count=context.linked_file_count, + managed_file_count=context.managed_file_count, + referenced_file_count=context.referenced_file_count, ) @@ -452,6 +455,7 @@ async def list_series_issues( IssueListResponse.model_validate( { **{c.key: getattr(i, c.key) for c in issue_mapper.columns}, + "issue_number_text": i.effective_issue_number_text, "has_file": i.library_file is not None, } ) @@ -563,6 +567,14 @@ async def refresh_series( session: DbSession, ) -> SeriesResponse: """Refresh series metadata from ComicVine.""" + series = await session.get(Series, series_id) + if series is None: + raise NotFoundError("Series", series_id) + if series.issue_catalog_state == IssueCatalogState.HYDRATING: + raise HTTPException( + status_code=409, + detail="Initial metadata sync is already in progress.", + ) metadata_svc = await _build_metadata_service(session) await metadata_svc.refresh_series(session, series_id, force=True) return await _load_series_response(session, series_id) diff --git a/src/pullbox/api/v1/story_arc_placements.py b/src/pullbox/api/v1/story_arc_placements.py new file mode 100644 index 00000000..134f5722 --- /dev/null +++ b/src/pullbox/api/v1/story_arc_placements.py @@ -0,0 +1,404 @@ +"""Authenticated story-arc placement policy and synchronization routes.""" + +from __future__ import annotations + +from typing import Annotated, NoReturn + +from fastapi import APIRouter, HTTPException, Query + +from pullbox.api.deps import AuthenticatedUser, DbSession # noqa: TC001 +from pullbox.schemas.pagination import PaginatedResponse +from pullbox.schemas.story_arc_placement import ( + StoryArcPlacementPolicyPayload, + StoryArcPlacementPolicyResponse, + StoryArcPlacementPolicyUpdate, + StoryArcPlacementPreviewItemResponse, + StoryArcPlacementPreviewPageResponse, + StoryArcPlacementRemovalResponse, + StoryArcPlacementResponse, + StoryArcPlacementSyncRequest, + StoryArcPlacementSyncResponse, + StoryArcPolicyMigrationConfirmationRequest, + StoryArcPolicyMigrationConfirmationResponse, + StoryArcPolicyMigrationPreviewItemResponse, + StoryArcPolicyMigrationPreviewRequest, + StoryArcPolicyMigrationPreviewResponse, +) +from pullbox.services.story_arc_placement_integration import ( + StoryArcPlacementIntegrationError, + StoryArcPlacementPolicy, + StoryArcPlacementPolicyInput, + StoryArcPlacementRemovalView, + StoryArcPlacementSyncResult, + StoryArcPlacementSyncService, +) +from pullbox.services.story_arc_policy_migration import ( + StoryArcPolicyMigrationService, +) + +router = APIRouter(prefix="/story-arcs", tags=["story-arc-placements"]) + +_service = StoryArcPlacementSyncService() +_migration_service = StoryArcPolicyMigrationService() + + +def _proposal(body: StoryArcPlacementPolicyPayload) -> StoryArcPlacementPolicyInput: + return StoryArcPlacementPolicyInput( + mode=body.mode, + target_library_root_id=body.target_library_root_id, + destination_root=body.destination_root, + folder_template=body.folder_template, + file_template=body.file_template, + symlink_style=body.symlink_style, + synchronize=body.synchronize, + ) + + +def _policy_response(policy: StoryArcPlacementPolicy) -> StoryArcPlacementPolicyResponse: + return StoryArcPlacementPolicyResponse( + configured=policy.configured, + revision=policy.revision, + mode=policy.mode, + target_library_root_id=policy.target_library_root_id, + destination_root=policy.destination_root, + folder_template=policy.folder_template, + file_template=policy.file_template, + symlink_style=policy.symlink_style, + synchronize=policy.synchronize, + snapshot=policy.snapshot, + ) + + +def _sync_response(result: StoryArcPlacementSyncResult) -> StoryArcPlacementSyncResponse: + return StoryArcPlacementSyncResponse( + membership_id=result.membership_id, + outcome=result.outcome, + placement=( + StoryArcPlacementResponse.model_validate(result.placement) + if result.placement is not None + else None + ), + ) + + +def _removal_response( + result: StoryArcPlacementRemovalView, +) -> StoryArcPlacementRemovalResponse: + return StoryArcPlacementRemovalResponse( + placement_id=result.placement_id, + ownership=result.ownership, + artifact_removed=result.artifact_removed, + canonical_preserved=result.canonical_preserved, + referenced_artifact_preserved=result.referenced_artifact_preserved, + automatic_sync_disabled=result.automatic_sync_disabled, + ) + + +def _raise_integration_error(exc: StoryArcPlacementIntegrationError) -> NoReturn: + status_code = ( + 404 + if exc.category == "not_found" + else 409 + if exc.category in {"conflict", "collision", "ownership", "cancelled"} + else 422 + ) + raise HTTPException( + status_code=status_code, + detail={ + "code": exc.code, + "category": exc.category, + "message": str(exc), + }, + ) from exc + + +@router.get( + "/{story_arc_id}/placement-policy", + response_model=StoryArcPlacementPolicyResponse, +) +async def get_story_arc_placement_policy( + story_arc_id: int, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcPlacementPolicyResponse: + """Read the complete effective policy without touching the filesystem.""" + try: + return _policy_response(await _service.get_policy(session, story_arc_id)) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + + +@router.post( + "/{story_arc_id}/placement-policy/preview", + response_model=StoryArcPlacementPreviewPageResponse, +) +async def preview_story_arc_placement_policy( + story_arc_id: int, + body: StoryArcPlacementPolicyPayload, + _user: AuthenticatedUser, + session: DbSession, + limit: Annotated[int, Query(ge=1, le=200)] = 50, + offset: Annotated[int, Query(ge=0)] = 0, +) -> StoryArcPlacementPreviewPageResponse: + """Validate and preview a complete candidate policy without persisting it.""" + proposal = _proposal(body) + try: + policy = await _service.validate_policy(session, story_arc_id, proposal) + page = await _service.preview_arc( + session, + story_arc_id, + limit=limit, + offset=offset, + proposal=proposal, + ) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + return StoryArcPlacementPreviewPageResponse( + policy=_policy_response(policy), + items=[StoryArcPlacementPreviewItemResponse.model_validate(item) for item in page.items], + total=page.total, + limit=page.limit, + offset=page.offset, + has_more=page.has_more, + ) + + +@router.post( + "/{story_arc_id}/placement-policy/migration-preview", + response_model=StoryArcPolicyMigrationPreviewResponse, +) +async def preview_story_arc_policy_migration( + story_arc_id: int, + body: StoryArcPolicyMigrationPreviewRequest, + user: AuthenticatedUser, + session: DbSession, + limit: Annotated[int, Query(ge=1, le=100)] = 50, + after_placement_id: Annotated[int, Query(ge=0)] = 0, +) -> StoryArcPolicyMigrationPreviewResponse: + """Preview every policy consequence while returning one bounded path page.""" + try: + actor_id = int(user.id) + # Authentication may update API-key audit/upgrade fields. Persist that + # short transaction before the preview service begins rollback-only scans. + await session.commit() + preview = await _migration_service.preview_policy_change( + session, + story_arc_id, + actor_id=actor_id, + expected_revision=body.expected_revision, + proposal=_proposal(body), + limit=limit, + after_placement_id=after_placement_id, + ) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + return StoryArcPolicyMigrationPreviewResponse( + story_arc_id=preview.story_arc_id, + expected_revision=preview.expected_revision, + current_policy=_policy_response(preview.current_policy), + proposed_policy=_policy_response(preview.proposed_policy), + scope_digest=preview.scope_digest, + preview_token=preview.preview_token, + required_confirmation=preview.required_confirmation, + total_placement_count=preview.total_placement_count, + managed_migrate_count=preview.managed_migrate_count, + managed_remove_count=preview.managed_remove_count, + managed_unchanged_count=preview.managed_unchanged_count, + referenced_preserved_count=preview.referenced_preserved_count, + collision_count=preview.collision_count, + blocked_count=preview.blocked_count, + required_bytes=preview.required_bytes, + available_bytes=preview.available_bytes, + global_block_codes=list(preview.global_block_codes), + items=[ + StoryArcPolicyMigrationPreviewItemResponse.model_validate(item) + for item in preview.items + ], + limit=preview.limit, + after_placement_id=preview.after_placement_id, + next_cursor=preview.next_cursor, + has_more=preview.has_more, + requires_confirmation=preview.requires_confirmation, + execution_supported=preview.execution_supported, + filesystem_mutated=preview.filesystem_mutated, + ) + + +@router.post( + "/{story_arc_id}/placement-policy/migration-confirmation", + response_model=StoryArcPolicyMigrationConfirmationResponse, +) +async def prepare_story_arc_policy_migration_confirmation( + story_arc_id: int, + body: StoryArcPolicyMigrationConfirmationRequest, + user: AuthenticatedUser, + session: DbSession, +) -> StoryArcPolicyMigrationConfirmationResponse: + """Validate signed, exact actor intent without mutating policy or files.""" + try: + actor_id = int(user.id) + await session.commit() + confirmation = await _migration_service.prepare_confirmation( + session, + story_arc_id, + actor_id=actor_id, + expected_revision=body.expected_revision, + proposal=_proposal(body), + preview_token=body.preview_token, + confirmation=body.confirmation, + ) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + return StoryArcPolicyMigrationConfirmationResponse( + story_arc_id=confirmation.story_arc_id, + expected_revision=confirmation.expected_revision, + scope_digest=confirmation.scope_digest, + confirmed=confirmation.confirmed, + ready_for_execution=confirmation.ready_for_execution, + execution_supported=confirmation.execution_supported, + mutation_performed=confirmation.mutation_performed, + policy_update_block_code=confirmation.policy_update_block_code, + ) + + +@router.put( + "/{story_arc_id}/placement-policy", + response_model=StoryArcPlacementPolicyResponse, +) +async def update_story_arc_placement_policy( + story_arc_id: int, + body: StoryArcPlacementPolicyUpdate, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcPlacementPolicyResponse: + """Freeze one complete policy with the arc's optimistic revision token.""" + try: + policy = await _service.update_policy( + session, + story_arc_id, + expected_revision=body.expected_revision, + proposal=_proposal(body), + ) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + return _policy_response(policy) + + +@router.get( + "/{story_arc_id}/placements", + response_model=PaginatedResponse[StoryArcPlacementResponse], +) +async def list_story_arc_placements( + story_arc_id: int, + _user: AuthenticatedUser, + session: DbSession, + limit: Annotated[int, Query(ge=1, le=200)] = 50, + offset: Annotated[int, Query(ge=0)] = 0, +) -> PaginatedResponse[StoryArcPlacementResponse]: + """List bounded placement ownership and synchronization state.""" + try: + page = await _service.list_placements( + session, + story_arc_id, + limit=limit, + offset=offset, + ) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + return PaginatedResponse[StoryArcPlacementResponse]( + items=[StoryArcPlacementResponse.model_validate(item) for item in page.items], + total=page.total, + limit=page.limit, + offset=page.offset, + has_more=page.has_more, + ) + + +@router.post( + "/{story_arc_id}/memberships/{membership_id}/placement-sync", + response_model=StoryArcPlacementSyncResponse, +) +async def sync_story_arc_membership_placement( + story_arc_id: int, + membership_id: int, + body: StoryArcPlacementSyncRequest, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcPlacementSyncResponse: + """Synchronize one resolved membership after canonical registration.""" + try: + result = await _service.sync_membership( + session, + story_arc_id, + membership_id, + adopt_identical_existing=body.adopt_identical_existing, + ) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + return _sync_response(result) + + +@router.post( + "/{story_arc_id}/placements/{placement_id}/retry", + response_model=StoryArcPlacementSyncResponse, +) +async def retry_story_arc_placement( + story_arc_id: int, + placement_id: int, + body: StoryArcPlacementSyncRequest, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcPlacementSyncResponse: + """Retry one failed or interrupted placement idempotently.""" + try: + result = await _service.retry_placement( + session, + story_arc_id, + placement_id, + adopt_identical_existing=body.adopt_identical_existing, + ) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + return _sync_response(result) + + +@router.post( + "/{story_arc_id}/placements/{placement_id}/repair", + response_model=StoryArcPlacementSyncResponse, +) +async def repair_story_arc_placement( + story_arc_id: int, + placement_id: int, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcPlacementSyncResponse: + """Repair only a safely evidenced Pullbox-managed placement.""" + try: + result = await _service.repair_placement(session, story_arc_id, placement_id) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + return _sync_response(result) + + +@router.delete( + "/{story_arc_id}/placements/{placement_id}", + response_model=StoryArcPlacementRemovalResponse, +) +async def remove_story_arc_placement( + story_arc_id: int, + placement_id: int, + _user: AuthenticatedUser, + session: DbSession, + confirm_managed_artifact_removal: bool = False, +) -> StoryArcPlacementRemovalResponse: + """Remove managed artifacts safely or forget referenced evidence without unlinking it.""" + try: + result = await _service.remove_placement( + session, + story_arc_id, + placement_id, + confirm_managed_artifact_removal=confirm_managed_artifact_removal, + ) + except StoryArcPlacementIntegrationError as exc: + _raise_integration_error(exc) + return _removal_response(result) diff --git a/src/pullbox/api/v1/story_arcs.py b/src/pullbox/api/v1/story_arcs.py new file mode 100644 index 00000000..803d079c --- /dev/null +++ b/src/pullbox/api/v1/story_arcs.py @@ -0,0 +1,559 @@ +"""First-class logical story-arc API routes.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, NoReturn + +from fastapi import APIRouter, HTTPException, Query, Response, status +from sqlalchemy import case, func, select +from sqlalchemy.exc import IntegrityError + +from pullbox.api.deps import AuthenticatedUser, DbSession # noqa: TC001 +from pullbox.config import get_settings +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.schemas.pagination import PaginatedResponse +from pullbox.schemas.story_arc import ( + StoryArcCreate, + StoryArcMembershipCreate, + StoryArcMembershipOrderResponse, + StoryArcMembershipReorder, + StoryArcMembershipResolve, + StoryArcMembershipResponse, + StoryArcMembershipUpdate, + StoryArcResponse, + StoryArcUpdate, +) +from pullbox.services.story_arc_editing_policy import ( + StoryArcManualEditingDisabledError, + require_manual_arc_edit, +) +from pullbox.services.story_arc_file_defaults import load_story_arc_file_defaults +from pullbox.services.story_arc_placement_integration import ( + STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + StoryArcPlacementIntegrationError, + validate_story_arc_placement_policy_input, +) +from pullbox.services.story_arc_service import ( + StoryArcConflictError, + StoryArcNotFoundError, + StoryArcService, + StoryArcServiceError, + StoryArcValidationError, +) + +if TYPE_CHECKING: + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Subquery + +router = APIRouter(prefix="/story-arcs", tags=["story-arcs"]) + +_story_arc_service = StoryArcService() + + +def _membership_counts_subquery() -> Subquery: + """Aggregate story-arc membership states without hydrating collections.""" + return ( + select( + IssueStoryArc.story_arc_id.label("story_arc_id"), + func.count(IssueStoryArc.id).label("membership_count"), + func.sum( + case( + (IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, 1), + else_=0, + ) + ).label("resolved_count"), + func.sum( + case( + (IssueStoryArc.resolution_state == StoryArcResolutionState.MISSING, 1), + else_=0, + ) + ).label("missing_count"), + func.sum( + case( + (IssueStoryArc.resolution_state == StoryArcResolutionState.CONFLICT, 1), + else_=0, + ) + ).label("conflict_count"), + ) + .group_by(IssueStoryArc.story_arc_id) + .subquery() + ) + + +def _arc_response( + arc: StoryArc, + *, + membership_count: int = 0, + resolved_count: int = 0, + missing_count: int = 0, + conflict_count: int = 0, +) -> StoryArcResponse: + """Serialize one arc with already-computed state counts.""" + return StoryArcResponse.model_validate( + { + "id": arc.id, + "name": arc.name, + "normalized_name": arc.normalized_name, + "description": arc.description, + "comicvine_id": arc.comicvine_id, + "comicvine_url": arc.comicvine_url, + "cover_path": arc.cover_path, + "cover_url": arc.cover_url, + "source_kind": arc.source_kind, + "lifecycle": arc.lifecycle, + "monitored": arc.monitored, + "search_missing": arc.search_missing, + "include_upcoming": arc.include_upcoming, + "sync_enabled": arc.sync_enabled, + "target_library_root_id": arc.target_library_root_id, + "revision": arc.revision, + "membership_count": membership_count, + "resolved_count": resolved_count, + "missing_count": missing_count, + "conflict_count": conflict_count, + "created_at": arc.created_at, + "updated_at": arc.updated_at, + } + ) + + +async def _load_arc_response(session: DbSession, story_arc_id: int) -> StoryArcResponse: + """Load an arc and its aggregate counts or return a stable 404.""" + counts = _membership_counts_subquery() + row = ( + await session.execute( + select( + StoryArc, + func.coalesce(counts.c.membership_count, 0), + func.coalesce(counts.c.resolved_count, 0), + func.coalesce(counts.c.missing_count, 0), + func.coalesce(counts.c.conflict_count, 0), + ) + .outerjoin(counts, counts.c.story_arc_id == StoryArc.id) + .where(StoryArc.id == story_arc_id) + ) + ).one_or_none() + if row is None: + raise HTTPException(status_code=404, detail=f"Story arc {story_arc_id} was not found") + arc, membership_count, resolved_count, missing_count, conflict_count = row + return _arc_response( + arc, + membership_count=int(membership_count), + resolved_count=int(resolved_count), + missing_count=int(missing_count), + conflict_count=int(conflict_count), + ) + + +async def _require_arc(session: DbSession, story_arc_id: int) -> StoryArc: + """Return one canonical arc or a safe API 404.""" + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise HTTPException(status_code=404, detail=f"Story arc {story_arc_id} was not found") + return arc + + +async def _require_membership( + session: DbSession, + story_arc_id: int, + membership_id: int, +) -> IssueStoryArc: + """Protect nested routes from mutating a membership in another arc.""" + membership = await session.get(IssueStoryArc, membership_id) + if membership is None or membership.story_arc_id != story_arc_id: + raise HTTPException( + status_code=404, + detail=f"Story-arc membership {membership_id} was not found", + ) + return membership + + +def _raise_service_error(exc: StoryArcServiceError) -> NoReturn: + """Translate domain failures into deterministic REST responses.""" + if isinstance(exc, StoryArcManualEditingDisabledError): + raise HTTPException(status_code=403, detail=str(exc)) from exc + if isinstance(exc, StoryArcNotFoundError): + raise HTTPException(status_code=404, detail=str(exc)) from exc + if isinstance(exc, StoryArcConflictError): + raise HTTPException(status_code=409, detail=str(exc)) from exc + if isinstance(exc, StoryArcValidationError): + raise HTTPException(status_code=422, detail=str(exc)) from exc + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +def _raise_integrity_conflict(exc: IntegrityError) -> NoReturn: + """Hide backend constraint details while preserving conflict semantics.""" + raise HTTPException( + status_code=409, + detail="Story arc mutation conflicts with existing data", + ) from exc + + +def _escaped_contains_pattern(value: str) -> str: + """Treat user search text literally inside a portable LIKE expression.""" + escaped = value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + return f"%{escaped}%" + + +@router.get("", response_model=PaginatedResponse[StoryArcResponse]) +async def list_story_arcs( + _user: AuthenticatedUser, + session: DbSession, + limit: Annotated[int, Query(ge=1, le=200)] = 50, + offset: Annotated[int, Query(ge=0)] = 0, + q: Annotated[str | None, Query(max_length=500)] = None, + lifecycle: Annotated[StoryArcLifecycle | None, Query()] = None, + monitored: Annotated[bool | None, Query()] = None, + source_kind: Annotated[StoryArcSourceKind | None, Query()] = None, +) -> PaginatedResponse[StoryArcResponse]: + """List bounded story arcs with filters, literal search, and state counts.""" + filters: list[ColumnElement[bool]] = [] + if q is not None and (search_text := q.strip()): + filters.append(StoryArc.name.ilike(_escaped_contains_pattern(search_text), escape="\\")) + if lifecycle is not None: + filters.append(StoryArc.lifecycle == lifecycle) + if monitored is not None: + filters.append(StoryArc.monitored.is_(monitored)) + if source_kind is not None: + filters.append(StoryArc.source_kind == source_kind) + + total = int(await session.scalar(select(func.count(StoryArc.id)).where(*filters)) or 0) + counts = _membership_counts_subquery() + rows = ( + await session.execute( + select( + StoryArc, + func.coalesce(counts.c.membership_count, 0), + func.coalesce(counts.c.resolved_count, 0), + func.coalesce(counts.c.missing_count, 0), + func.coalesce(counts.c.conflict_count, 0), + ) + .outerjoin(counts, counts.c.story_arc_id == StoryArc.id) + .where(*filters) + .order_by(StoryArc.normalized_name.asc(), StoryArc.id.asc()) + .limit(limit) + .offset(offset) + ) + ).all() + items = [ + _arc_response( + arc, + membership_count=int(membership_count), + resolved_count=int(resolved_count), + missing_count=int(missing_count), + conflict_count=int(conflict_count), + ) + for arc, membership_count, resolved_count, missing_count, conflict_count in rows + ] + return PaginatedResponse[StoryArcResponse]( + items=items, + total=total, + limit=limit, + offset=offset, + has_more=(offset + limit) < total, + ) + + +@router.post("", response_model=StoryArcResponse, status_code=status.HTTP_201_CREATED) +async def create_story_arc( + body: StoryArcCreate, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcResponse: + """Create one Pullbox-owned arc with a snapshot of global file defaults.""" + if not get_settings().story_arc_manual_create_enabled: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") + try: + defaults = await load_story_arc_file_defaults(session) + policy = await validate_story_arc_placement_policy_input( + session, defaults.proposal(), revision=1 + ) + arc = await _story_arc_service.create( + session, + name=body.name, + description=body.description, + monitored=body.monitored, + search_missing=body.search_missing, + include_upcoming=body.include_upcoming, + sync_enabled=policy.synchronize, + source_kind=StoryArcSourceKind.PULLBOX, + ) + story_arc_id = arc.id + arc.target_library_root_id = policy.target_library_root_id + arc.policy_schema_version = STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION + arc.policy_snapshot = policy.snapshot + await session.commit() + except StoryArcPlacementIntegrationError as exc: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc) + ) from exc + except StoryArcServiceError as exc: + _raise_service_error(exc) + except IntegrityError as exc: + _raise_integrity_conflict(exc) + return await _load_arc_response(session, story_arc_id) + + +@router.get("/{story_arc_id}", response_model=StoryArcResponse) +async def get_story_arc( + story_arc_id: int, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcResponse: + """Read one logical story arc with membership counts.""" + return await _load_arc_response(session, story_arc_id) + + +@router.patch("/{story_arc_id}", response_model=StoryArcResponse) +async def update_story_arc( + story_arc_id: int, + body: StoryArcUpdate, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcResponse: + """Patch arc metadata using one optimistic revision token.""" + existing = await _require_arc(session, story_arc_id) + fields = body.model_fields_set + try: + if fields & {"name", "description"}: + require_manual_arc_edit(existing) + await _story_arc_service.update( + session, + story_arc_id, + expected_revision=body.expected_revision, + name=body.name if "name" in fields and body.name is not None else existing.name, + description=body.description if "description" in fields else existing.description, + monitored=body.monitored if body.monitored is not None else existing.monitored, + search_missing=( + body.search_missing if body.search_missing is not None else existing.search_missing + ), + include_upcoming=( + body.include_upcoming + if body.include_upcoming is not None + else existing.include_upcoming + ), + sync_enabled=( + body.sync_enabled if body.sync_enabled is not None else existing.sync_enabled + ), + ) + await session.commit() + except StoryArcServiceError as exc: + _raise_service_error(exc) + except IntegrityError as exc: + _raise_integrity_conflict(exc) + return await _load_arc_response(session, story_arc_id) + + +@router.delete("/{story_arc_id}", response_model=StoryArcResponse) +async def archive_story_arc( + story_arc_id: int, + _user: AuthenticatedUser, + session: DbSession, + expected_revision: Annotated[int, Query(ge=1)], +) -> StoryArcResponse: + """Safely archive an arc without deleting memberships or canonical issues.""" + try: + await _story_arc_service.archive( + session, + story_arc_id, + expected_revision=expected_revision, + ) + await session.commit() + except StoryArcServiceError as exc: + _raise_service_error(exc) + except IntegrityError as exc: + _raise_integrity_conflict(exc) + return await _load_arc_response(session, story_arc_id) + + +@router.get( + "/{story_arc_id}/memberships", + response_model=PaginatedResponse[StoryArcMembershipResponse], +) +async def list_story_arc_memberships( + story_arc_id: int, + _user: AuthenticatedUser, + session: DbSession, + limit: Annotated[int, Query(ge=1, le=200)] = 50, + offset: Annotated[int, Query(ge=0)] = 0, +) -> PaginatedResponse[StoryArcMembershipResponse]: + """List a bounded page in deterministic reading order.""" + await _require_arc(session, story_arc_id) + total = int( + await session.scalar( + select(func.count(IssueStoryArc.id)).where(IssueStoryArc.story_arc_id == story_arc_id) + ) + or 0 + ) + memberships = list( + ( + await session.scalars( + select(IssueStoryArc) + .where(IssueStoryArc.story_arc_id == story_arc_id) + .order_by( + IssueStoryArc.sequence_number.asc(), + IssueStoryArc.source_ordinal.asc(), + IssueStoryArc.id.asc(), + ) + .limit(limit) + .offset(offset) + ) + ).all() + ) + return PaginatedResponse[StoryArcMembershipResponse]( + items=[StoryArcMembershipResponse.model_validate(item) for item in memberships], + total=total, + limit=limit, + offset=offset, + has_more=(offset + limit) < total, + ) + + +@router.post( + "/{story_arc_id}/memberships", + response_model=StoryArcMembershipResponse, + status_code=status.HTTP_201_CREATED, +) +async def add_story_arc_membership( + story_arc_id: int, + body: StoryArcMembershipCreate, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcMembershipResponse: + """Add one resolved or unresolved membership and commit it.""" + try: + require_manual_arc_edit(await _require_arc(session, story_arc_id)) + membership = await _story_arc_service.add_membership( + session, + story_arc_id, + issue_id=body.issue_id, + sequence_number=body.sequence_number, + source_ordinal=body.source_ordinal, + source_issue_number_text=body.source_issue_number_text, + source_kind=StoryArcSourceKind.PULLBOX, + ) + await session.commit() + except StoryArcServiceError as exc: + _raise_service_error(exc) + except IntegrityError as exc: + _raise_integrity_conflict(exc) + return StoryArcMembershipResponse.model_validate(membership) + + +@router.put( + "/{story_arc_id}/memberships/reorder", + response_model=StoryArcMembershipOrderResponse, +) +async def reorder_story_arc_memberships( + story_arc_id: int, + body: StoryArcMembershipReorder, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcMembershipOrderResponse: + """Atomically apply one complete, duplicate-free membership order.""" + try: + memberships = await _story_arc_service.reorder_memberships( + session, + story_arc_id, + ordered_membership_ids=body.membership_ids, + expected_revision=body.expected_revision, + ) + arc = await _require_arc(session, story_arc_id) + revision = arc.revision + await session.commit() + except StoryArcServiceError as exc: + _raise_service_error(exc) + except IntegrityError as exc: + _raise_integrity_conflict(exc) + return StoryArcMembershipOrderResponse( + items=[StoryArcMembershipResponse.model_validate(item) for item in memberships], + revision=revision, + ) + + +@router.patch( + "/{story_arc_id}/memberships/{membership_id}", + response_model=StoryArcMembershipResponse, +) +async def update_story_arc_membership( + story_arc_id: int, + membership_id: int, + body: StoryArcMembershipUpdate, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcMembershipResponse: + """Patch order, exact source number, or intentional skip state.""" + await _require_membership(session, story_arc_id, membership_id) + try: + if "source_issue_number_text" in body.model_fields_set: + require_manual_arc_edit(await _require_arc(session, story_arc_id)) + membership = await _story_arc_service.update_membership( + session, + membership_id, + sequence_number=body.sequence_number, + source_ordinal=body.source_ordinal, + source_issue_number_text=body.source_issue_number_text, + intentionally_skipped=body.intentionally_skipped, + ) + await session.commit() + except StoryArcServiceError as exc: + _raise_service_error(exc) + except IntegrityError as exc: + _raise_integrity_conflict(exc) + return StoryArcMembershipResponse.model_validate(membership) + + +@router.post( + "/{story_arc_id}/memberships/{membership_id}/resolve", + response_model=StoryArcMembershipResponse, +) +async def resolve_story_arc_membership( + story_arc_id: int, + membership_id: int, + body: StoryArcMembershipResolve, + _user: AuthenticatedUser, + session: DbSession, +) -> StoryArcMembershipResponse: + """Resolve an entry only to an existing canonical issue.""" + await _require_membership(session, story_arc_id, membership_id) + try: + membership = await _story_arc_service.resolve_membership( + session, + membership_id, + issue_id=body.issue_id, + ) + await session.commit() + except StoryArcServiceError as exc: + _raise_service_error(exc) + except IntegrityError as exc: + _raise_integrity_conflict(exc) + return StoryArcMembershipResponse.model_validate(membership) + + +@router.delete( + "/{story_arc_id}/memberships/{membership_id}", + status_code=status.HTTP_204_NO_CONTENT, +) +async def remove_story_arc_membership( + story_arc_id: int, + membership_id: int, + _user: AuthenticatedUser, + session: DbSession, +) -> Response: + """Remove only the association, never its canonical issue.""" + await _require_membership(session, story_arc_id, membership_id) + try: + require_manual_arc_edit(await _require_arc(session, story_arc_id)) + await _story_arc_service.remove_membership(session, membership_id) + await session.commit() + except StoryArcServiceError as exc: + _raise_service_error(exc) + except IntegrityError as exc: + _raise_integrity_conflict(exc) + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/src/pullbox/app.py b/src/pullbox/app.py index 5832ee5e..877f58d5 100644 --- a/src/pullbox/app.py +++ b/src/pullbox/app.py @@ -287,8 +287,13 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]: async with factory() as session: reconciliation = await reconcile_runtime_library_paths(session, settings.library_root) if reconciliation: - await session.commit() - logger.info("library_paths_reconciled_at_startup", **reconciliation) + if reconciliation.get("rebind_required") is True: + logger.warning("library_path_rebind_required", **reconciliation) + elif reconciliation.get("status") == "root_unavailable": + logger.warning("runtime_library_root_unavailable", **reconciliation) + else: + await session.commit() + logger.info("runtime_library_root_bootstrapped", **reconciliation) except Exception: logger.warning("library_path_reconciliation_failed", exc_info=True) @@ -414,24 +419,14 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]: factory = get_session_factory() import_runner = ImportRunner(factory) set_import_runner(import_runner) - import_runner_task = asyncio.create_task(import_runner.recover_and_dispatch()) - _startup_background_tasks.add(import_runner_task) - - def _cleanup_import_runner_task(task: asyncio.Task[object]) -> None: - _startup_background_tasks.discard(task) - with suppress(asyncio.CancelledError): - exc = task.exception() - if exc is not None: - logger.warning("import_runner_startup_failed", exc_info=exc) - return - - recovered = task.result() - if isinstance(recovered, int) and recovered: - logger.info("import_jobs_recovered_at_startup", count=recovered) - - import_runner_task.add_done_callback(_cleanup_import_runner_task) - except Exception: + # Reconcile durable import/control state before the scheduler can claim + # Story Arc work. Dispatch itself remains asynchronous inside the runner. + recovered = await import_runner.recover_and_dispatch() + if recovered: + logger.info("import_jobs_recovered_at_startup", count=recovered) + except Exception as exc: logger.warning("import_recovery_failed", subsystem="import_recovery", exc_info=True) + raise RuntimeError("Import recovery failed before scheduler startup") from exc # Recover native direct-download attempts. Signed artifact URLs remain # ephemeral and are reconstructed by the runner only when work resumes. @@ -497,9 +492,24 @@ def _cleanup_direct_recovery_task(task: asyncio.Task[object]) -> None: async def _resume_deferred_import_metadata() -> tuple[int, int]: async with factory() as session: import_service = await build_import_service(session) - comicinfo_jobs = await import_service.recover_pending_comicinfo_enrichment(factory) - hydrated_series = await import_service.recover_pending_catalog_hydration(factory) - return comicinfo_jobs, hydrated_series + # Each lane owns its sessions and already shares provider rate limiting. + # A long archive rewrite queue must not block visible catalog hydration. + results = await asyncio.gather( + import_service.recover_pending_comicinfo_enrichment(factory), + import_service.recover_pending_catalog_hydration(factory), + return_exceptions=True, + ) + for lane, result in zip(("comicinfo", "catalog"), results, strict=True): + if isinstance(result, BaseException): + logger.warning( + "import_metadata_lane_recovery_failed", + lane=lane, + exc_info=result, + ) + return ( + results[0] if isinstance(results[0], int) else 0, + results[1] if isinstance(results[1], int) else 0, + ) import_metadata_recovery_task = asyncio.create_task(_resume_deferred_import_metadata()) _startup_background_tasks.add(import_metadata_recovery_task) @@ -798,6 +808,19 @@ async def _startup_update_check() -> None: exc_info=True, ) + async def _startup_catalog_check() -> None: + from pullbox.services.catalog.contract import CatalogError + from pullbox.services.catalog.service import get_catalog_service + + try: + await get_catalog_service().sync() + except CatalogError: + logger.warning("startup_catalog_check_failed") + + catalog_startup_task = asyncio.create_task(_startup_catalog_check()) + _startup_background_tasks.add(catalog_startup_task) + catalog_startup_task.add_done_callback(_startup_background_tasks.discard) + if settings.startup_update_check_enabled: startup_update_task = asyncio.create_task(_startup_update_check()) _startup_background_tasks.add(startup_update_task) diff --git a/src/pullbox/cli.py b/src/pullbox/cli.py index b5b75539..1169b727 100644 --- a/src/pullbox/cli.py +++ b/src/pullbox/cli.py @@ -12,7 +12,9 @@ import argparse import asyncio import getpass +import json import sys +from pathlib import Path from sqlalchemy import select from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine @@ -21,6 +23,8 @@ from pullbox.core.password_policy import validate_password from pullbox.models.user import User from pullbox.services.auth_service import AuthService +from pullbox.services.import_path_reconciliation import reconcile_saved_mylar_paths +from pullbox.services.import_review_recheck import prepare_import_recheck async def _reset_password(username: str, candidate_secret: str) -> None: @@ -77,6 +81,46 @@ def _read_password(*, password_stdin: bool) -> str: return secret +async def _recheck_import(args: argparse.Namespace) -> None: + """Run maintenance against a stopped app; default is a non-mutating preview.""" + engine = create_async_engine(get_settings().db_url) + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + async with factory() as session: + options = dict( + source_roots=[Path(root) for root in args.source_root], + series_ids=args.series_id, + apply=args.apply, + ) + if args.command == "reconcile-import-paths": + report = await reconcile_saved_mylar_paths(session, args.job, **options) + else: + report = await prepare_import_recheck( + session, + args.job, + **options, + accept_replaced_files=args.accept_replaced_files, + ) + if args.apply: + await session.commit() + else: + await session.rollback() + print(json.dumps({"applied": args.apply, **report}, sort_keys=True)) + if args.apply: + if report.get("series_prepared"): + print( + "Restart Pullbox to resume local matching of the saved review. " + "Sources were not modified." + ) + elif report.get("files_prepared"): + print( + "Restart Pullbox, open the completed import, and choose Retry failed. " + "Only rechecked failures will run again." + ) + finally: + await engine.dispose() + + def build_parser() -> argparse.ArgumentParser: """Build the Pullbox management CLI parser.""" parser = argparse.ArgumentParser( @@ -96,6 +140,55 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Read the new password from stdin instead of prompting", ) + recheck = subparsers.add_parser( + "recheck-import", help="Recheck saved import evidence while Pullbox is stopped" + ) + recheck.add_argument("--job", required=True, type=int, help="Saved REVIEW or COMPLETED job ID") + recheck.add_argument( + "--source-root", + required=True, + action="append", + help="Permitted container-visible source directory; repeat as needed", + ) + recheck.add_argument( + "--series-id", + type=int, + action="append", + help="Limit to these review series IDs; otherwise check automatic identity conflicts", + ) + recheck.add_argument( + "--offline", + required=True, + action="store_true", + help="Acknowledge the Pullbox app is stopped and its database is backed up", + ) + recheck.add_argument( + "--apply", action="store_true", help="Persist changes; omitted means dry run" + ) + recheck.add_argument( + "--accept-replaced-files", + action="store_true", + help=( + "Explicitly re-inspect changed or formerly missing files " + "rather than retain a source-changed block" + ), + ) + reconcile = subparsers.add_parser( + "reconcile-import-paths", + help="Reconcile proven stale Mylar paths in an offline saved review", + ) + reconcile.add_argument("--job", required=True, type=int) + reconcile.add_argument("--source-root", required=True, action="append") + reconcile.add_argument("--series-id", type=int, action="append") + reconcile.add_argument( + "--offline", + required=True, + action="store_true", + help="Acknowledge Pullbox is stopped and its database is backed up", + ) + reconcile.add_argument( + "--apply", action="store_true", help="Persist repairs; default is preview only" + ) return parser @@ -108,6 +201,8 @@ def main() -> None: if args.command == "reset-password": candidate_secret = _read_password(password_stdin=args.password_stdin) asyncio.run(_reset_password(args.user, candidate_secret)) + elif args.command in {"recheck-import", "reconcile-import-paths"}: + asyncio.run(_recheck_import(args)) if __name__ == "__main__": diff --git a/src/pullbox/composition/providers.py b/src/pullbox/composition/providers.py index 464dc651..169b0bbc 100644 --- a/src/pullbox/composition/providers.py +++ b/src/pullbox/composition/providers.py @@ -143,6 +143,8 @@ async def register_indexers( indexer_rankings=prowlarr_indexer_rankings, ) registry.register_indexer(_PROWLARR_AGGREGATE_CONFIG_ID, aggregate) + for config_id, _priority in prowlarr_indexer_rankings.values(): + registry.register_indexer_alias(config_id, _PROWLARR_AGGREGATE_CONFIG_ID) logger.debug( "prowlarr_aggregate_registered", indexer_count=len(prowlarr_torznab_ids), diff --git a/src/pullbox/composition/services.py b/src/pullbox/composition/services.py index 861ba04e..d37f5213 100644 --- a/src/pullbox/composition/services.py +++ b/src/pullbox/composition/services.py @@ -206,14 +206,18 @@ def _datanodes_login_failure(exc: Exception) -> ArtifactHostResolutionError: async def build_metadata_service(session: AsyncSession) -> MetadataService: """Construct a MetadataService using persisted ComicVine settings.""" + from pullbox.services.catalog.reader import get_catalog_reader + settings = get_settings() api_key = await get_comicvine_api_key(session) provider = ComicVineProvider(api_key=api_key) + provider = build_persistent_import_metadata_provider(session, provider) covers_dir = await resolve_covers_dir(session) return MetadataService( provider=provider, covers_dir=covers_dir, refresh_days=settings.metadata_refresh_days, + catalog=get_catalog_reader(), ) @@ -238,6 +242,8 @@ async def build_import_service( min_burst_limit: int | None = None, ) -> ImportService: """Construct an ImportService using persisted ComicVine settings.""" + from pullbox.services.catalog.reader import get_catalog_reader + settings = get_settings() api_key = await get_comicvine_api_key(session) persisted_rate_config = await session.get(SystemConfig, "comicvine_rate_limit_per_second") @@ -270,6 +276,7 @@ async def build_import_service( provider, covers_dir=await resolve_covers_dir(session), refresh_days=settings.metadata_refresh_days, + catalog=get_catalog_reader(), ) event_bus = build_scoped_event_bus() series_svc = SeriesService(metadata_svc, event_bus) diff --git a/src/pullbox/config.py b/src/pullbox/config.py index 1e006691..38fc8bc8 100644 --- a/src/pullbox/config.py +++ b/src/pullbox/config.py @@ -52,6 +52,8 @@ class PullboxSettings(BaseSettings): debug: bool = False startup_update_check_enabled: bool = True airdcpp_enabled: bool = False + story_arc_manual_create_enabled: bool = False + story_arc_manual_edit_enabled: bool = False # ── Database ─────────────────────────────────────────────────────── db_url: str = "sqlite+aiosqlite:////data/pullbox.db" @@ -109,6 +111,7 @@ class PullboxSettings(BaseSettings): import_debug_phase_delay_seconds: float = 1.25 import_debug_item_delay_seconds: float = 0.4 import_file_worker_count: int = 2 + import_scan_worker_count: int = Field(default=0, ge=0, le=16) # ── Scheduler ────────────────────────────────────────────────────── search_interval_hours: int = 6 diff --git a/src/pullbox/core/archive.py b/src/pullbox/core/archive.py index 901c45da..d0de1e79 100644 --- a/src/pullbox/core/archive.py +++ b/src/pullbox/core/archive.py @@ -24,6 +24,18 @@ _ARCHIVE_IMAGE_SUFFIXES = frozenset({".jpg", ".jpeg", ".png", ".webp", ".gif", ".tif", ".tiff"}) +def comicinfo_member_sort_key(name: str) -> tuple[int, int, str, str]: + """Prefer the canonical root ComicInfo.xml, then the shallowest nested copy.""" + normalized = name.replace("\\", "/") + member = PurePosixPath(normalized) + return ( + 0 if len(member.parts) == 1 else 1, + len(member.parts), + normalized.casefold(), + normalized, + ) + + class ArchiveError(Exception): """Raised when an archive cannot be read or is corrupt.""" @@ -124,19 +136,22 @@ def read_file(self, name: str, *, max_bytes: int | None = None) -> bytes: raise ArchiveError(f"Unsupported format: {self._extension}") - def read_comicinfo(self) -> ComicInfoData | None: + def read_comicinfo(self, *, entries: list[str] | None = None) -> ComicInfoData | None: """Extract and parse ComicInfo.xml from the archive. Returns ``None`` if the archive does not contain a ComicInfo.xml. """ - files = self.list_files() - - # ComicInfo.xml can be at root or in a subdirectory - comicinfo_name = None - for name in files: - if name.lower().endswith("comicinfo.xml"): - comicinfo_name = name - break + files = entries if entries is not None else self.list_files() + + comicinfo_names = sorted( + ( + name + for name in files + if PurePosixPath(name.replace("\\", "/")).name.casefold() == "comicinfo.xml" + ), + key=comicinfo_member_sort_key, + ) + comicinfo_name = comicinfo_names[0] if comicinfo_names else None if comicinfo_name is None: return None diff --git a/src/pullbox/core/collection_scan_grouping.py b/src/pullbox/core/collection_scan_grouping.py index 8989ee39..03478648 100644 --- a/src/pullbox/core/collection_scan_grouping.py +++ b/src/pullbox/core/collection_scan_grouping.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +from pathlib import Path from typing import TYPE_CHECKING from pullbox.core.type_semantics import TypeFamily, issue_type_family @@ -10,15 +11,17 @@ if TYPE_CHECKING: from collections.abc import Sequence - from pathlib import Path from typing import Protocol class DiscoveredFileLike(Protocol): + file_name: str + parsed_series: str | None parsed_issue_number: float | None comicvine_issue_id: int | None comicvine_series_id: int | None has_comicinfo: bool issue_type: IssueType + metadata_signals: dict[str, str] _LOW_SIGNAL_GROUPING_TOKENS: frozenset[str] = frozenset( @@ -39,6 +42,35 @@ class DiscoveredFileLike(Protocol): re.IGNORECASE, ) +_GENERIC_SERIES_CONTAINER_IDENTITIES: frozenset[str] = frozenset( + { + "books", + "collection", + "comics", + "downloads", + "imports", + "incoming", + "library", + "media", + "raw imports", + "staging", + } +) + +_EXACT_SERIES_METADATA_SIGNALS: frozenset[str] = frozenset( + {"comicinfo", "mylar3", "pullbox_folder", "sidecar", "source_layout"} +) + +_LEADING_ISSUE_TITLE_RE = re.compile( + r"^(?:issue|issues)\s*#?\s*\d+(?:\.\d+)?\b", + re.IGNORECASE, +) + +_EXPLICIT_ISSUE_MARKER_RE = re.compile( + r"\bissue\s*#?\s*\d+(?:\.\d+)?\b", + re.IGNORECASE, +) + _OBVIOUS_NON_COMIC_PDF_RE = re.compile( r"\b(?:" r"booking\s+confirmation|" @@ -88,6 +120,41 @@ def _should_collapse_to_folder_identity(parsed_series: str, folder_name: str) -> return bool(extras) and extras.issubset(_LOW_SIGNAL_GROUPING_TOKENS) +def _series_identity_token_sequence(value: str) -> tuple[str, ...]: + """Return ordered, non-numeric identity tokens for prefix comparisons.""" + normalized = _normalize_series_identity(_strip_year_volume_tokens(value)) + return tuple(token for token in normalized.split() if token and not token.isdigit()) + + +def _should_use_folder_identity_for_issue_title_file( + discovered_file: DiscoveredFileLike, + folder_name: str, +) -> bool: + """Recognize title-bearing issue filenames inside a real series folder.""" + if ( + discovered_file.parsed_issue_number is None + or not discovered_file.parsed_series + or discovered_file.comicvine_series_id is not None + or discovered_file.metadata_signals.get("series_name") in _EXACT_SERIES_METADATA_SIGNALS + ): + return False + + normalized_folder = _normalize_series_identity(folder_name) + if not normalized_folder or normalized_folder in _GENERIC_SERIES_CONTAINER_IDENTITIES: + return False + + file_stem = re.sub(r"[._-]+", " ", Path(discovered_file.file_name).stem) + file_stem = re.sub(r"\s+", " ", file_stem).strip() + if _LEADING_ISSUE_TITLE_RE.match(file_stem): + return True + if not _EXPLICIT_ISSUE_MARKER_RE.search(file_stem): + return False + + folder_tokens = _series_identity_token_sequence(folder_name) + file_tokens = _series_identity_token_sequence(file_stem) + return bool(folder_tokens) and file_tokens[: len(folder_tokens)] == folder_tokens + + def _is_low_signal_file_series_name(parsed_series: str) -> bool: """Return true when a parsed filename series looks like a generic placeholder.""" normalized = re.sub(r"[_-]+", " ", parsed_series.strip()) @@ -103,6 +170,7 @@ def _has_strong_file_identity(discovered_file: DiscoveredFileLike) -> bool: discovered_file.comicvine_issue_id is not None, discovered_file.comicvine_series_id is not None, discovered_file.has_comicinfo, + discovered_file.metadata_signals.get("series_name") == "source_layout", ) ) diff --git a/src/pullbox/core/collection_scanner.py b/src/pullbox/core/collection_scanner.py index bdb041e8..78414f2a 100644 --- a/src/pullbox/core/collection_scanner.py +++ b/src/pullbox/core/collection_scanner.py @@ -8,14 +8,21 @@ from __future__ import annotations import asyncio +import hashlib +import os import re +import sqlite3 +import tempfile +import threading import time +from contextlib import aclosing, closing, suppress from dataclasses import dataclass, field -from pathlib import Path -from typing import TYPE_CHECKING +from functools import partial +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Awaitable, Callable + from collections.abc import AsyncGenerator, Awaitable, Callable, Iterable import structlog @@ -26,10 +33,23 @@ _non_standard_group_discriminator, _normalize_series_identity, _should_collapse_to_folder_identity, + _should_use_folder_identity_for_issue_title_file, _source_issue_type_for_group, _strip_year_volume_tokens, _type_qualified_issue_like_label, ) +from pullbox.core.exceptions import ConfigurationError +from pullbox.core.issue_numbers import format_issue_number +from pullbox.core.library_file_ownership import build_file_identity_signature +from pullbox.core.library_layout import ( + ImportLayoutMode, + SourceLayoutMatch, + SourceLayoutSpec, + compile_source_layout, + resolve_source_layout_spec, +) +from pullbox.core.naming_type_detection import detect_issue_type +from pullbox.core.release_parser import normalize_issue_number from pullbox.core.source_metadata import MetadataSignal, SourceMetadata, SourceMetadataExtractor from pullbox.models.issue import IssueType @@ -37,13 +57,27 @@ SERIES_SCAN_WORKERS = 4 ARCHIVE_READ_CONCURRENCY = 8 +SCAN_ACTIVE_PATH_BUDGET = ARCHIVE_READ_CONCURRENCY * 2 +SCAN_BUCKET_PAGE_SIZE = SERIES_SCAN_WORKERS +SCAN_SQL_READ_PAGE_SIZE = 256 SCAN_PROGRESS_EMIT_INTERVAL_SECONDS = 0.25 +SCAN_CANCELLATION_POLL_INTERVAL_SECONDS = 0.05 + +_EXACT_METADATA_SIGNALS = frozenset( + { + MetadataSignal.COMICINFO, + MetadataSignal.MYLAR3, + MetadataSignal.PULLBOX_FOLDER, + MetadataSignal.SIDECAR, + } +) # Regex: folder name with a year token, optionally followed by release tags _FOLDER_YEAR_RE = re.compile(r"^(.+?)\s*[\[(](\d{4})[\])](?:\s*(?:\([^)]*\)|\[[^\]]*\]))*\s*$") _PULLBOX_FOLDER_CV_ID_RE = re.compile( - r"^(?P.+?)\s+\((?P(?:19|20)\d{2})\)\s+" - r"(?:(?P\d{4,})|\[cv-(?P\d{4,})\])\s*$" + r"^(?P.+?)\s+\((?P(?:\d{4}|Unknown Year))\)\s+" + r"(?:(?P\d{4,})|\[cv-(?P[1-9]\d*)\])\s*$", + re.IGNORECASE, ) # Comic file extensions (lowercase, with dot) @@ -70,6 +104,21 @@ ) +def _validate_spool_path_text(value: str) -> None: + """Fail closed when SQLite TEXT cannot losslessly represent a source path.""" + try: + value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + msg = "Scan source path cannot be represented safely in the inventory spool" + raise ConfigurationError(msg) from exc + + +def _directory_sort_key(value: str, *, windows: bool | None = None) -> str: + """Match legacy Path ordering on the active platform.""" + use_windows_order = os.name == "nt" if windows is None else windows + return value.casefold() if use_windows_order else value + + @dataclass class DiscoveredFile: """A single comic file discovered during a scan. @@ -94,6 +143,9 @@ class DiscoveredFile: issue_count_hint: int | None = None metadata_signals: dict[str, str] = field(default_factory=dict) metadata_diagnostics: dict[str, object] = field(default_factory=dict) + source_signature: dict[str, int | str] = field(default_factory=dict) + source_folder_cohort_key: str | None = None + source_ordinal: int | None = None @dataclass @@ -128,6 +180,465 @@ class ScanInventory: file_count: int +@dataclass(frozen=True, slots=True) +class _SpoolBuildResult: + directory_count: int + file_count: int + leaf_directory_count: int + active_path_high_water: int + spool_bytes: int + + +@dataclass(frozen=True, slots=True) +class _SpoolBucket: + scan_order: int + relative_directory: str + file_count: int + is_leaf: bool + + +class _ScanInventorySpool: + """Private disk-backed inventory containing root-relative paths only.""" + + def __init__(self, root: Path) -> None: + self._root = root + self._temporary_directory = tempfile.TemporaryDirectory(prefix="pullbox-scan-") + self._directory_path = Path(self._temporary_directory.name) + self._directory_path.chmod(0o700) + self._database_path = self._directory_path / "inventory.sqlite3" + + def close(self) -> None: + """Remove the private spool after every scan outcome.""" + self._temporary_directory.cleanup() + + def build( + self, + progress_callback: Callable[[int, int], None] | None, + cancellation_event: threading.Event, + extensions: frozenset[str], + ) -> _SpoolBuildResult: + """Walk once, stage relative paths, and freeze global source ordinals.""" + directory_count = 0 + file_count = 0 + active_path_high_water = 0 + last_emit_at = time.monotonic() + progress_handler_installed = False + + connection = sqlite3.connect(self._database_path) + try: + # The containing directory is already private, and the database itself + # is restricted before any inventory rows are written. + self._database_path.chmod(0o600) + connection.execute("PRAGMA journal_mode = OFF") + connection.execute("PRAGMA synchronous = OFF") + connection.execute("PRAGMA temp_store = FILE") + connection.execute("PRAGMA cache_size = -2048") + connection.executescript( + """ + CREATE TABLE buckets ( + relative_directory TEXT PRIMARY KEY, + sort_relative_directory TEXT NOT NULL, + file_count INTEGER NOT NULL, + is_leaf INTEGER NOT NULL + ); + CREATE TABLE staged_files ( + relative_directory TEXT NOT NULL, + relative_path TEXT PRIMARY KEY, + folded_relative_path TEXT NOT NULL + ); + """ + ) + + def on_error(exc: OSError) -> None: + logger.warning("import_scan_walk_error", path=str(self._root), error=str(exc)) + + for dirpath, dirnames, filenames in self._root.walk(on_error=on_error): + if cancellation_event.is_set(): + break + dirnames[:] = [ + name + for name in dirnames + if name not in IGNORE_DIRS and not name.startswith(".") + ] + dirnames.sort() + filenames.sort() + directory_count += 1 + + relative_directory_path = dirpath.relative_to(self._root) + relative_directory = ( + "" + if relative_directory_path == Path(".") + else relative_directory_path.as_posix() + ) + bucket_file_count = 0 + for index, filename in enumerate(filenames): + if index % 128 == 0 and cancellation_event.is_set(): + break + file_path = dirpath / filename + if not _is_scan_candidate_file(file_path, extensions): + continue + relative_path = file_path.relative_to(self._root).as_posix() + _validate_spool_path_text(relative_path) + connection.execute( + """ + INSERT INTO staged_files ( + relative_directory, + relative_path, + folded_relative_path + ) VALUES (?, ?, ?) + """, + (relative_directory, relative_path, relative_path.casefold()), + ) + bucket_file_count += 1 + file_count += 1 + + if cancellation_event.is_set(): + break + active_path_high_water = max(active_path_high_water, bucket_file_count) + if bucket_file_count: + _validate_spool_path_text(relative_directory) + connection.execute( + """ + INSERT INTO buckets ( + relative_directory, + sort_relative_directory, + file_count, + is_leaf + ) VALUES (?, ?, ?, ?) + """, + ( + relative_directory, + _directory_sort_key(relative_directory), + bucket_file_count, + 1, + ), + ) + ancestor = dirpath.parent + while True: + try: + relative_ancestor_path = ancestor.relative_to(self._root) + except ValueError: + break + relative_ancestor = ( + "" + if relative_ancestor_path == Path(".") + else relative_ancestor_path.as_posix() + ) + connection.execute( + "UPDATE buckets SET is_leaf = ? WHERE relative_directory = ?", + (0, relative_ancestor), + ) + if ancestor == self._root: + break + ancestor = ancestor.parent + + if progress_callback is not None: + now = time.monotonic() + if ( + directory_count == 1 + or (now - last_emit_at) >= SCAN_PROGRESS_EMIT_INTERVAL_SECONDS + ): + progress_callback(directory_count, file_count) + last_emit_at = now + + if progress_callback is not None: + progress_callback(directory_count, file_count) + + leaf_directory_count = 0 + if cancellation_event.is_set(): + connection.rollback() + else: + connection.commit() + connection.set_progress_handler( + lambda: 1 if cancellation_event.is_set() else 0, + 1_000, + ) + progress_handler_installed = True + connection.executescript( + """ + CREATE TABLE inventory_files ( + relative_directory TEXT NOT NULL, + relative_path TEXT PRIMARY KEY, + source_ordinal INTEGER NOT NULL + ); + INSERT INTO inventory_files ( + relative_directory, + relative_path, + source_ordinal + ) + SELECT + relative_directory, + relative_path, + ROW_NUMBER() OVER ( + ORDER BY folded_relative_path, relative_path + ) + FROM staged_files; + CREATE INDEX inventory_files_by_bucket + ON inventory_files (relative_directory, relative_path); + + CREATE TABLE ordered_buckets ( + scan_order INTEGER PRIMARY KEY, + relative_directory TEXT NOT NULL UNIQUE, + file_count INTEGER NOT NULL, + is_leaf INTEGER NOT NULL + ); + INSERT INTO ordered_buckets ( + scan_order, + relative_directory, + file_count, + is_leaf + ) + SELECT + ROW_NUMBER() OVER ( + ORDER BY CASE WHEN is_leaf = 1 THEN 0 ELSE 1 END, + sort_relative_directory, + relative_directory + ), + relative_directory, + file_count, + is_leaf + FROM buckets; + + DROP TABLE staged_files; + DROP TABLE buckets; + """ + ) + row = connection.execute( + "SELECT COUNT(*) FROM ordered_buckets WHERE is_leaf = ?", + (1,), + ).fetchone() + leaf_directory_count = int(row[0]) if row is not None else 0 + connection.commit() + finally: + try: + if progress_handler_installed: + connection.set_progress_handler(None, 0) + finally: + connection.close() + + spool_bytes = self._database_path.stat().st_size + return _SpoolBuildResult( + directory_count=directory_count, + file_count=file_count, + leaf_directory_count=leaf_directory_count, + active_path_high_water=active_path_high_water, + spool_bytes=spool_bytes, + ) + + def load_bucket_page( + self, + *, + after_order: int, + limit: int, + cancellation_event: threading.Event | None = None, + ) -> list[_SpoolBucket]: + """Load one bounded page of bucket descriptors in legacy scan order.""" + with closing(sqlite3.connect(self._database_path)) as connection: + if cancellation_event is not None: + connection.set_progress_handler( + lambda: 1 if cancellation_event.is_set() else 0, + 1_000, + ) + try: + rows = connection.execute( + """ + SELECT scan_order, relative_directory, file_count, is_leaf + FROM ordered_buckets + WHERE scan_order > ? + ORDER BY scan_order + LIMIT ? + """, + (after_order, limit), + ).fetchall() + finally: + if cancellation_event is not None: + connection.set_progress_handler(None, 0) + return [ + _SpoolBucket( + scan_order=int(row[0]), + relative_directory=str(row[1]), + file_count=int(row[2]), + is_leaf=bool(row[3]), + ) + for row in rows + ] + + def load_bucket_files( + self, + relative_directory: str, + cancellation_event: threading.Event | None = None, + ) -> list[tuple[Path, int]]: + """Materialize only one active bucket from root-relative spool rows.""" + materialized: list[tuple[Path, int]] = [] + with closing(sqlite3.connect(self._database_path)) as connection: + if cancellation_event is not None: + connection.set_progress_handler( + lambda: 1 if cancellation_event.is_set() else 0, + 1_000, + ) + try: + cursor = connection.execute( + """ + SELECT relative_path, source_ordinal + FROM inventory_files + WHERE relative_directory = ? + ORDER BY relative_path + """, + (relative_directory,), + ) + while True: + if cancellation_event is not None and cancellation_event.is_set(): + msg = "Inventory spool read interrupted" + raise sqlite3.OperationalError(msg) + page = cast( + "list[tuple[str, int]]", + cursor.fetchmany(SCAN_SQL_READ_PAGE_SIZE), + ) + if not page: + break + for relative_path_value, source_ordinal in page: + relative_path = PurePosixPath(relative_path_value) + if relative_path.is_absolute() or ".." in relative_path.parts: + msg = "Inventory spool contained a non-relative source path" + raise ValueError(msg) + materialized.append( + ( + self._root.joinpath(*relative_path.parts), + source_ordinal, + ) + ) + finally: + if cancellation_event is not None: + connection.set_progress_handler(None, 0) + return materialized + + +class _CancellationBridge: + """Poll cancellation inline while the scanner owns the current ``anext`` call.""" + + def __init__(self, cancellation_check: Callable[[], Awaitable[None]] | None) -> None: + self.event = threading.Event() + self._cancellation_check = cancellation_check + self._check_lock = asyncio.Lock() + self._last_check_at = 0.0 + + async def checkpoint(self) -> None: + """Run the callback serially on the consumer's current scanner turn.""" + if self._cancellation_check is None or self.event.is_set(): + return + async with self._check_lock: + if self.event.is_set(): + return + now = asyncio.get_running_loop().time() + if now - self._last_check_at < SCAN_CANCELLATION_POLL_INTERVAL_SECONDS: + return + self._last_check_at = now + try: + await self._cancellation_check() + except BaseException: + self.event.set() + raise + + async def run_blocking[BlockingResult]( + self, + call: Callable[[], BlockingResult], + ) -> BlockingResult: + """Run thread work while prioritizing cooperative callback failures.""" + blocking_task = asyncio.create_task(asyncio.to_thread(call)) + try: + if self._cancellation_check is None: + return await asyncio.shield(blocking_task) + while not blocking_task.done(): + done, _pending = await asyncio.wait( + {blocking_task}, + timeout=SCAN_CANCELLATION_POLL_INTERVAL_SECONDS, + ) + if done: + break + await self.checkpoint() + return blocking_task.result() + except asyncio.CancelledError: + self.event.set() + await self._drain_blocking_task(blocking_task) + raise + except BaseException: + self.event.set() + await self._drain_blocking_task(blocking_task) + raise + + @staticmethod + async def _drain_blocking_task(blocking_task: asyncio.Task[object]) -> None: + """Retrieve a worker outcome despite any repeated caller cancellation.""" + while not blocking_task.done(): + try: + await asyncio.shield(blocking_task) + except asyncio.CancelledError: + continue + except BaseException: + break + with suppress(BaseException): + blocking_task.result() + + +class _CoalescingProgressMailbox: + """Deliver only the latest pending progress snapshot in a bounded queue.""" + + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + self._loop = loop + self.queue: asyncio.Queue[tuple[int, int] | None] = asyncio.Queue(maxsize=1) + + def publish(self, directory_count: int, file_count: int) -> None: + """Coalesce a thread-produced snapshot on the owning event loop.""" + self._loop.call_soon_threadsafe( + self._offer, + (directory_count, file_count), + ) + + def _offer(self, counts: tuple[int, int]) -> None: + if self.queue.full(): + with suppress(asyncio.QueueEmpty): + self.queue.get_nowait() + self.queue.put_nowait(counts) + + async def finish(self, drain_task: asyncio.Task[None]) -> None: + """Preserve the last snapshot, then stop and retrieve the drain task.""" + sentinel_task: asyncio.Task[None] | None = None + try: + # All publishers have stopped before finish is called. Yield once so + # scheduled thread-safe callbacks run before the sentinel is added. + await asyncio.sleep(0) + if drain_task.done(): + await drain_task + return + + sentinel_task = asyncio.create_task(self.queue.put(None)) + done, _pending = await asyncio.wait( + {sentinel_task, drain_task}, + return_when=asyncio.FIRST_COMPLETED, + ) + if drain_task in done and not sentinel_task.done(): + sentinel_task.cancel() + await asyncio.gather(sentinel_task, return_exceptions=True) + else: + await sentinel_task + await drain_task + except BaseException: + cleanup_tasks = [drain_task] + if sentinel_task is not None: + cleanup_tasks.append(sentinel_task) + for task in cleanup_tasks: + if not task.done(): + task.cancel() + cleanup = asyncio.gather(*cleanup_tasks, return_exceptions=True) + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + continue + cleanup.result() + raise + + class CollectionScanner: """Walks a directory tree and extracts series candidates. @@ -147,14 +658,42 @@ def __init__( progress_callback: Callable[[int, int], Awaitable[None]] | None = None, file_progress_callback: Callable[[int], Awaitable[None]] | None = None, inventory_progress_callback: Callable[[int, int], Awaitable[None]] | None = None, + cancellation_check: Callable[[], Awaitable[None]] | None = None, extensions: frozenset[str] | None = None, + source_layout: SourceLayoutSpec | None = None, ) -> None: self._min_file_count = min_file_count self._max_sample_paths = max_sample_paths self._progress_callback = progress_callback self._file_progress_callback = file_progress_callback self._inventory_progress_callback = inventory_progress_callback + self._cancellation_check = cancellation_check self._extensions = extensions or COMIC_EXTENSIONS + self._source_layout = resolve_source_layout_spec(source_layout or SourceLayoutSpec()) + self._compiled_source_layout = ( + None + if self._source_layout.mode == ImportLayoutMode.AUTO + else compile_source_layout(self._source_layout) + ) + self._retained_inventory_path_high_water = 0 + self._active_file_task_count = 0 + self._active_file_task_high_water = 0 + self._inventory_spool_bytes = 0 + + @property + def retained_inventory_path_high_water(self) -> int: + """Largest in-memory filesystem inventory retained by this scanner.""" + return self._retained_inventory_path_high_water + + @property + def active_file_task_high_water(self) -> int: + """Largest number of per-file materialization tasks active at once.""" + return self._active_file_task_high_water + + @property + def inventory_spool_bytes(self) -> int: + """Size of the last folder scan's private inventory spool.""" + return self._inventory_spool_bytes async def inventory(self, root_path: str | Path) -> ScanInventory: """Count candidate directories and comic files before the main scan.""" @@ -163,109 +702,204 @@ async def inventory(self, root_path: str | Path) -> ScanInventory: msg = f"Scan root is not a directory: {root}" raise ValueError(msg) - if self._inventory_progress_callback is None: - directory_count, file_count = await asyncio.to_thread(self._inventory_tree, root) + cancellation_bridge = _CancellationBridge(self._cancellation_check) + cancellation_event = cancellation_bridge.event + progress_callback = self._inventory_progress_callback + if progress_callback is None: + directory_count, file_count = await cancellation_bridge.run_blocking( + partial( + self._inventory_tree, + root, + None, + cancellation_event, + ) + ) return ScanInventory(directory_count=directory_count, file_count=file_count) - loop = asyncio.get_running_loop() - progress_queue: asyncio.Queue[tuple[int, int] | None] = asyncio.Queue() + mailbox = _CoalescingProgressMailbox(asyncio.get_running_loop()) async def drain_inventory_progress() -> None: - last_emitted: tuple[int, int] | None = None - while True: - counts = await progress_queue.get() - if counts is None: - break - if counts is not None and counts != last_emitted: - last_emitted = counts - progress_callback = self._inventory_progress_callback - if progress_callback is not None: + try: + last_emitted: tuple[int, int] | None = None + while True: + counts = await mailbox.queue.get() + if counts is None: + break + if counts != last_emitted: + last_emitted = counts await progress_callback(*counts) - - def report_inventory_progress(directory_count: int, file_count: int) -> None: - loop.call_soon_threadsafe( - progress_queue.put_nowait, - (directory_count, file_count), - ) + except BaseException: + cancellation_event.set() + raise drain_task = asyncio.create_task(drain_inventory_progress()) try: - directory_count, file_count = await asyncio.to_thread( - self._inventory_tree, - root, - report_inventory_progress, + directory_count, file_count = await cancellation_bridge.run_blocking( + partial( + self._inventory_tree, + root, + mailbox.publish, + cancellation_event, + ) ) finally: - loop.call_soon_threadsafe(progress_queue.put_nowait, None) - await drain_task + await mailbox.finish(drain_task) return ScanInventory(directory_count=directory_count, file_count=file_count) async def scan(self, root_path: str | Path) -> AsyncGenerator[DiscoveredSeries, None]: - """Async generator yielding DiscoveredSeries as they are found. - - Strategy: - 1. Walk the tree collecting comic files per directory. - 2. Identify series directories — leaf dirs containing comic files. - 3. Extract raw_series_name, raw_year, raw_publisher from folder names. - 4. Build DiscoveredFile objects for each comic file (with filename - parsing and ComicInfo extraction). - 5. Yield a DiscoveredSeries for each identified series directory. + """Yield series from a disk inventory with bounded ordinary bucket work. + + A single oversized directory is processed alone, but its paths and + grouped results still materialize together until chunked grouping lands. """ root = Path(root_path).resolve() if not root.is_dir(): msg = f"Scan root is not a directory: {root}" raise ValueError(msg) + self._retained_inventory_path_high_water = 0 + self._active_file_task_count = 0 + self._active_file_task_high_water = 0 + self._inventory_spool_bytes = 0 + log = logger.bind(scan_root=str(root)) log.info("import_scan_started") + cancellation_bridge = _CancellationBridge(self._cancellation_check) + spool = _ScanInventorySpool(root) + try: + build_result = await self._build_spooled_inventory(spool, cancellation_bridge) + self._retained_inventory_path_high_water = max( + self._retained_inventory_path_high_water, + build_result.active_path_high_water, + ) + self._inventory_spool_bytes = build_result.spool_bytes - walk_progress_queue: asyncio.Queue[tuple[int, int] | None] = asyncio.Queue() - loop = asyncio.get_running_loop() + loose_series_count = 0 + async with aclosing( + self._iter_spooled_bucket_results( + spool=spool, + root=root, + cancellation_bridge=cancellation_bridge, + ) + ) as bucket_results: + async for candidates in bucket_results: + for candidate in candidates: + if len(candidate.files) < self._min_file_count: + continue + if ( + candidate.source_folder_relative == "(root)" + or candidate.source_folder == str(root) + ): + loose_series_count += 1 + yield candidate + + total_series = build_result.leaf_directory_count + loose_series_count + log.info( + "import_scan_completed", + series_count=total_series, + files_scanned=build_result.file_count, + dirs_scanned=build_result.directory_count, + loose_series=loose_series_count, + inventory_spool_bytes=self._inventory_spool_bytes, + retained_inventory_path_high_water=self._retained_inventory_path_high_water, + active_file_task_high_water=self._active_file_task_high_water, + ) + finally: + spool.close() - async def drain_walk_progress() -> None: - last_emitted: tuple[int, int] | None = None - while True: - counts = await walk_progress_queue.get() - if counts is None: - break - if counts is not None and counts != last_emitted: - last_emitted = counts - file_count, dir_count = counts[1], counts[0] - if self._progress_callback: - await self._progress_callback(file_count, dir_count) - if self._inventory_progress_callback: - await self._inventory_progress_callback(dir_count, file_count) - - def report_walk_progress(directory_count: int, file_count: int) -> None: - loop.call_soon_threadsafe( - walk_progress_queue.put_nowait, - (directory_count, file_count), + async def _build_spooled_inventory( + self, + spool: _ScanInventorySpool, + cancellation_bridge: _CancellationBridge, + ) -> _SpoolBuildResult: + """Build the disk inventory while preserving live progress and cancellation.""" + cancellation_event = cancellation_bridge.event + if self._progress_callback is None and self._inventory_progress_callback is None: + return await cancellation_bridge.run_blocking( + lambda: spool.build( + None, + cancellation_event, + self._extensions, + ) ) - walk_progress_task = asyncio.create_task(drain_walk_progress()) + mailbox = _CoalescingProgressMailbox(asyncio.get_running_loop()) + + async def drain_progress() -> None: + try: + last_emitted: tuple[int, int] | None = None + while True: + counts = await mailbox.queue.get() + if counts is None: + break + if counts != last_emitted: + last_emitted = counts + directory_count, file_count = counts + if self._progress_callback is not None: + await self._progress_callback(file_count, directory_count) + if self._inventory_progress_callback is not None: + await self._inventory_progress_callback(directory_count, file_count) + except BaseException: + cancellation_event.set() + raise + + progress_task = asyncio.create_task(drain_progress()) try: - # Collect all directories and their comic files in one filesystem pass. - dir_files, dirs_scanned, files_scanned = await asyncio.to_thread( - self._walk_tree_with_counts, - root, - report_walk_progress, + return await cancellation_bridge.run_blocking( + lambda: spool.build( + mailbox.publish, + cancellation_event, + self._extensions, + ) ) finally: - loop.call_soon_threadsafe(walk_progress_queue.put_nowait, None) - await walk_progress_task - - # Identify series directories (leaf dirs with comics, not parents of other series dirs). - series_dirs = self._identify_series_dirs(dir_files) + await mailbox.finish(progress_task) - # Limit concurrent archive reads and per-series materialization work. + async def _iter_spooled_bucket_results( + self, + *, + spool: _ScanInventorySpool, + root: Path, + cancellation_bridge: _CancellationBridge, + ) -> AsyncGenerator[list[DiscoveredSeries], None]: + """Bound ordinary bucket work; process an oversized bucket alone.""" comicinfo_sem = asyncio.Semaphore(ARCHIVE_READ_CONCURRENCY) + file_task_slots = asyncio.Semaphore(ARCHIVE_READ_CONCURRENCY) series_worker_sem = asyncio.Semaphore(SERIES_SCAN_WORKERS) + pending: dict[asyncio.Task[list[DiscoveredSeries]], int] = {} + active_path_count = 0 + after_order = 0 + page: list[_SpoolBucket] = [] + page_index = 0 + next_bucket: _SpoolBucket | None = None + exhausted = False + + async def load_next_bucket() -> _SpoolBucket | None: + nonlocal after_order, page, page_index + if page_index >= len(page): + await cancellation_bridge.checkpoint() + page = await cancellation_bridge.run_blocking( + partial( + spool.load_bucket_page, + after_order=after_order, + limit=SCAN_BUCKET_PAGE_SIZE, + cancellation_event=cancellation_bridge.event, + ) + ) + page_index = 0 + if not page: + return None + after_order = page[-1].scan_order + bucket = page[page_index] + page_index += 1 + return bucket async def process_bucket( + *, series_dir: Path, comic_files: list[Path], - *, + source_ordinal_by_path: dict[str, int], folder_name: str, folder_year: int | None, folder_publisher: str | None, @@ -276,9 +910,18 @@ async def process_bucket( discovered_files = await self._build_discovered_files( comic_files, comicinfo_sem, + file_task_slots=file_task_slots, + perform_cancellation_checks=False, + root=root, folder_publisher=folder_publisher, allow_weak_file_identity=allow_weak_file_identity, ) + self._stamp_source_folder_cohort( + discovered_files, + source_dir=series_dir, + root=root, + source_ordinal_by_path=source_ordinal_by_path, + ) return self._build_series_candidates( source_dir=series_dir, root=root, @@ -290,68 +933,111 @@ async def process_bucket( allow_weak_file_identity=allow_weak_file_identity, ) - tasks: list[asyncio.Task[list[DiscoveredSeries]]] = [] - for series_dir in sorted(series_dirs): - comic_files = dir_files[series_dir] - if len(comic_files) < self._min_file_count: - continue - - name, year, folder_cv_id = self._extract_folder_identity(series_dir.name) - publisher = self._infer_publisher_from_hierarchy(series_dir, root) - tasks.append( - asyncio.create_task( - process_bucket( - series_dir, - comic_files, - folder_name=name, - folder_year=year, - folder_publisher=publisher, - folder_cv_id=folder_cv_id, - allow_weak_file_identity=series_dir == root, + try: + while pending or not exhausted: + while len(pending) < SERIES_SCAN_WORKERS and not exhausted: + if next_bucket is None: + next_bucket = await load_next_bucket() + if next_bucket is None: + exhausted = True + break + + bucket = next_bucket + if bucket.is_leaf and bucket.file_count < self._min_file_count: + next_bucket = None + continue + + oversized = bucket.file_count > SCAN_ACTIVE_PATH_BUDGET + if oversized and pending: + break + if ( + not oversized + and active_path_count + bucket.file_count > SCAN_ACTIVE_PATH_BUDGET + ): + break + + next_bucket = None + relative_directory_path = PurePosixPath(bucket.relative_directory) + if ( + relative_directory_path.is_absolute() + or ".." in relative_directory_path.parts + ): + msg = "Inventory spool contained a non-relative source directory" + raise ValueError(msg) + series_dir = root.joinpath(*relative_directory_path.parts) + bucket_files = await cancellation_bridge.run_blocking( + partial( + spool.load_bucket_files, + bucket.relative_directory, + cancellation_bridge.event, + ) ) - ) - ) - - # Handle loose files in non-leaf directories (e.g. root folder). - loose_dirs = set(dir_files.keys()) - set(series_dirs) - loose_series_count = 0 - for loose_dir in sorted(loose_dirs): - loose_files = dir_files[loose_dir] - tasks.append( - asyncio.create_task( - process_bucket( - loose_dir, - loose_files, - folder_name=loose_dir.name if loose_dir.name else "(root)", - folder_year=None, - folder_publisher=None, - folder_cv_id=None, - allow_weak_file_identity=loose_dir == root, + if len(bucket_files) != bucket.file_count: + msg = "Inventory spool bucket count changed during materialization" + raise ValueError(msg) + comic_files = [path for path, _ordinal in bucket_files] + source_ordinal_by_path = {str(path): ordinal for path, ordinal in bucket_files} + + if bucket.is_leaf: + folder_name, folder_year, folder_cv_id = self._extract_folder_identity( + series_dir.name + ) + folder_publisher = self._infer_publisher_from_hierarchy(series_dir, root) + else: + folder_name = series_dir.name if series_dir.name else "(root)" + folder_year = None + folder_cv_id = None + folder_publisher = None + + task = asyncio.create_task( + process_bucket( + series_dir=series_dir, + comic_files=comic_files, + source_ordinal_by_path=source_ordinal_by_path, + folder_name=folder_name, + folder_year=folder_year, + folder_publisher=folder_publisher, + folder_cv_id=folder_cv_id, + allow_weak_file_identity=series_dir == root, + ) ) - ) - ) + pending[task] = bucket.file_count + active_path_count += bucket.file_count + self._retained_inventory_path_high_water = max( + self._retained_inventory_path_high_water, + active_path_count, + ) + if oversized: + break - for task in asyncio.as_completed(tasks): - candidates = await task - for candidate in candidates: - if len(candidate.files) < self._min_file_count: + if not pending: continue - if candidate.source_folder_relative == "(root)" or candidate.source_folder == str( - root - ): - loose_series_count += 1 - yield candidate - total_series = len(series_dirs) + loose_series_count - log.info( - "import_scan_completed", - series_count=total_series, - files_scanned=files_scanned, - dirs_scanned=dirs_scanned, - loose_series=loose_series_count, - ) + # Keep inventory order deterministic even though adjacent + # buckets are inspected concurrently. Completion timing must + # not determine the order of the resulting review groups. + oldest_task = next(iter(pending)) + done, _not_done = await asyncio.wait( + (oldest_task,), + return_when=asyncio.FIRST_COMPLETED, + timeout=SCAN_CANCELLATION_POLL_INTERVAL_SECONDS, + ) + await cancellation_bridge.checkpoint() + for task in done: + active_path_count -= pending.pop(task) + yield task.result() + finally: + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) - async def scan_files(self, file_paths: list[str]) -> list[DiscoveredSeries]: + async def scan_files( + self, + file_paths: list[str], + *, + root_path: str | Path | None = None, + ) -> list[DiscoveredSeries]: """Build DiscoveredSeries from explicit file paths (no directory walk). Groups files by parent directory, treating each directory as a @@ -371,16 +1057,28 @@ async def scan_files(self, file_paths: list[str]) -> list[DiscoveredSeries]: comicinfo_sem = asyncio.Semaphore(ARCHIVE_READ_CONCURRENCY) discovered_list: list[DiscoveredSeries] = [] + root = Path(root_path).resolve() if root_path is not None else None + source_ordinal_by_path = self._build_source_ordinal_index( + (path for files in dir_files.values() for path in files), + root=root, + ) for series_dir, comic_files in sorted(dir_files.items()): name, year, folder_cv_id = self._extract_folder_identity(series_dir.name) discovered_files = await self._build_discovered_files( comic_files, comicinfo_sem, + root=root, allow_weak_file_identity=True, ) if not discovered_files: continue + self._stamp_source_folder_cohort( + discovered_files, + source_dir=series_dir, + root=root, + source_ordinal_by_path=source_ordinal_by_path, + ) publisher: str | None = None discovered_list.extend( @@ -403,11 +1101,62 @@ async def scan_files(self, file_paths: list[str]) -> list[DiscoveredSeries]: ) return discovered_list + @staticmethod + def _relative_source_path(path: Path, *, root: Path | None) -> str: + """Return a stable non-absolute source path when a scan root is known.""" + if root is not None: + try: + relative = path.relative_to(root).as_posix() + except ValueError: + pass + else: + return "(root)" if relative == "." else relative + digest = hashlib.sha256(str(path.resolve(strict=False)).encode("utf-8")).hexdigest() + return f"selected:{digest}" + + @classmethod + def _build_source_ordinal_index( + cls, + paths: Iterable[Path], + *, + root: Path | None, + ) -> dict[str, int]: + """Assign deterministic one-based ordinals across a scan's source files.""" + path_items = list(paths) + ordered = sorted( + path_items, + key=lambda item: ( + cls._relative_source_path(item, root=root).casefold(), + cls._relative_source_path(item, root=root), + str(item), + ), + ) + return {str(path): ordinal for ordinal, path in enumerate(ordered, start=1)} + + @classmethod + def _stamp_source_folder_cohort( + cls, + discovered_files: list[DiscoveredFile], + *, + source_dir: Path, + root: Path | None, + source_ordinal_by_path: dict[str, int], + ) -> None: + """Attach the complete pre-split folder cohort to every discovered file.""" + cohort_key = cls._relative_source_path(source_dir, root=root) + for discovered_file in discovered_files: + discovered_file.source_folder_cohort_key = cohort_key + discovered_file.source_ordinal = source_ordinal_by_path.get(discovered_file.file_path) + async def _build_discovered_files( self, comic_files: list[Path], sem: asyncio.Semaphore, *, + file_task_slots: asyncio.Semaphore | None = None, + perform_cancellation_checks: bool = True, + cancellation_checkpoint: Callable[[], Awaitable[None]] | None = None, + root: Path | None = None, folder_publisher: str | None = None, allow_weak_file_identity: bool = False, ) -> list[DiscoveredFile]: @@ -423,10 +1172,12 @@ async def _process_one(fpath: Path, *, is_series_sample: bool) -> DiscoveredFile file_name = fpath.name file_format = fpath.suffix.lstrip(".").lower() - # Get file size (sync but fast — just a stat call) + # Capture the identity used to detect scan-to-execution changes. try: - file_size = fpath.stat().st_size - except OSError: + source_signature = build_file_identity_signature(fpath) + file_size = int(source_signature["size"]) + except (OSError, ConfigurationError): + source_signature = {} file_size = 0 initial_metadata = extractor.from_path( @@ -463,55 +1214,245 @@ async def _process_one(fpath: Path, *, is_series_sample: bool) -> DiscoveredFile else: metadata = initial_metadata - parsed = metadata.parsed_release - issue_number_raw: str | None = None - if parsed is not None and parsed.issue_number is not None: - issue_number_raw = ( - str(int(parsed.issue_number)) - if parsed.issue_number == int(parsed.issue_number) - else str(parsed.issue_number) + parsed_series = metadata.series_name + parsed_issue_number = metadata.issue_number + parsed_year = metadata.year + parsed_publisher = metadata.publisher + issue_type = metadata.issue_type + metadata_signals = dict(metadata.signals) + metadata_diagnostics = dict(metadata.diagnostics) + + layout_match, relative_path = self._match_selected_layout(fpath, root=root) + if self._compiled_source_layout is not None and relative_path is not None: + layout_diagnostics: dict[str, object] = { + "fit": layout_match is not None, + "fallback_used": layout_match is None and self._source_layout.fallback_to_auto, + "relative_path": relative_path, + } + if layout_match is None and not self._source_layout.fallback_to_auto: + layout_diagnostics.update( + { + "review_required": True, + "review_reason": "selected_layout_no_match", + } + ) + if layout_match is not None and layout_match.issue_title is not None: + layout_diagnostics["issue_title"] = layout_match.issue_title + metadata_diagnostics["source_layout"] = layout_diagnostics + + conflicts: dict[str, dict[str, object]] = {} + + def apply_layout_value( + field_name: str, + current: object, + selected: object | None, + ) -> object: + if selected is None: + return current + current_signal = metadata_signals.get(field_name) + if current is not None and current_signal in _EXACT_METADATA_SIGNALS: + if str(current).casefold() != str(selected).casefold(): + conflicts[field_name] = { + "selected": selected, + "preserved_signal": current_signal.value, + } + return current + metadata_signals[field_name] = MetadataSignal.SOURCE_LAYOUT + return selected + + layout_issue_number_raw: str | None = None + if layout_match is not None: + parsed_series = cast( + "str | None", + apply_layout_value( + "series_name", + parsed_series, + layout_match.series, + ), ) - elif metadata.issue_number is not None: - issue_number_raw = ( - str(int(metadata.issue_number)) - if metadata.issue_number == int(metadata.issue_number) - else str(metadata.issue_number) + parsed_year = cast( + "int | None", + apply_layout_value("year", parsed_year, layout_match.year), ) + parsed_publisher = cast( + "str | None", + apply_layout_value( + "publisher", + parsed_publisher, + layout_match.publisher, + ), + ) + if layout_match.issue_number is not None: + selected_issue_number = normalize_issue_number(layout_match.issue_number) + parsed_issue_number = cast( + "float | None", + apply_layout_value( + "issue_number", + parsed_issue_number, + selected_issue_number, + ), + ) + if metadata_signals.get("issue_number") == MetadataSignal.SOURCE_LAYOUT: + layout_issue_number_raw = layout_match.issue_number + if layout_match.issue_type is not None: + selected_issue_type = IssueType(detect_issue_type(layout_match.issue_type)) + issue_type = cast( + "IssueType", + apply_layout_value( + "issue_type", + issue_type, + selected_issue_type, + ), + ) + + if conflicts: + metadata_diagnostics["source_layout_conflicts"] = conflicts + + parsed = metadata.parsed_release + issue_number_raw: str | None = None + if layout_issue_number_raw is not None: + issue_number_raw = layout_issue_number_raw + elif parsed is not None and parsed.issue_number is not None: + issue_number_raw = format_issue_number(parsed.issue_number) + elif parsed_issue_number is not None: + issue_number_raw = format_issue_number(parsed_issue_number) return DiscoveredFile( file_path=str(fpath), file_name=file_name, file_size=file_size, file_format=file_format, - parsed_series=metadata.series_name, - parsed_issue_number=metadata.issue_number, - parsed_year=metadata.year, - parsed_publisher=metadata.publisher, + parsed_series=str(parsed_series) if parsed_series is not None else None, + parsed_issue_number=( + float(parsed_issue_number) if parsed_issue_number is not None else None + ), + parsed_year=int(parsed_year) if parsed_year is not None else None, + parsed_publisher=(str(parsed_publisher) if parsed_publisher is not None else None), has_comicinfo=bool(metadata.diagnostics.get("has_comicinfo")), comicvine_issue_id=metadata.comicvine_issue_id, issue_number_raw=issue_number_raw, - issue_type=metadata.issue_type, + issue_type=IssueType(issue_type), comicvine_series_id=metadata.comicvine_series_id, series_status=metadata.series_status, issue_count_hint=metadata.issue_count_hint, metadata_signals={ key: value.value if isinstance(value, MetadataSignal) else str(value) - for key, value in metadata.signals.items() + for key, value in metadata_signals.items() }, - metadata_diagnostics=dict(metadata.diagnostics), + metadata_diagnostics=metadata_diagnostics, + source_signature=source_signature, + ) + + async def _tracked_process_one( + fpath: Path, + *, + is_series_sample: bool, + ) -> DiscoveredFile: + self._active_file_task_count += 1 + self._active_file_task_high_water = max( + self._active_file_task_high_water, + self._active_file_task_count, ) + try: + return await _process_one(fpath, is_series_sample=is_series_sample) + finally: + self._active_file_task_count -= 1 sorted_files = sorted(comic_files) - tasks = [ - _process_one(f, is_series_sample=index == 0) for index, f in enumerate(sorted_files) - ] + task_slots = file_task_slots or asyncio.Semaphore(ARCHIVE_READ_CONCURRENCY) + effective_checkpoint = cancellation_checkpoint + if effective_checkpoint is None and perform_cancellation_checks: + effective_checkpoint = self._cancellation_check + next_index = 0 + pending: set[asyncio.Task[DiscoveredFile]] = set() results: list[DiscoveredFile] = [] - for task in asyncio.as_completed(tasks): - results.append(await task) - if self._file_progress_callback is not None: - await self._file_progress_callback(1) + + async def start_next() -> bool: + nonlocal next_index + if next_index >= len(sorted_files): + return False + if effective_checkpoint is None: + await task_slots.acquire() + else: + while True: + try: + await asyncio.wait_for( + task_slots.acquire(), + timeout=SCAN_CANCELLATION_POLL_INTERVAL_SECONDS, + ) + except TimeoutError: + await effective_checkpoint() + continue + break + try: + task = asyncio.create_task( + _tracked_process_one( + sorted_files[next_index], + is_series_sample=next_index == 0, + ) + ) + except BaseException: + task_slots.release() + raise + task.add_done_callback(lambda _task: task_slots.release()) + pending.add(task) + next_index += 1 + return True + + try: + for _ in range(min(ARCHIVE_READ_CONCURRENCY, len(sorted_files))): + await start_next() + + while pending: + done, pending = await asyncio.wait( + pending, + return_when=asyncio.FIRST_COMPLETED, + timeout=( + SCAN_CANCELLATION_POLL_INTERVAL_SECONDS + if effective_checkpoint is not None + else None + ), + ) + batch_results: list[DiscoveredFile] = [] + first_error: BaseException | None = None + for task in done: + try: + batch_results.append(task.result()) + except BaseException as exc: + if first_error is None: + first_error = exc + if first_error is not None: + raise first_error + if effective_checkpoint is not None: + await effective_checkpoint() + for discovered_file in batch_results: + results.append(discovered_file) + if self._file_progress_callback is not None: + await self._file_progress_callback(1) + await start_next() + finally: + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) return sorted(results, key=lambda df: df.file_name) + def _match_selected_layout( + self, + path: Path, + *, + root: Path | None, + ) -> tuple[SourceLayoutMatch | None, str | None]: + """Match one root-relative path against the frozen selected layout.""" + compiled = self._compiled_source_layout + if compiled is None or root is None: + return None, None + try: + relative_path = path.relative_to(root).as_posix() + except ValueError: + return None, None + return compiled.match(relative_path), relative_path + def _should_load_archive_metadata( self, metadata: SourceMetadata, @@ -565,8 +1506,63 @@ def _build_series_candidates( tuple[str, int | None, str | None], ] = {} folder_identity = _normalize_series_identity(folder_name) + folder_is_series_boundary = root is None or source_dir != root for discovered_file in discovered_files: + embedded_series_id = discovered_file.comicvine_series_id + sidecar_folder_identity = ( + folder_is_series_boundary + and embedded_series_id is not None + and discovered_file.metadata_signals.get("comicvine_series_id") + == MetadataSignal.SIDECAR.value + ) + belongs_to_trusted_folder = folder_cv_id is not None and embedded_series_id in { + None, + folder_cv_id, + } + if belongs_to_trusted_folder or embedded_series_id is not None: + # A Comic Vine volume ID is a stronger series boundary than an + # issue filename's publication year or publisher text. Real + # ComicInfo files commonly omit ``Volume`` while retaining the + # issue-level ``Year``; grouping on that year splits one series + # into a candidate per issue year. A trusted folder identity + # also safely absorbs files whose archive metadata was deferred. + exact_series_id = folder_cv_id if belongs_to_trusted_folder else embedded_series_id + identity_key: tuple[str, int | None, str | None] = ( + f"comicvine:{exact_series_id}", + None, + None, + ) + classified.setdefault(identity_key, []).append(discovered_file) + + if belongs_to_trusted_folder or ( + sidecar_folder_identity and discovered_file.parsed_issue_number is not None + ): + discovered_file.parsed_series = folder_name + label_name = folder_name + label_year = folder_year + label_publisher = folder_publisher or discovered_file.parsed_publisher + else: + label_name = discovered_file.parsed_series or folder_name + year_signal = discovered_file.metadata_signals.get("year") + label_year = ( + discovered_file.parsed_year + if year_signal in {"comicinfo", "sidecar", "source_layout"} + else folder_year + ) + label_publisher = discovered_file.parsed_publisher or folder_publisher + + existing_label = labels.get(identity_key) + if existing_label is None: + labels[identity_key] = (label_name, label_year, label_publisher) + else: + labels[identity_key] = ( + existing_label[0], + existing_label[1] or label_year, + existing_label[2] or label_publisher, + ) + continue + identity = self._identity_for_file( discovered_file, allow_weak_file_identity=allow_weak_file_identity, @@ -578,9 +1574,18 @@ def _build_series_candidates( collapse_to_folder = bool( discovered_file.parsed_series and folder_identity - and _should_collapse_to_folder_identity( - discovered_file.parsed_series, - folder_name, + and ( + _should_collapse_to_folder_identity( + discovered_file.parsed_series, + folder_name, + ) + or ( + folder_is_series_boundary + and _should_use_folder_identity_for_issue_title_file( + discovered_file, + folder_name, + ) + ) ) ) type_discriminator = _non_standard_group_discriminator(discovered_file.issue_type) @@ -725,6 +1730,7 @@ def _inventory_tree( self, root: Path, progress_callback: Callable[[int, int], None] | None = None, + cancellation_event: threading.Event | None = None, ) -> tuple[int, int]: """Walk the tree and count visited directories plus supported comic files.""" directory_count = 0 @@ -735,74 +1741,27 @@ def _on_error(exc: OSError) -> None: logger.warning("import_scan_walk_error", path=str(root), error=str(exc)) for dirpath, dirnames, filenames in root.walk(on_error=_on_error): + if cancellation_event is not None and cancellation_event.is_set(): + break dirnames[:] = [d for d in dirnames if d not in IGNORE_DIRS and not d.startswith(".")] dirnames.sort() filenames.sort() directory_count += 1 - file_count += sum( - 1 - for fname in filenames - if _is_scan_candidate_file(dirpath / fname, self._extensions) - ) - if progress_callback is not None: - now = time.monotonic() + for index, fname in enumerate(filenames): if ( - directory_count == 1 - or directory_count % 100 == 0 - or (now - last_emit_at) >= 0.2 + index % 128 == 0 + and cancellation_event is not None + and cancellation_event.is_set() ): - progress_callback(directory_count, file_count) - last_emit_at = now - - if progress_callback is not None: - progress_callback(directory_count, file_count) - - return directory_count, file_count - - def _walk_tree(self, root: Path) -> dict[Path, list[Path]]: - """Walk the directory tree and collect comic files per directory. - - Runs synchronously in a thread pool. - """ - dir_files, _directory_count, _file_count = self._walk_tree_with_counts(root) - return dir_files - - def _walk_tree_with_counts( - self, - root: Path, - progress_callback: Callable[[int, int], None] | None = None, - ) -> tuple[dict[Path, list[Path]], int, int]: - """Walk the directory tree once and return comic files plus live totals.""" - dir_files: dict[Path, list[Path]] = {} - directory_count = 0 - file_count = 0 - last_emit_at = time.monotonic() - - def _on_error(exc: OSError) -> None: - logger.warning("import_scan_walk_error", path=str(root), error=str(exc)) - - for dirpath, dirnames, filenames in root.walk(on_error=_on_error): - # Prune ignored directories - dirnames[:] = [d for d in dirnames if d not in IGNORE_DIRS and not d.startswith(".")] - dirnames.sort() - filenames.sort() - directory_count += 1 - - comics: list[Path] = [] - for fname in filenames: - fpath = dirpath / fname - if _is_scan_candidate_file(fpath, self._extensions): - comics.append(fpath) - file_count += len(comics) - - if comics: - dir_files[dirpath] = comics - + break + if _is_scan_candidate_file(dirpath / fname, self._extensions): + file_count += 1 + if cancellation_event is not None and cancellation_event.is_set(): + break if progress_callback is not None: now = time.monotonic() if ( directory_count == 1 - or directory_count % 100 == 0 or (now - last_emit_at) >= SCAN_PROGRESS_EMIT_INTERVAL_SECONDS ): progress_callback(directory_count, file_count) @@ -811,7 +1770,7 @@ def _on_error(exc: OSError) -> None: if progress_callback is not None: progress_callback(directory_count, file_count) - return dir_files, directory_count, file_count + return directory_count, file_count def _identify_series_dirs(self, dir_files: dict[Path, list[Path]]) -> list[Path]: """Identify leaf directories that should be treated as series roots. @@ -820,17 +1779,22 @@ def _identify_series_dirs(self, dir_files: dict[Path, list[Path]]) -> list[Path] of its subdirectories also contain comic files. """ all_dirs = set(dir_files.keys()) - series_dirs: list[Path] = [] + dirs_with_comic_descendants: set[Path] = set() - for d in all_dirs: - # Check if any child directory also has comics - has_child_with_comics = any( - other != d and _is_descendant(other, d) for other in all_dirs - ) - if not has_child_with_comics: - series_dirs.append(d) + # Walk each comic directory's ancestry once instead of comparing every + # directory with every other directory. Path depth is bounded by the + # source tree, so classification grows with D * depth rather than D^2. + for directory in all_dirs: + ancestor = directory.parent + while True: + if ancestor in all_dirs: + dirs_with_comic_descendants.add(ancestor) + parent = ancestor.parent + if parent == ancestor: + break + ancestor = parent - return series_dirs + return [directory for directory in all_dirs if directory not in dirs_with_comic_descendants] def _extract_from_folder_name(self, folder_name: str) -> tuple[str, int | None]: """Extract (series_name, year) from a folder name. @@ -873,9 +1837,10 @@ def _extract_folder_identity(self, folder_name: str) -> tuple[str, int | None, i return name, year, None comicvine_id = match.group("bare_cv_id") or match.group("bracketed_cv_id") assert comicvine_id is not None + year_text = match.group("year") return ( re.sub(r"\s{2,}", " ", match.group("name")).strip(), - int(match.group("year")), + int(year_text) if year_text.isdigit() else None, int(comicvine_id), ) diff --git a/src/pullbox/core/comicinfo.py b/src/pullbox/core/comicinfo.py index 8e9b1c03..c7094085 100644 --- a/src/pullbox/core/comicinfo.py +++ b/src/pullbox/core/comicinfo.py @@ -44,6 +44,7 @@ class ComicInfoData: genre: str | None = None web: str | None = None story_arc: str | None = None + story_arc_number: str | None = None series_group: str | None = None language: str | None = None @@ -86,6 +87,7 @@ def parse_comicinfo(xml_string: str) -> ComicInfoData: genre=_text(root, "Genre"), web=_trusted_text(root, "Web"), story_arc=_text(root, "StoryArc"), + story_arc_number=_text(root, "StoryArcNumber"), series_group=_text(root, "SeriesGroup"), language=_text(root, "LanguageISO"), ) diff --git a/src/pullbox/core/exceptions.py b/src/pullbox/core/exceptions.py index 55f28dc4..9ae28c0c 100644 --- a/src/pullbox/core/exceptions.py +++ b/src/pullbox/core/exceptions.py @@ -91,6 +91,15 @@ def __init__(self, message: str) -> None: super().__init__(message=message, code="CONFIG_ERROR", status_code=500) +class ImportDestinationValidationError(ConfigurationError): + """Fail-closed managed-import destination requiring explicit review.""" + + def __init__(self, reason: str, message: str) -> None: + self.reason = reason + super().__init__(message) + self.code = "IMPORT_DESTINATION_REVIEW" + + class BackupError(PullboxError): """Raised when a backup operation fails.""" diff --git a/src/pullbox/core/file_ops.py b/src/pullbox/core/file_ops.py index 9916189b..ba0fc1d3 100644 --- a/src/pullbox/core/file_ops.py +++ b/src/pullbox/core/file_ops.py @@ -33,6 +33,12 @@ from pullbox.core.library_comicinfo import ( prepare_source_artifact as _prepare_source_artifact, ) +from pullbox.core.library_file_ownership import ( + build_file_identity_signature, + build_managed_placement_signature, + resolve_referenced_library_root, + validate_file_identity_signature, +) from pullbox.core.library_leave_in_place import handle_leave_in_place as _handle_leave_in_place from pullbox.core.library_materialization import ( paths_on_same_filesystem, @@ -42,10 +48,8 @@ build_naming_snapshot as _build_naming_snapshot, ) from pullbox.core.library_naming import ( - build_series_folder_name as _build_series_folder_name, -) -from pullbox.core.library_naming import ( - compute_target_filename as _compute_target_filename, + build_series_folder_name, + compute_target_filename, ) from pullbox.core.library_naming import ( resolve_naming_issue_type as _resolve_naming_issue_type, @@ -53,7 +57,11 @@ from pullbox.core.library_permission_application import ( apply_materialized_file_permissions as _apply_materialized_file_permissions, ) -from pullbox.core.library_policy import LibraryIngestPolicy, load_library_ingest_policy +from pullbox.core.library_policy import ( + LibraryIngestPolicy, + load_effective_library_ingest_policy, + load_library_ingest_policy, +) from pullbox.core.library_root_resolution import materialize_series_path as _materialize_series_path from pullbox.core.library_root_resolution import path_is_inside_root as _path_is_inside_root from pullbox.core.library_root_resolution import resolve_library_root as _resolve_library_root @@ -66,8 +74,15 @@ from pullbox.core.library_transfer import safe_move as _library_safe_move from pullbox.core.library_transfer import transfer_into_library as _transfer_into_library from pullbox.models.issue import Issue, IssueStatus -from pullbox.models.library import FileFormat, LibraryFile, LibraryRoot, MatchConfidence +from pullbox.models.library import ( + FileFormat, + LibraryFile, + LibraryFileStorageMode, + LibraryRoot, + MatchConfidence, +) from pullbox.models.series import Series +from pullbox.services.library_root_management import validate_managed_library_root from pullbox.services.series_delete_targets import trash_relative_path from pullbox.utilities.executors.file_converter import convert_file from pullbox.utilities.settings import move_file_to_utility_trash, restore_file_from_utility_trash @@ -79,11 +94,35 @@ from pullbox.core.library_permissions import LibraryPermissionPolicy from pullbox.models.download import DownloadClientType +_build_series_folder_name = build_series_folder_name +_compute_target_filename = compute_target_filename + logger = structlog.get_logger(__name__) _DEFERRED_REPLACEMENT_STASHES_KEY = "pullbox_deferred_replacement_stashes" +async def _enqueue_story_arc_sync_safely( + session: AsyncSession, + library_file: LibraryFile, +) -> None: + """Persist automatic arc work without allowing outbox failure to abort ingestion.""" + from pullbox.services.story_arc_sync_queue import enqueue_story_arc_sync_work + + try: + async with session.begin_nested(): + await enqueue_story_arc_sync_work(session, library_file) + except Exception: + # The scheduled discrepancy pass can recover missing work. Canonical + # registration must remain durable even when this optional producer fails. + logger.warning( + "story_arc_sync_enqueue_deferred", + library_file_id=library_file.id, + issue_id=library_file.issue_id, + exc_info=True, + ) + + def _safe_move(src: Path, dst: Path) -> None: """Compatibility wrapper for older move-path callers.""" _library_safe_move(src, dst) @@ -96,6 +135,8 @@ class LibraryFileRegistrationOutcome: library_file: LibraryFile series_folder_created: bool series_folder_path: Path | None + created_directory_paths: tuple[Path, ...] = () + directory_ownership_boundary_path: Path | None = None permission_results: tuple[PermissionChangeResult, ...] = () @@ -118,6 +159,8 @@ def _pending_replacement_stashes(session: AsyncSession) -> list[_ReplacementStas @event.listens_for(AsyncSession.sync_session_class, "after_commit") def _cleanup_deferred_replacement_stashes(session: Any) -> None: """Discard staged originals only after the database commit succeeds.""" + if session.in_nested_transaction(): + return stashes = session.info.pop(_DEFERRED_REPLACEMENT_STASHES_KEY, []) for stash in stashes: _discard_replacement_stash_sync(stash) @@ -126,6 +169,8 @@ def _cleanup_deferred_replacement_stashes(session: Any) -> None: @event.listens_for(AsyncSession.sync_session_class, "after_rollback") def _restore_deferred_replacement_stashes(session: Any) -> None: """Restore staged originals when a caller rolls back after registration.""" + if session.in_nested_transaction(): + return stashes = session.info.pop(_DEFERRED_REPLACEMENT_STASHES_KEY, []) for stash in reversed(stashes): _restore_replacement_stash_sync(stash) @@ -169,9 +214,9 @@ async def resolve_library_destination( library_root_id: int | None = None, ) -> tuple[Path, LibraryRoot]: """Resolve the canonical library destination path for an issue/source pair.""" - ingest_policy = await load_library_ingest_policy(session) + global_ingest_policy = await load_library_ingest_policy(session) if rename is None: - rename = ingest_policy.rename_on_import + rename = global_ingest_policy.rename_on_import loaded_issue = await _load_issue_with_series_and_publisher(session, issue) series = loaded_issue.series @@ -182,24 +227,18 @@ async def resolve_library_destination( library_root_id, series=series, ) - if isinstance(series, Series) and series.path: - series_folder = Path(series.path) - else: - series_folder = Path(root.path) / _build_series_folder_name(series, ingest_policy) - - if rename: - effective_issue_type = await _resolve_naming_issue_type(session, loaded_issue) - target_name = _compute_target_filename( - loaded_issue, - series, - source_path, - ingest_policy, - issue_type_override=effective_issue_type, - ) - else: - target_name = source_path.name - - return series_folder / target_name, root + await validate_managed_library_root(root) + ingest_policy = await load_effective_library_ingest_policy(session, root) + target_path = await _predict_library_target_path( + session, + source_path, + loaded_issue, + series, + root, + ingest_policy, + bool(rename), + ) + return target_path, root async def register_library_file( @@ -209,6 +248,8 @@ async def register_library_file( confidence: MatchConfidence, *, move_to_library: bool = True, + storage_mode: LibraryFileStorageMode | None = None, + expected_source_signature: dict[str, object] | None = None, rename: bool | None = None, library_root_id: int | None = None, transfer_method: str | None = None, @@ -226,9 +267,15 @@ async def register_library_file( artifact_transfer: Callable[..., Any] | None = None, comicinfo_materializer: Callable[..., Any] | None = None, placement_started_callback: Callable[..., Any] | None = None, + placement_completed_callback: Callable[..., Any] | None = None, + placement_temp_paths: Callable[[Path, Path], tuple[Path, ...]] | None = None, allow_resource_safety_exception: bool = False, replace_existing_library_file: bool = False, replacement_trash_dir: Path | None = None, + preserve_replaced_artifact: bool = False, + source_scan_root: Path | None = None, + strict_import_target: bool = False, + recover_existing_managed_artifact: bool = False, ) -> LibraryFile: """Register a file in the library, optionally moving/renaming it.""" outcome = await register_library_file_with_metadata( @@ -237,6 +284,8 @@ async def register_library_file( issue, confidence, move_to_library=move_to_library, + storage_mode=storage_mode, + expected_source_signature=expected_source_signature, rename=rename, library_root_id=library_root_id, transfer_method=transfer_method, @@ -254,9 +303,15 @@ async def register_library_file( artifact_transfer=artifact_transfer, comicinfo_materializer=comicinfo_materializer, placement_started_callback=placement_started_callback, + placement_completed_callback=placement_completed_callback, + placement_temp_paths=placement_temp_paths, allow_resource_safety_exception=allow_resource_safety_exception, replace_existing_library_file=replace_existing_library_file, replacement_trash_dir=replacement_trash_dir, + preserve_replaced_artifact=preserve_replaced_artifact, + source_scan_root=source_scan_root, + strict_import_target=strict_import_target, + recover_existing_managed_artifact=recover_existing_managed_artifact, ) return outcome.library_file @@ -268,6 +323,8 @@ async def register_library_file_with_metadata( confidence: MatchConfidence, *, move_to_library: bool = True, + storage_mode: LibraryFileStorageMode | None = None, + expected_source_signature: dict[str, object] | None = None, rename: bool | None = None, library_root_id: int | None = None, transfer_method: str | None = None, @@ -285,9 +342,15 @@ async def register_library_file_with_metadata( artifact_transfer: Callable[..., Any] | None = None, comicinfo_materializer: Callable[..., Any] | None = None, placement_started_callback: Callable[..., Any] | None = None, + placement_completed_callback: Callable[..., Any] | None = None, + placement_temp_paths: Callable[[Path, Path], tuple[Path, ...]] | None = None, allow_resource_safety_exception: bool = False, replace_existing_library_file: bool = False, replacement_trash_dir: Path | None = None, + preserve_replaced_artifact: bool = False, + source_scan_root: Path | None = None, + strict_import_target: bool = False, + recover_existing_managed_artifact: bool = False, ) -> LibraryFileRegistrationOutcome: """Register a file in the library, optionally moving/renaming it. @@ -318,6 +381,38 @@ async def register_library_file_with_metadata( comicinfo_already_embedded = False replacement_stash: _ReplacementStash | None = None replacement_finalized = False + requested_rename = rename + requested_transfer_method = transfer_method + requested_normalize_to_cbz = normalize_to_cbz + requested_comicinfo_update = update_embedded_comicinfo_from_match + effective_storage_mode = storage_mode or ( + LibraryFileStorageMode.MANAGED + if move_to_library or recover_existing_managed_artifact + else LibraryFileStorageMode.REFERENCED + ) + referenced_signature: dict[str, int | str] | None = None + managed_placement_signature: dict[str, int | str] | None = None + placement_started = False + created_directory_paths: tuple[Path, ...] = () + directory_ownership_boundary_path: Path | None = None + + if effective_storage_mode == LibraryFileStorageMode.REFERENCED and move_to_library: + raise ConfigurationError("Referenced storage cannot materialize a managed library file.") + if recover_existing_managed_artifact and move_to_library: + raise ConfigurationError( + "Managed artifact recovery registers an existing destination without a transfer." + ) + if ( + recover_existing_managed_artifact + and effective_storage_mode != LibraryFileStorageMode.MANAGED + ): + raise ConfigurationError("Recovered Pullbox artifacts must retain managed ownership.") + if ( + effective_storage_mode == LibraryFileStorageMode.MANAGED + and not move_to_library + and not recover_existing_managed_artifact + ): + raise ConfigurationError("Managed storage requires library materialization.") async def notify_placement_started( *, @@ -325,11 +420,18 @@ async def notify_placement_started( target_path: Path, effective_transfer_method: str, series_folder_created: bool, + created_directories: tuple[Path, ...], + directory_ownership_boundary: Path | None, ) -> None: + nonlocal placement_started if placement_started_callback is None: return - temp_paths: list[Path] = [] - if ( + temp_paths = ( + list(placement_temp_paths(artifact_source, target_path)) + if placement_temp_paths is not None + else [] + ) + if placement_temp_paths is None and ( update_embedded_comicinfo_from_match and artifact_source.suffix.lower() == ".cbz" and target_path.suffix.lower() == ".cbz" @@ -342,10 +444,26 @@ async def notify_placement_started( transfer_method=effective_transfer_method, series_folder_created=series_folder_created, series_folder_path=target_path.parent, + created_directory_paths=created_directories, + directory_ownership_boundary_path=directory_ownership_boundary, temp_paths=tuple(temp_paths), ) if inspect.isawaitable(callback_result): await callback_result + placement_started = True + + async def notify_placement_completed(target_path: Path) -> None: + nonlocal managed_placement_signature + if placement_completed_callback is None or not placement_started: + return + destination_signature = build_managed_placement_signature(target_path) + managed_placement_signature = destination_signature + callback_result = placement_completed_callback( + target_path=target_path, + destination_signature=destination_signature, + ) + if inspect.isawaitable(callback_result): + await callback_result try: # Prevent callers with dirty ORM state from opening a SQLite write @@ -366,6 +484,53 @@ async def notify_placement_started( effective_ingest_policy.update_embedded_comicinfo_from_match ) + if effective_storage_mode == LibraryFileStorageMode.REFERENCED: + if requested_rename is True: + raise ConfigurationError("Referenced library files cannot rename source files.") + if requested_normalize_to_cbz is True: + raise ConfigurationError( + "Referenced library files cannot normalize or convert source files." + ) + if requested_comicinfo_update is True: + raise ConfigurationError( + "Referenced library files cannot update embedded ComicInfo.xml." + ) + if replace_existing_library_file: + raise ConfigurationError( + "Referenced library files cannot replace an existing artifact." + ) + if requested_transfer_method not in {None, "leave_in_place", "referenced"}: + raise ConfigurationError( + "Referenced library files cannot use a transfer method." + ) + rename = False + normalize_to_cbz = False + update_embedded_comicinfo_from_match = False + transfer_method = "leave_in_place" + elif recover_existing_managed_artifact: + if requested_rename is True: + raise ConfigurationError("Recovered managed artifacts cannot be renamed.") + if requested_normalize_to_cbz is True: + raise ConfigurationError( + "Recovered managed artifacts cannot be normalized or converted." + ) + if requested_comicinfo_update is True: + raise ConfigurationError( + "Recovered managed artifacts cannot rewrite embedded ComicInfo.xml." + ) + if requested_transfer_method not in {None, "recovered"}: + raise ConfigurationError( + "Recovered managed artifacts cannot use a transfer method." + ) + if replace_existing_library_file: + raise ConfigurationError( + "Recovered managed artifacts cannot replace an existing artifact." + ) + rename = False + normalize_to_cbz = False + update_embedded_comicinfo_from_match = False + transfer_method = "recovered" + # 4. Load series with publisher (need for naming and root resolution) effective_issue = ( await _load_issue_with_series_and_publisher(session, issue) @@ -375,19 +540,38 @@ async def notify_placement_started( series = effective_issue.series # 5. Resolve library root - root = await _resolve_library_root( - session, - source_path, - library_root_id, - series=series, - ) + if effective_storage_mode == LibraryFileStorageMode.REFERENCED: + root, source_path, referenced_signature = await resolve_referenced_library_root( + session, + source_path, + library_root_id, + ) + if expected_source_signature is not None: + validate_file_identity_signature( + expected_source_signature, + referenced_signature, + ) + prepared_source = source_path + else: + root = await _resolve_library_root( + session, + source_path, + library_root_id, + series=series, + ) + await validate_managed_library_root(root) + if ingest_policy is None: + effective_ingest_policy = await load_effective_library_ingest_policy( + session, + root, + ) replace_existing_path = ( Path(effective_issue.library_file.file_path) if replace_existing_library_file and effective_issue.library_file is not None else None ) - if not source_path.exists(): + if not source_path.exists() and not strict_import_target: recovered = await _recover_materialized_target_without_source( session, source_path=source_path, @@ -402,6 +586,7 @@ async def notify_placement_started( ) if recovered is not None: return recovered + if not source_path.exists(): raise FileNotFoundError(f"Source file not found: {source_path}") seed_safe_torrent_import = ( @@ -451,6 +636,8 @@ async def notify_placement_started( effective_ingest_policy, rename, replace_existing_path=replace_existing_path, + source_scan_root=source_scan_root, + strict_import=strict_import_target, ) target_path = target.path replacement_stash = await _stage_replacement_file( @@ -458,6 +645,7 @@ async def notify_placement_started( prepared_source, replace_existing_library_file=replace_existing_library_file, replacement_trash_dir=replacement_trash_dir, + preserve_replaced_artifact=preserve_replaced_artifact, ) same_filesystem = await asyncio.to_thread( paths_on_same_filesystem, @@ -478,6 +666,8 @@ async def notify_placement_started( target_path=target_path, effective_transfer_method=transfer_method, series_folder_created=target.series_folder_created, + created_directories=target.created_directory_paths, + directory_ownership_boundary=target.directory_ownership_boundary_path, ) final_path = await transfer_artifact( prepared_source, @@ -486,6 +676,8 @@ async def notify_placement_started( transfer_progress_callback=transfer_progress_callback, ) series_folder_created = target.series_folder_created + created_directory_paths = target.created_directory_paths + directory_ownership_boundary_path = target.directory_ownership_boundary_path else: target = await _resolve_library_target_path( session, @@ -496,12 +688,15 @@ async def notify_placement_started( effective_ingest_policy, rename, replace_existing_path=replace_existing_path, + source_scan_root=source_scan_root, + strict_import=strict_import_target, ) replacement_stash = await _stage_replacement_file( effective_issue, prepared_source, replace_existing_library_file=replace_existing_library_file, replacement_trash_dir=replacement_trash_dir, + preserve_replaced_artifact=preserve_replaced_artifact, ) if _can_materialize_cbz_with_comicinfo( prepared_source, @@ -524,6 +719,8 @@ async def notify_placement_started( target_path=target.path, effective_transfer_method=transfer_method, series_folder_created=target.series_folder_created, + created_directories=target.created_directory_paths, + directory_ownership_boundary=target.directory_ownership_boundary_path, ) materialize_result = materializer( prepared_source, @@ -537,6 +734,8 @@ async def notify_placement_started( await materialize_result final_path = target.path series_folder_created = target.series_folder_created + created_directory_paths = target.created_directory_paths + directory_ownership_boundary_path = target.directory_ownership_boundary_path comicinfo_already_embedded = True else: await notify_placement_started( @@ -544,6 +743,8 @@ async def notify_placement_started( target_path=target.path, effective_transfer_method=transfer_method, series_folder_created=target.series_folder_created, + created_directories=target.created_directory_paths, + directory_ownership_boundary=target.directory_ownership_boundary_path, ) final_path = await transfer_artifact( prepared_source, @@ -552,6 +753,15 @@ async def notify_placement_started( transfer_progress_callback=transfer_progress_callback, ) series_folder_created = target.series_folder_created + created_directory_paths = target.created_directory_paths + directory_ownership_boundary_path = target.directory_ownership_boundary_path + elif recover_existing_managed_artifact: + if not _path_is_inside_root(prepared_source, root): + raise ConfigurationError( + "Recovered managed artifacts must be inside their managed library root." + ) + final_path = prepared_source.resolve(strict=True) + series_folder_created = False else: final_path = await _handle_leave_in_place( session, @@ -590,9 +800,15 @@ async def notify_placement_started( ) if inspect.isawaitable(embed_result): await embed_result + await notify_placement_completed(final_path) effective_issue_type = await _resolve_naming_issue_type(session, effective_issue) if move_to_library or _path_is_inside_root(final_path, root): - _materialize_series_path(series, final_path.parent, root) + _materialize_series_path( + series, + final_path.parent, + root, + storage_mode=effective_storage_mode, + ) naming_snapshot = _build_naming_snapshot( source_path=source_path, prepared_source=prepared_source, @@ -604,12 +820,20 @@ async def notify_placement_started( rename=bool(rename), effective_issue_type=effective_issue_type, transfer_method=transfer_method, - move_to_library=move_to_library, + move_to_library=move_to_library or recover_existing_managed_artifact, normalized_source=normalized_source, update_embedded_comicinfo_from_match=bool(update_embedded_comicinfo_from_match), normalize_to_cbz=bool(normalize_to_cbz), ) if existing is not None: + if existing.issue_id is not None and existing.issue_id != effective_issue.id: + raise ConfigurationError( + "This library path is already registered to a different issue." + ) + if existing.storage_mode != effective_storage_mode: + raise ConfigurationError( + "Existing library-file ownership cannot be changed during registration." + ) # Update match info on existing record await _update_existing_library_file_from_path( existing, @@ -619,6 +843,13 @@ async def notify_placement_started( root=root, confidence=confidence, naming_snapshot=naming_snapshot, + storage_mode=effective_storage_mode, + source_signature=( + referenced_signature + if referenced_signature is not None + else managed_placement_signature + or build_file_identity_signature(final_path) + ), ) await _finalize_replacement_stash_db_state( session, @@ -626,6 +857,7 @@ async def notify_placement_started( registered_file=existing, ) await session.flush() + await _enqueue_story_arc_sync_safely(session, existing) _defer_replacement_stash_cleanup(session, replacement_stash) replacement_finalized = True logger.info( @@ -644,6 +876,8 @@ async def notify_placement_started( library_file=existing, series_folder_created=series_folder_created, series_folder_path=final_path.parent, + created_directory_paths=created_directory_paths, + directory_ownership_boundary_path=directory_ownership_boundary_path, ) permission_results = await _apply_materialized_file_permissions( @@ -661,21 +895,44 @@ async def notify_placement_started( extension = final_path.suffix.lstrip(".").lower() file_format = _FORMAT_MAP.get(extension, FileFormat.CBZ) - lf = LibraryFile( - file_path=str(final_path), - file_name=final_path.name, - file_size=stat.st_size, - file_format=file_format, - file_modified_at=datetime.fromtimestamp(stat.st_mtime, tz=UTC), - match_confidence=confidence, - parsed_series=series.title if series else None, - parsed_issue_number=effective_issue.issue_number, - parsed_year=series.year_start if series else None, - issue_id=issue.id, - library_root_id=root.id, - naming_snapshot=naming_snapshot, + registered_signature = ( + referenced_signature + if referenced_signature is not None + else managed_placement_signature or build_file_identity_signature(final_path) ) - session.add(lf) + if replacement_stash is not None: + # Keep the canonical row identity so every dependent FK survives a + # path-changing replacement, including clean-library adoption. + lf = replacement_stash.library_file + await _update_existing_library_file_from_path( + lf, + final_path, + issue=effective_issue, + series=series, + root=root, + confidence=confidence, + naming_snapshot=naming_snapshot, + storage_mode=effective_storage_mode, + source_signature=registered_signature, + ) + else: + lf = LibraryFile( + file_path=str(final_path), + file_name=final_path.name, + file_size=stat.st_size, + file_format=file_format, + file_modified_at=datetime.fromtimestamp(stat.st_mtime, tz=UTC), + match_confidence=confidence, + parsed_series=series.title if series else None, + parsed_issue_number=effective_issue.issue_number, + parsed_year=series.year_start if series else None, + issue_id=issue.id, + library_root_id=root.id, + naming_snapshot=naming_snapshot, + storage_mode=effective_storage_mode, + source_signature=registered_signature, + ) + session.add(lf) # 10. Set Issue status to OWNED issue.status = IssueStatus.OWNED @@ -688,6 +945,7 @@ async def notify_placement_started( ) await session.flush() + await _enqueue_story_arc_sync_safely(session, lf) _defer_replacement_stash_cleanup(session, replacement_stash) replacement_finalized = True @@ -709,6 +967,8 @@ async def notify_placement_started( library_file=lf, series_folder_created=series_folder_created, series_folder_path=final_path.parent, + created_directory_paths=created_directory_paths, + directory_ownership_boundary_path=directory_ownership_boundary_path, permission_results=permission_results, ) except asyncio.CancelledError: @@ -755,6 +1015,7 @@ async def _stage_replacement_file( *, replace_existing_library_file: bool, replacement_trash_dir: Path | None, + preserve_replaced_artifact: bool = False, ) -> _ReplacementStash | None: """Move an existing issue file aside before materializing a replacement.""" if not replace_existing_library_file or issue.library_file is None: @@ -762,6 +1023,12 @@ async def _stage_replacement_file( library_file = issue.library_file original_path = Path(library_file.file_path) + if preserve_replaced_artifact: + return _ReplacementStash( + library_file=library_file, + original_path=original_path, + staged_path=None, + ) if not await asyncio.to_thread(original_path.exists): return _ReplacementStash( library_file=library_file, @@ -875,6 +1142,8 @@ async def _update_existing_library_file_from_path( root: LibraryRoot, confidence: MatchConfidence, naming_snapshot: dict[str, Any], + storage_mode: LibraryFileStorageMode, + source_signature: dict[str, int | str], ) -> None: """Refresh an existing LibraryFile row from the current artifact on disk.""" stat = await asyncio.to_thread(final_path.stat) @@ -892,6 +1161,8 @@ async def _update_existing_library_file_from_path( library_file.issue_id = issue.id library_file.library_root_id = root.id library_file.naming_snapshot = naming_snapshot + library_file.storage_mode = storage_mode + library_file.source_signature = source_signature issue.status = IssueStatus.OWNED @@ -948,7 +1219,12 @@ async def _recover_materialized_target_without_source( return None effective_issue_type = await _resolve_naming_issue_type(session, issue) - _materialize_series_path(series, target_path.parent, root) + _materialize_series_path( + series, + target_path.parent, + root, + storage_mode=LibraryFileStorageMode.MANAGED, + ) naming_snapshot = _build_naming_snapshot( source_path=source_path, prepared_source=recovery_source, diff --git a/src/pullbox/core/file_safety.py b/src/pullbox/core/file_safety.py index af0bad7a..e0e75fe3 100644 --- a/src/pullbox/core/file_safety.py +++ b/src/pullbox/core/file_safety.py @@ -14,11 +14,15 @@ from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path, PurePosixPath, PureWindowsPath +from stat import S_ISLNK from typing import TYPE_CHECKING, Any import structlog +from pullbox.core.archive import comicinfo_member_sort_key +from pullbox.core.comicinfo import ComicInfoData, parse_comicinfo from pullbox.core.filesystem_scan import iter_supported_files_with_handler +from pullbox.core.page_sources.base import canonical_page_names if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -117,9 +121,26 @@ def to_diagnostics(self) -> dict[str, Any]: class ZipArchiveSafetyReport: """Single-pass safety facts for one ZIP-based archive.""" + archive_path: Path total_size: int traversal_entries: list[str] dangerous_entries: list[str] + entry_names: tuple[str, ...] + comicinfo: ComicInfoData | None + comicinfo_entry: str | None + comicinfo_entry_count: int + comicinfo_error: str | None + page_count: int | None = None + + +@dataclass(frozen=True, slots=True) +class FileSafetyInspection: + """Immutable transient evidence returned for a single-file safety check.""" + + archives: tuple[ZipArchiveSafetyReport, ...] = () + + +_MAX_COMICINFO_XML_BYTES = 2 * 1024 * 1024 _ARCHIVE_SIZE_MARKERS = ( @@ -189,6 +210,8 @@ def is_resource_safety_exception_allowed(diagnostics: Mapping[str, Any] | None) previous_block = safety_exception.get("previous_block") if not isinstance(previous_block, Mapping): return False + if previous_block.get("code") in {"archive_no_pages", "single_page_comic"}: + return False return bool(previous_block.get("overrideable", True)) @@ -411,6 +434,50 @@ def inspect_zip_archive_safety( try: with zipfile.ZipFile(archive_path, "r") as zf: entries = zf.infolist() + + total_size = 0 + traversal_entries: list[str] = [] + dangerous_entries: list[str] = [] + for entry in entries: + entry_name = entry.filename + total_size += entry.file_size + if _has_path_traversal(entry_name): + traversal_entries.append(entry_name) + if block_dangerous and Path(entry_name).suffix.lower() in DANGEROUS_EXTENSIONS: + dangerous_entries.append(entry_name) + + comicinfo_entries = sorted( + ( + entry + for entry in entries + if not entry.is_dir() + and PurePosixPath(entry.filename.replace("\\", "/")).name.lower() + == "comicinfo.xml" + ), + key=lambda entry: comicinfo_member_sort_key(entry.filename), + ) + comicinfo: ComicInfoData | None = None + comicinfo_entry = comicinfo_entries[0].filename if comicinfo_entries else None + comicinfo_error: str | None = None + if comicinfo_entries: + comicinfo_member = comicinfo_entries[0] + if comicinfo_member.file_size > _MAX_COMICINFO_XML_BYTES: + comicinfo_error = "comicinfo_size_limit" + else: + try: + with zf.open(comicinfo_member, "r") as member: + xml_bytes = member.read(_MAX_COMICINFO_XML_BYTES + 1) + if len(xml_bytes) > _MAX_COMICINFO_XML_BYTES: + comicinfo_error = "comicinfo_size_limit" + else: + comicinfo = parse_comicinfo(xml_bytes.decode("utf-8", errors="replace")) + except (OSError, RuntimeError, zipfile.BadZipFile) as exc: + logger.warning( + "archive_comicinfo_inspection_failed", + path=str(archive_path), + error=str(exc), + ) + comicinfo_error = "comicinfo_unreadable" except (zipfile.BadZipFile, OSError) as exc: logger.warning( "archive_inspection_failed", @@ -422,21 +489,27 @@ def inspect_zip_archive_safety( details=[str(archive_path)], ) from exc - total_size = 0 - traversal_entries: list[str] = [] - dangerous_entries: list[str] = [] - for entry in entries: - entry_name = entry.filename - total_size += entry.file_size - if _has_path_traversal(entry_name): - traversal_entries.append(entry_name) - if block_dangerous and Path(entry_name).suffix.lower() in DANGEROUS_EXTENSIONS: - dangerous_entries.append(entry_name) - return ZipArchiveSafetyReport( + archive_path=archive_path, total_size=total_size, traversal_entries=traversal_entries, dangerous_entries=dangerous_entries, + entry_names=tuple(entry.filename for entry in entries), + comicinfo=comicinfo, + comicinfo_entry=comicinfo_entry, + comicinfo_entry_count=len(comicinfo_entries), + comicinfo_error=comicinfo_error, + page_count=len( + canonical_page_names( + [ + entry.filename + for entry in entries + if not entry.is_dir() + and entry.file_size > 0 + and not S_ISLNK(entry.external_attr >> 16) + ] + ) + ), ) @@ -448,7 +521,7 @@ def run_safety_checks( *, block_dangerous: bool, max_archive_size: int, -) -> None: +) -> FileSafetyInspection: """Run all file safety checks synchronously. This is a pure-sync function that performs filesystem I/O only (no @@ -460,6 +533,9 @@ def run_safety_checks( """ log = logger.bind(download_path=str(download_path)) + if download_path.is_file() and download_path.stat().st_size == 0: + raise FileSafetyError("zero_byte_file", details=[str(download_path)]) + # 1. Dangerous files on disk if block_dangerous: dangerous_files = scan_directory_for_dangerous_files(download_path) @@ -501,6 +577,8 @@ def _on_archive_scan_error(root: Path, exc: OSError) -> None: ) ) + inspected_archives: list[ZipArchiveSafetyReport] = [] + collect_evidence = download_path.is_file() for archive in archive_files: safety_report = inspect_zip_archive_safety( archive, @@ -542,7 +620,11 @@ def _on_archive_scan_error(root: Path, exc: OSError) -> None: details=safety_report.dangerous_entries, ) + if collect_evidence: + inspected_archives.append(safety_report) + log.debug("file_safety_checks_passed") + return FileSafetyInspection(archives=tuple(inspected_archives)) async def check_download_safety( diff --git a/src/pullbox/core/filesystem_policy.py b/src/pullbox/core/filesystem_policy.py new file mode 100644 index 00000000..dd66f71e --- /dev/null +++ b/src/pullbox/core/filesystem_policy.py @@ -0,0 +1,74 @@ +"""Shared import path-text and sensitive-directory policy.""" + +import unicodedata +from pathlib import Path + +_SUPPORTED_PATH_FORMAT_CHARACTERS = frozenset( + "\u00ad\u061c\u200b\u200c\u200d\u200e\u200f\u2060\ufeff" +) +_MAX_PATH_TEXT_LENGTH = 4096 +_BLOCKED_DIRS = ("/etc", "/proc", "/sys", "/dev", "/run", "/boot", "/root", "/var/log", "/var/run") +BLOCKED_DIRECTORY_PREFIXES = frozenset( + prefix for directory in _BLOCKED_DIRS for prefix in (directory, str(Path(directory).resolve())) +) + + +def is_invalid_path_text(value: str) -> bool: + """Accept literal multilingual paths without weakening containment checks. + + Existing filenames may contain nonbreaking spaces, soft hyphens, joiners, + zero-width spaces, or Arabic/left-to-right/right-to-left marks. Preserve + those characters exactly; stripping them could select a different file. + Other non-printable characters, including controls, surrogates, line/paragraph + separators, and bidi embeddings/overrides/isolates, remain rejected. + Callers must still resolve paths and check root containment separately. + """ + if not value or len(value) > _MAX_PATH_TEXT_LENGTH: + return True + if value.isprintable(): + return False + return any( + not character.isprintable() + and character not in _SUPPORTED_PATH_FORMAT_CHARACTERS + and unicodedata.category(character) != "Zs" + for character in value + ) + + +def is_sensitive_path(path: Path) -> bool: + """Check an already-resolved path against the browser's system-directory policy.""" + for prefix in BLOCKED_DIRECTORY_PREFIXES: + blocked = Path(prefix) + if path == blocked or blocked in path.parents: + return True + # resolve() retains directory casing on case-insensitive filesystems. + # Probe identity only for case aliases, not every sampled comic path. + ancestor = Path(*path.parts[: len(blocked.parts)]) + if str(ancestor).casefold() == str(blocked).casefold(): + try: + if ancestor.samefile(blocked): + return True + except OSError: + return True + return False + + +def resolve_preview_source(source: str | Path) -> Path: + """Reject unsafe preview sources without redirecting the scan to another root. + + External import folders are allowed; only in-place adoption requires an + enabled library root. Validation is repeated by analyzers before use. + """ + raw = str(source) + if is_invalid_path_text(raw) or ".." in Path(raw).parts: + raise ValueError("Import preview source contains unsafe path components") + try: + # Resolution only probes the path. Sensitive aliases are rejected below + # before either analyzer can enumerate a directory or open a database. + # codeql[py/path-injection] + resolved = Path(raw).expanduser().resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ValueError("Import preview source is unavailable") from exc + if is_sensitive_path(resolved): + raise ValueError("Import preview cannot inspect sensitive system directories") + return resolved diff --git a/src/pullbox/core/import_resources.py b/src/pullbox/core/import_resources.py new file mode 100644 index 00000000..0857ebb9 --- /dev/null +++ b/src/pullbox/core/import_resources.py @@ -0,0 +1,169 @@ +"""Conservative container-aware budgets for read-only import inspection.""" + +from __future__ import annotations + +import asyncio +import math +import os +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +import psutil + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Iterable + +_MIB = 1024**2 + + +@dataclass(frozen=True) +class ImportResources: + cpu_count: int + available_memory_bytes: int + + def inspection_workers(self, *, requested: int = 0) -> int: + """Reserve CPU and memory for requests and other running services.""" + cpu_budget = max(1, self.cpu_count - math.ceil(self.cpu_count / 4)) + memory_budget = max(1, (self.available_memory_bytes - 512 * _MIB) // (512 * _MIB)) + ceiling = max(1, min(cpu_budget, memory_budget, 16)) + # Local SSD measurements show no gain from saturating the CPU with ZIP parsing. + return min(ceiling, requested) if requested > 0 else min(ceiling, 4) + + +def _visible_cpus() -> int: + try: + affinity_reader = getattr(psutil.Process(), "cpu_affinity", None) + affinity = affinity_reader() if affinity_reader is not None else [] + if affinity: + return len(affinity) + except (AttributeError, OSError, psutil.Error): + pass + return os.cpu_count() or 1 + + +def _available_memory() -> int: + try: + return int(psutil.virtual_memory().available) + except (OSError, psutil.Error): + return 0 + + +def _read(path: Path) -> str: + try: + return path.read_text().strip() + except (OSError, UnicodeError): + return "" + + +def _positive_integer(value: str) -> int | None: + try: + parsed = int(value) + except ValueError: + return None + return parsed if parsed >= 0 else None + + +def detect_import_resources( + *, cgroup_path: Path | None = None, cgroup_root: Path = Path("/sys/fs/cgroup") +) -> ImportResources: + """Use visible resources, honoring cgroup v2 ancestors and v1 limits.""" + cpus = max(1, _visible_cpus()) + memory = max(0, _available_memory()) + if cgroup_path is None: + cgroup_path = cgroup_root + for line in _read(Path("/proc/self/cgroup")).splitlines(): + if line.startswith("0::"): + candidate = cgroup_root / line[3:].lstrip("/") + if ".." not in candidate.parts and candidate.is_dir(): + cgroup_path = candidate + break + groups = [cgroup_path] + if cgroup_path.is_relative_to(cgroup_root): + groups.extend( + parent for parent in cgroup_path.parents if parent.is_relative_to(cgroup_root) + ) + for group in groups: + quota = _read(group / "cpu.max").split() + if len(quota) == 2: + limit, period = (_positive_integer(value) for value in quota) + if limit is not None and period: + cpus = min(cpus, max(1, limit // period)) + maximum = _positive_integer(_read(group / "memory.max")) + used = _positive_integer(_read(group / "memory.current")) + if maximum is not None: + memory = min(memory, max(0, maximum - used)) if used is not None else 0 + # Common v1 container mounts. Affinity also accounts for cpuset restrictions. + quota_v1 = _positive_integer(_read(cgroup_root / "cpu/cpu.cfs_quota_us")) + period_v1 = _positive_integer(_read(cgroup_root / "cpu/cpu.cfs_period_us")) + if quota_v1 is not None and period_v1: + cpus = min(cpus, max(1, quota_v1 // period_v1)) + maximum_v1 = _positive_integer(_read(cgroup_root / "memory/memory.limit_in_bytes")) + used_v1 = _positive_integer(_read(cgroup_root / "memory/memory.usage_in_bytes")) + if maximum_v1 is not None: + memory = min(memory, max(0, maximum_v1 - used_v1)) if used_v1 is not None else 0 + return ImportResources(cpus, memory) + + +@asynccontextmanager +async def bounded_thread_map[Input, Output]( + function: Callable[[Input], Output], values: Iterable[Input], *, workers: int +) -> AsyncIterator[AsyncIterator[Output]]: + """Run bounded filesystem work without passing a database session to threads.""" + + async def run(value: Input) -> Output: + return await asyncio.to_thread(function, value) + + async with bounded_async_map(run, values, workers=workers) as results: + yield results + + +@asynccontextmanager +async def bounded_async_map[Input, Output]( + function: Callable[[Input], Awaitable[Output]], values: Iterable[Input], *, workers: int +) -> AsyncIterator[AsyncIterator[Output]]: + """Yield completed work with bounded submission and drain active work on exit. + + Canceling an asyncio wrapper cannot stop a running filesystem operation. + Keep ownership until it finishes, including when the consumer raises or exits. + """ + source = iter(values) + tasks: set[asyncio.Task[Output]] = set() + + async def run(item: Input) -> Output: + return await function(item) + + async def results() -> AsyncGenerator[Output, None]: + exhausted = False + while tasks or not exhausted: + while not exhausted and len(tasks) < max(1, workers): + try: + value = next(source) + except StopIteration: + exhausted = True + else: + tasks.add(asyncio.create_task(run(value))) + if not tasks: + break + done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for task in done: + result = task.result() + tasks.remove(task) + yield result + + iterator = results() + try: + yield iterator + finally: + await iterator.aclose() + if tasks: + drain = asyncio.gather(*tasks, return_exceptions=True) + interrupted = False + while not drain.done(): + try: + await asyncio.shield(drain) + except asyncio.CancelledError: + interrupted = True + if interrupted: + raise asyncio.CancelledError diff --git a/src/pullbox/core/issue_numbers.py b/src/pullbox/core/issue_numbers.py new file mode 100644 index 00000000..826404f6 --- /dev/null +++ b/src/pullbox/core/issue_numbers.py @@ -0,0 +1,95 @@ +"""Canonical issue-number rendering shared across application boundaries.""" + +from __future__ import annotations + +import math +import re +from decimal import Decimal, InvalidOperation + +_MAX_ISSUE_NUMBER_TEXT_LENGTH = 320 +_NUMERIC_SUFFIX_PATTERN = re.compile(r"^([+-]?(?:\d+(?:\.\d*)?|\.\d+))([A-Za-z]+)$") + + +def _format_decimal(value: Decimal) -> str: + """Render a finite decimal without exponent or insignificant zeros.""" + if value == 0: + return "0" + rendered = format(value, "f") + if "." in rendered: + rendered = rendered.rstrip("0").rstrip(".") + return rendered + + +def format_issue_number(value: float | int) -> str: + """Render an issue number without scientific notation or trailing zeros.""" + try: + decimal_value = Decimal(str(value)) + except InvalidOperation: + return str(value) + if not decimal_value.is_finite(): + return str(value) + return _format_decimal(decimal_value) + + +def _issue_number_parts(value: str | float | int) -> tuple[Decimal, str]: + raw_value = str(value).strip() + if not raw_value: + raise ValueError("issue number must not be blank") + if len(raw_value) > _MAX_ISSUE_NUMBER_TEXT_LENGTH: + raise ValueError("issue number exceeds the supported exact-text length") + + normalized_fraction = raw_value.replace("½", ".5").replace("¼", ".25").replace("¾", ".75") + suffix_start = len(normalized_fraction) + while suffix_start > 0: + character = normalized_fraction[suffix_start - 1] + if not character.isascii() or not character.isalpha(): + break + suffix_start -= 1 + if suffix_start < len(normalized_fraction): + numeric_prefix = normalized_fraction[:suffix_start].rstrip() + if numeric_prefix and numeric_prefix[-1] in "0123456789.": + normalized_fraction = numeric_prefix + normalized_fraction[suffix_start:] + suffix = "" + try: + numeric_value = Decimal(normalized_fraction) + except InvalidOperation: + match = _NUMERIC_SUFFIX_PATTERN.fullmatch(normalized_fraction) + if match is None: + raise ValueError(f"invalid issue number: {raw_value!r}") from None + numeric_value = Decimal(match.group(1)) + suffix = match.group(2).upper() + + if not numeric_value.is_finite(): + raise ValueError(f"invalid issue number: {raw_value!r}") + + exact_text = f"{_format_decimal(numeric_value)}{suffix}" + if len(exact_text) > _MAX_ISSUE_NUMBER_TEXT_LENGTH: + raise ValueError("issue number exceeds the supported exact-text length") + return numeric_value, exact_text + + +def normalize_issue_number_text(value: str | float | int) -> str: + """Normalize an exact numeric issue designation while preserving a suffix.""" + return _issue_number_parts(value)[1] + + +def issue_number_text_matches_numeric( + issue_number: float | int, + issue_number_text: str, +) -> bool: + """Return whether exact text has the same numeric compatibility value.""" + exact_numeric_value, _ = _issue_number_parts(issue_number_text) + try: + numeric_value = Decimal(str(issue_number)) + except InvalidOperation: + return False + return numeric_value.is_finite() and numeric_value == exact_numeric_value + + +def parse_issue_number_text(value: str | float | int) -> tuple[float, str]: + """Return the numeric compatibility value and normalized exact designation.""" + numeric_value, exact_text = _issue_number_parts(value) + float_value = float(numeric_value) + if not math.isfinite(float_value): + raise ValueError("issue number exceeds the numeric compatibility range") + return float_value, exact_text diff --git a/src/pullbox/core/library_comicinfo.py b/src/pullbox/core/library_comicinfo.py index a614c9bc..c32d7124 100644 --- a/src/pullbox/core/library_comicinfo.py +++ b/src/pullbox/core/library_comicinfo.py @@ -13,6 +13,7 @@ from pullbox.core.archive import inspect_archive_page_count from pullbox.core.exceptions import ConfigurationError +from pullbox.core.issue_numbers import format_issue_number from pullbox.models.publisher import Publisher from pullbox.models.series import Series from pullbox.utilities.comicinfo import embed_comicinfo_in_cbz @@ -95,7 +96,7 @@ async def build_comicinfo_payload_for_issue( payload: dict[str, Any] = { "Series": series.title, - "Number": format_comicinfo_issue_number(issue.issue_number), + "Number": issue.effective_issue_number_text, "Title": issue.title, "Summary": issue.description, "Publisher": publisher_name, @@ -140,9 +141,7 @@ async def apply_comicinfo_to_imported_artifact( def format_comicinfo_issue_number(issue_number: float | None) -> str | None: if issue_number is None: return None - if float(issue_number).is_integer(): - return str(int(issue_number)) - return f"{issue_number:g}" + return format_issue_number(issue_number) def cleanup_prepared_paths(paths: list[Path]) -> None: diff --git a/src/pullbox/core/library_file_ownership.py b/src/pullbox/core/library_file_ownership.py new file mode 100644 index 00000000..61cc0ff8 --- /dev/null +++ b/src/pullbox/core/library_file_ownership.py @@ -0,0 +1,319 @@ +"""Ownership and path validation for managed and referenced library files.""" + +from __future__ import annotations + +import hashlib +import os +import re +from pathlib import Path +from typing import TYPE_CHECKING + +from sqlalchemy import or_, select + +from pullbox.core.exceptions import ConfigurationError, ValidationError +from pullbox.models.library import LibraryFile, LibraryFileStorageMode, LibraryRoot + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + + +_CONTROL_CHARACTER_RE = re.compile(r"[\x00-\x1f\x7f]") +_SIGNATURE_KEYS = ( + "schema_version", + "resolved_path", + "size", + "mtime_ns", + "device", + "inode", +) +_MANAGED_PLACEMENT_DIGEST_ALGORITHM = "sha256" +_MANAGED_PLACEMENT_DIGEST_CHUNK_BYTES = 1024 * 1024 + + +class ReferencedFileValidationError(ConfigurationError): + """Referenced-file validation failure with a stable review reason.""" + + def __init__(self, reason: str, message: str) -> None: + self.reason = reason + super().__init__(message) + + +class ReferencedFileMutationError(ValidationError): + """Raised when an operation would mutate a user-owned referenced artifact.""" + + +async def _reference_capable_roots( + session: AsyncSession, + explicit_root_id: int | None, +) -> list[LibraryRoot]: + """Load only enabled roots authorized to register referenced files.""" + if explicit_root_id is not None: + root = await session.get(LibraryRoot, explicit_root_id) + if root is None: + raise ConfigurationError("Selected library root does not exist.") + if not root.enabled: + raise ConfigurationError("Selected library root is disabled.") + if not root.allow_referenced_registrations: + raise ConfigurationError( + "Selected library root does not allow referenced registrations." + ) + return [root] + + return list( + ( + await session.execute( + select(LibraryRoot).where( + LibraryRoot.enabled.is_(True), + LibraryRoot.allow_referenced_registrations.is_(True), + ) + ) + ) + .scalars() + .all() + ) + + +async def referenced_library_files_for_target( + session: AsyncSession, + target: Path, + *, + include_descendants: bool, +) -> list[LibraryFile]: + """Return referenced records at a file path or below a folder path.""" + target_paths = { + str(target.expanduser().absolute()).rstrip("/"), + str(target.expanduser().resolve(strict=False)).rstrip("/"), + } + query = select(LibraryFile).where(LibraryFile.storage_mode == LibraryFileStorageMode.REFERENCED) + if include_descendants: + conditions: list[ColumnElement[bool]] = [] + for target_path in target_paths: + conditions.extend( + ( + LibraryFile.file_path == target_path, + LibraryFile.file_path.startswith(f"{target_path}/", autoescape=True), + ) + ) + query = query.where(or_(*conditions)) + else: + query = query.where(LibraryFile.file_path.in_(target_paths)) + return list((await session.execute(query)).scalars().all()) + + +async def require_mutable_library_target( + session: AsyncSession, + target: Path, + *, + include_descendants: bool, + operation: str, +) -> None: + """Reject mutations that include any referenced library file.""" + referenced_files = await referenced_library_files_for_target( + session, + target, + include_descendants=include_descendants, + ) + if referenced_files: + raise ReferencedFileMutationError( + f"Referenced library files cannot be {operation}. They must stay unchanged on disk." + ) + + +def build_file_identity_signature(path: Path) -> dict[str, int | str]: + """Capture the portable scan/execution identity used by library files.""" + resolved = path.expanduser().resolve(strict=True) + if not resolved.is_file(): + raise ConfigurationError("Library path must be an existing file.") + stat_result = resolved.stat() + return { + "schema_version": 1, + "resolved_path": str(resolved), + "size": stat_result.st_size, + "mtime_ns": stat_result.st_mtime_ns, + "device": stat_result.st_dev, + "inode": stat_result.st_ino, + } + + +def build_managed_placement_signature(path: Path) -> dict[str, int | str]: + """Capture identity plus content proof for one newly managed placement. + + General library scans intentionally use the stat-only identity signature. + Import publication calls this narrower helper only after it has created a + managed destination whose later cleanup or rollback must prove ownership. + """ + resolved = path.expanduser().resolve(strict=True) + digest = hashlib.sha256() + with resolved.open("rb") as stream: + before = os.fstat(stream.fileno()) + while chunk := stream.read(_MANAGED_PLACEMENT_DIGEST_CHUNK_BYTES): + digest.update(chunk) + after = os.fstat(stream.fileno()) + identity_keys = ("st_size", "st_mtime_ns", "st_dev", "st_ino") + if any(getattr(before, key) != getattr(after, key) for key in identity_keys): + raise ConfigurationError("Managed placement changed while its ownership proof was built.") + final_stat = resolved.stat() + if any(getattr(after, key) != getattr(final_stat, key) for key in identity_keys): + raise ConfigurationError("Managed placement path changed while ownership was recorded.") + signature: dict[str, int | str] = { + "schema_version": 1, + "resolved_path": str(resolved), + "size": after.st_size, + "mtime_ns": after.st_mtime_ns, + "device": after.st_dev, + "inode": after.st_ino, + } + signature["content_digest_algorithm"] = _MANAGED_PLACEMENT_DIGEST_ALGORITHM + signature["content_digest"] = digest.hexdigest() + return signature + + +def validate_file_identity_signature( + expected: dict[str, object], + current: dict[str, int | str], +) -> None: + """Fail closed when a referenced file no longer matches its scan evidence.""" + if not expected or any(key not in expected for key in _SIGNATURE_KEYS): + raise ReferencedFileValidationError( + "source_signature_missing", + "Referenced file is missing the scan evidence required for in-place import.", + ) + if expected.get("schema_version") != 1: + raise ReferencedFileValidationError( + "source_signature_unsupported", + "Referenced file uses an unsupported scan-evidence version.", + ) + if any(expected.get(key) != current.get(key) for key in _SIGNATURE_KEYS): + raise ReferencedFileValidationError( + "source_changed", + "Referenced file changed after it was scanned. Rescan before importing it in place.", + ) + + +async def resolve_referenced_library_root( + session: AsyncSession, + source_path: Path, + explicit_root_id: int | None, +) -> tuple[LibraryRoot, Path, dict[str, int | str]]: + """Resolve a file inside one unambiguous enabled reference-capable root.""" + raw_path = str(source_path) + if _CONTROL_CHARACTER_RE.search(raw_path) or ".." in source_path.parts: + raise ConfigurationError("Referenced library path contains unsafe path components.") + + try: + lexical_source = source_path.expanduser().absolute() + resolved_source = source_path.expanduser().resolve(strict=True) + except (OSError, RuntimeError, ValueError) as exc: + raise ConfigurationError("Referenced library path is unavailable.") from exc + + if not resolved_source.is_file(): + raise ConfigurationError("Referenced library path must be an existing file.") + if not os.access(resolved_source, os.R_OK): + raise ConfigurationError("Referenced library file is not readable by Pullbox.") + + roots = await _reference_capable_roots(session, explicit_root_id) + + candidates: list[LibraryRoot] = [] + for root in roots: + try: + lexical_root = Path(root.path).expanduser().absolute() + resolved_root = Path(root.path).expanduser().resolve(strict=True) + except (OSError, RuntimeError, ValueError): + continue + lexical_inside = lexical_source == lexical_root or lexical_source.is_relative_to( + lexical_root + ) + resolved_inside = resolved_source == resolved_root or resolved_source.is_relative_to( + resolved_root + ) + if lexical_inside and resolved_inside: + candidates.append(root) + + if not candidates: + raise ConfigurationError( + "Referenced files must be inside an enabled library root that allows " + "referenced registrations." + ) + if len(candidates) != 1: + raise ReferencedFileValidationError( + "source_root_ambiguous", + "Referenced library file matches multiple enabled reference-capable library roots.", + ) + + root = candidates[0] + return root, resolved_source, build_file_identity_signature(resolved_source) + + +async def resolve_referenced_source_root( + session: AsyncSession, + source_path: Path, + explicit_root_id: int | None, +) -> tuple[LibraryRoot, Path]: + """Resolve a source directory inside one unambiguous reference-capable root.""" + raw_path = str(source_path) + if _CONTROL_CHARACTER_RE.search(raw_path) or ".." in source_path.parts: + raise ReferencedFileValidationError( + "source_path_unsafe", + "In-place import source contains unsafe path components.", + ) + + try: + # These probes only normalize the candidate. Both lexical and resolved + # enabled-root containment are required below before accepting a source. + # codeql[py/path-injection] + lexical_source = source_path.expanduser().absolute() + # codeql[py/path-injection] + resolved_source = source_path.expanduser().resolve(strict=True) + except (OSError, RuntimeError, ValueError) as exc: + raise ReferencedFileValidationError( + "source_missing", + "In-place import source is unavailable.", + ) from exc + + if not resolved_source.is_dir(): + raise ReferencedFileValidationError( + "source_missing", + "In-place import source must be an existing directory.", + ) + if not os.access(resolved_source, os.R_OK | os.X_OK): + raise ReferencedFileValidationError( + "source_unreadable", + "In-place import source is not readable by Pullbox.", + ) + + try: + roots = await _reference_capable_roots(session, explicit_root_id) + except ConfigurationError as exc: + raise ReferencedFileValidationError("source_outside_root", exc.message) from exc + + candidates: list[LibraryRoot] = [] + for root in roots: + try: + lexical_root = Path(root.path).expanduser().absolute() + resolved_root = Path(root.path).expanduser().resolve(strict=True) + except (OSError, RuntimeError, ValueError): + continue + lexical_inside = lexical_source == lexical_root or lexical_source.is_relative_to( + lexical_root + ) + resolved_inside = resolved_source == resolved_root or resolved_source.is_relative_to( + resolved_root + ) + if lexical_inside and resolved_inside: + candidates.append(root) + + if not candidates: + raise ReferencedFileValidationError( + "source_outside_root", + "In-place import source must be inside an enabled library root that allows " + "referenced registrations.", + ) + if len(candidates) != 1: + raise ReferencedFileValidationError( + "source_root_ambiguous", + "In-place import source matches multiple enabled reference-capable library roots.", + ) + + root = candidates[0] + return root, resolved_source diff --git a/src/pullbox/core/library_layout.py b/src/pullbox/core/library_layout.py new file mode 100644 index 00000000..a9b9c4fa --- /dev/null +++ b/src/pullbox/core/library_layout.py @@ -0,0 +1,538 @@ +"""Versioned, data-only source layout templates for collection imports. + +The grammar intentionally accepts only registered semantic tokens. User input +is escaped before the matcher is compiled; it never becomes executable regex, +code, shell syntax, or an absolute filesystem path. +""" + +from __future__ import annotations + +import enum +import re +import unicodedata +from dataclasses import dataclass, replace +from decimal import Decimal, InvalidOperation +from pathlib import PurePosixPath + +from pullbox.core.source_metadata import SourceMetadataExtractor + +LAYOUT_SCHEMA_VERSION = 1 +MAX_LAYOUT_TEMPLATE_BYTES = 1024 +MAX_LAYOUT_SEGMENT_BYTES = 255 +MAX_LAYOUT_PATH_BYTES = 4096 + +_TOKEN_RE = re.compile(r"\{(?P[A-Za-z][A-Za-z0-9]*)(?::(?P[^{}]+))?\}") +_ISSUE_RE = re.compile(r"(?P[+-]?\d+(?:\.\d+)?)(?P[A-Za-z]*)") +_SERIES_YEAR_RE = re.compile( + r"^(?P.+?)\s*(?:[\[(](?P(?:19|20)\d{2})[\])]|" + r"(?P(?:19|20)\d{2}))\s*$" +) +_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f]") +_RESERVED_WINDOWS_NAMES = frozenset( + { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{number}" for number in range(1, 10)), + *(f"LPT{number}" for number in range(1, 10)), + } +) + +_PATH_TOKENS = frozenset({"Publisher", "Series", "Year", "ComicVineId", "Type"}) +_ISSUE_TOKENS = frozenset({"Publisher", "Series", "Year", "Issue", "IssueTitle", "Title", "Type"}) +_FREE_TEXT_TOKENS = frozenset({"Publisher", "Series", "IssueTitle", "Type"}) + + +class ImportLayoutMode(enum.StrEnum): + """How Pullbox chooses a source layout matcher.""" + + AUTO = "auto" + PRESET = "preset" + CUSTOM = "custom" + + +class LayoutClassification(enum.StrEnum): + """Request-local classification of a source or layout cluster.""" + + NORMAL_LIBRARY = "normal_library" + STORY_ARC = "story_arc" + MIXED = "mixed" + NEEDS_REVIEW = "needs_review" + + +class LayoutTemplateError(ValueError): + """A source-layout specification is unsupported, unsafe, or ambiguous.""" + + +class LayoutValueError(ValueError): + """A matched semantic value is unsafe to expose as layout evidence.""" + + +@dataclass(frozen=True, slots=True) +class SourceLayoutSpec: + """Serializable source-layout selection stored by a later import snapshot.""" + + schema_version: int = LAYOUT_SCHEMA_VERSION + mode: ImportLayoutMode = ImportLayoutMode.AUTO + preset: str | None = None + series_path_template: str | None = None + issue_filename_template: str | None = None + selected_cluster_id: str | None = None + fallback_to_auto: bool = True + + def to_dict(self) -> dict[str, object]: + """Return a deterministic JSON-compatible representation.""" + return { + "schema_version": self.schema_version, + "mode": self.mode.value, + "preset": self.preset, + "series_path_template": self.series_path_template, + "issue_filename_template": self.issue_filename_template, + "selected_cluster_id": self.selected_cluster_id, + "fallback_to_auto": self.fallback_to_auto, + } + + @classmethod + def from_dict(cls, value: dict[str, object]) -> SourceLayoutSpec: + """Restore a layout spec from its JSON-compatible representation.""" + try: + mode = ImportLayoutMode(str(value.get("mode", ImportLayoutMode.AUTO.value))) + except ValueError as exc: + raise LayoutTemplateError("Unknown source layout mode") from exc + return cls( + schema_version=_required_int( + value.get("schema_version", LAYOUT_SCHEMA_VERSION), + field_name="schema_version", + ), + mode=mode, + preset=_optional_string(value.get("preset")), + series_path_template=_optional_string(value.get("series_path_template")), + issue_filename_template=_optional_string(value.get("issue_filename_template")), + selected_cluster_id=_optional_string(value.get("selected_cluster_id")), + fallback_to_auto=_required_bool( + value.get("fallback_to_auto", True), + field_name="fallback_to_auto", + ), + ) + + +@dataclass(frozen=True, slots=True) +class SourceLayoutMatch: + """Semantic evidence decoded from one root-relative comic path.""" + + relative_path: str + publisher: str | None = None + series: str | None = None + year: int | None = None + issue_number: str | None = None + issue_title: str | None = None + comicvine_id: int | None = None + issue_type: str | None = None + + +@dataclass(frozen=True, slots=True) +class _TemplateToken: + semantic_name: str + capture_name: str + value_pattern: str + numeric_width: int | None = None + + +@dataclass(frozen=True, slots=True) +class _CompiledSegment: + pattern: re.Pattern[str] + pattern_text: str + tokens: tuple[_TemplateToken, ...] + + def match( + self, + value: str, + existing_captures: dict[str, list[str]] | None = None, + ) -> dict[str, list[str]] | None: + pattern = self.pattern + if existing_captures: + constrained_pattern = self.pattern_text + constrained = False + for token in self.tokens: + existing_values = existing_captures.get(token.semantic_name) + if token.semantic_name not in _FREE_TEXT_TOKENS or not existing_values: + continue + original = f"(?P<{token.capture_name}>{token.value_pattern})" + replacement = ( + f"(?P<{token.capture_name}>{_escaped_literal_pattern(existing_values[0])})" + ) + constrained_pattern = constrained_pattern.replace(original, replacement, 1) + constrained = True + if constrained: + pattern = re.compile(constrained_pattern, flags=re.IGNORECASE) + + matched = pattern.fullmatch(value) + if matched is None: + return None + captures: dict[str, list[str]] = {} + for token in self.tokens: + raw_value = matched.group(token.capture_name) + normalized = _normalize_token_value(token.semantic_name, raw_value) + captures.setdefault(token.semantic_name, []).append(normalized) + return captures + + +@dataclass(frozen=True, slots=True) +class CompiledSourceLayout: + """A bounded matcher compiled from a validated source-layout spec.""" + + spec: SourceLayoutSpec + path_segments: tuple[_CompiledSegment, ...] + filename_segment: _CompiledSegment | None + + def match(self, relative_path: str | PurePosixPath) -> SourceLayoutMatch | None: + """Match one comic path relative to the selected import root.""" + raw_path = str(relative_path) + _validate_relative_input_path(raw_path) + path = PurePosixPath(raw_path) + if len(path.parts) < 2 or len(path.parts[:-1]) != len(self.path_segments): + return None + + captures: dict[str, list[str]] = {} + for compiled, value in zip(self.path_segments, path.parts[:-1], strict=True): + segment_captures = compiled.match(value, captures) + if segment_captures is None: + return None + _extend_captures(captures, segment_captures) + + if self.filename_segment is not None: + filename_captures = self.filename_segment.match(path.stem, captures) + if filename_captures is None: + return None + _extend_captures(captures, filename_captures) + else: + metadata = SourceMetadataExtractor().from_release_title( + path.name, + folder_name=path.parent.name, + ) + if metadata.issue_number is not None: + captures.setdefault("Issue", []).append( + _normalize_issue_value(str(metadata.issue_number)) + ) + if metadata.year is not None: + captures.setdefault("Year", []).append(str(metadata.year)) + + agreed = _agreed_capture_values(captures) + if agreed is None: + return None + series = agreed.get("Series") + year = _optional_int(agreed.get("Year")) + if series is not None and year is None: + series, inferred_year = split_series_year(series) + year = inferred_year + + return SourceLayoutMatch( + relative_path=path.as_posix(), + publisher=agreed.get("Publisher"), + series=series, + year=year, + issue_number=agreed.get("Issue"), + issue_title=agreed.get("IssueTitle"), + comicvine_id=_optional_int(agreed.get("ComicVineId")), + issue_type=agreed.get("Type"), + ) + + +_PRESETS: dict[str, SourceLayoutSpec] = { + "series_folders": SourceLayoutSpec( + mode=ImportLayoutMode.PRESET, + preset="series_folders", + series_path_template="{Series}", + ), + "publisher_series": SourceLayoutSpec( + mode=ImportLayoutMode.PRESET, + preset="publisher_series", + series_path_template="{Publisher}/{Series}", + ), +} + + +def resolve_source_layout_spec(spec: SourceLayoutSpec) -> SourceLayoutSpec: + """Validate and normalize a registered or custom source layout.""" + if spec.schema_version != LAYOUT_SCHEMA_VERSION: + raise LayoutTemplateError( + f"Unsupported source layout schema version: {spec.schema_version}" + ) + if spec.selected_cluster_id is not None and ( + not spec.selected_cluster_id + or len(spec.selected_cluster_id.encode("utf-8")) > 128 + or _CONTROL_RE.search(spec.selected_cluster_id) + ): + raise LayoutTemplateError("Selected layout cluster ID is invalid") + + if spec.mode == ImportLayoutMode.AUTO: + if spec.preset or spec.series_path_template or spec.issue_filename_template: + raise LayoutTemplateError("Automatic layout cannot include preset or custom templates") + return spec + + if spec.mode == ImportLayoutMode.PRESET: + if not spec.preset or spec.preset not in _PRESETS: + raise LayoutTemplateError("Unknown source layout preset") + registered = _PRESETS[spec.preset] + if spec.series_path_template not in {None, registered.series_path_template}: + raise LayoutTemplateError("Preset path template does not match the registry") + if spec.issue_filename_template not in {None, registered.issue_filename_template}: + raise LayoutTemplateError("Preset filename template does not match the registry") + return replace( + spec, + series_path_template=registered.series_path_template, + issue_filename_template=registered.issue_filename_template, + ) + + if spec.mode != ImportLayoutMode.CUSTOM: + raise LayoutTemplateError("Unknown source layout mode") + if spec.preset is not None: + raise LayoutTemplateError("Custom layout cannot include a preset") + if not spec.series_path_template: + raise LayoutTemplateError("Custom layout requires a series path template") + return spec + + +def compile_source_layout(spec: SourceLayoutSpec) -> CompiledSourceLayout: + """Compile a preset or custom spec into escaped, deterministic matchers.""" + effective = resolve_source_layout_spec(spec) + if effective.mode == ImportLayoutMode.AUTO or effective.series_path_template is None: + raise LayoutTemplateError( + "Automatic layout is resolved per path and cannot be compiled once" + ) + + path_segments = _compile_path_template(effective.series_path_template) + filename_segment = ( + _compile_filename_template(effective.issue_filename_template) + if effective.issue_filename_template is not None + else None + ) + return CompiledSourceLayout( + spec=effective, + path_segments=path_segments, + filename_segment=filename_segment, + ) + + +def split_series_year(value: str) -> tuple[str, int | None]: + """Split supported inline or bracketed year suffixes from a series segment.""" + matched = _SERIES_YEAR_RE.fullmatch(value.strip()) + if matched is None: + return value.strip(), None + year_text = matched.group("bracketed_year") or matched.group("bare_year") + return matched.group("series").strip(), int(year_text) + + +def registered_source_layout_presets() -> tuple[SourceLayoutSpec, ...]: + """Return registered presets in stable identifier order.""" + return tuple(_PRESETS[key] for key in sorted(_PRESETS)) + + +def _compile_path_template(template: str) -> tuple[_CompiledSegment, ...]: + _validate_template_size(template) + if template.startswith(("/", "\\")) or re.match(r"^[A-Za-z]:", template): + raise LayoutTemplateError("Source layout paths must be root-relative") + if "\\" in template: + raise LayoutTemplateError("Source layout paths must use forward-slash separators") + parts = template.split("/") + if any(part in {"", ".", ".."} for part in parts): + raise LayoutTemplateError("Source layout contains an empty or unsafe path segment") + return tuple(_compile_segment(part, allowed_tokens=_PATH_TOKENS) for part in parts) + + +def _compile_filename_template(template: str) -> _CompiledSegment: + if re.search(r"\.(?:cbz|cbr|cb7|cbt|pdf|epub)$", template, flags=re.IGNORECASE): + raise LayoutTemplateError("Comic extensions are handled separately from layout tokens") + return _compile_segment(template, allowed_tokens=_ISSUE_TOKENS) + + +def _compile_segment(template: str, *, allowed_tokens: frozenset[str]) -> _CompiledSegment: + _validate_template_size(template) + if len(template.encode("utf-8")) > MAX_LAYOUT_SEGMENT_BYTES: + raise LayoutTemplateError("A layout segment exceeds the supported byte length") + if "/" in template or "\\" in template: + raise LayoutTemplateError("A layout segment cannot contain a path separator") + if _CONTROL_RE.search(template): + raise LayoutTemplateError("A layout template cannot contain control characters") + + pattern_parts: list[str] = [] + tokens: list[_TemplateToken] = [] + cursor = 0 + previous_was_token = False + for index, matched in enumerate(_TOKEN_RE.finditer(template)): + literal = template[cursor : matched.start()] + if "{" in literal or "}" in literal: + raise LayoutTemplateError("A layout template contains an invalid token") + if previous_was_token and not literal: + raise LayoutTemplateError("Adjacent semantic tokens are ambiguous") + pattern_parts.append(_escaped_literal_pattern(literal)) + + raw_name = matched.group("name") + semantic_name = "IssueTitle" if raw_name == "Title" else raw_name + if raw_name not in allowed_tokens: + raise LayoutTemplateError(f"Unknown or unsupported layout token: {raw_name}") + raw_format = matched.group("format") + numeric_width = _validate_token_format(semantic_name, raw_format) + capture_name = f"layout_value_{index}" + value_pattern = _token_value_pattern(semantic_name) + pattern_parts.append(f"(?P<{capture_name}>{value_pattern})") + tokens.append( + _TemplateToken( + semantic_name=semantic_name, + capture_name=capture_name, + value_pattern=value_pattern, + numeric_width=numeric_width, + ) + ) + cursor = matched.end() + previous_was_token = True + + trailing = template[cursor:] + if "{" in trailing or "}" in trailing: + raise LayoutTemplateError("A layout template contains an invalid token") + pattern_parts.append(_escaped_literal_pattern(trailing)) + if not tokens: + raise LayoutTemplateError("A layout template must include at least one semantic token") + + pattern_text = "".join(pattern_parts) + return _CompiledSegment( + pattern=re.compile(pattern_text, flags=re.IGNORECASE), + pattern_text=pattern_text, + tokens=tuple(tokens), + ) + + +def _validate_template_size(template: str) -> None: + if not template or len(template.encode("utf-8")) > MAX_LAYOUT_TEMPLATE_BYTES: + raise LayoutTemplateError("A layout template is empty or too long") + + +def _validate_token_format(semantic_name: str, raw_format: str | None) -> int | None: + if raw_format is None: + return None + if semantic_name != "Issue": + raise LayoutTemplateError(f"Formatting is not supported for {semantic_name}") + matched = re.fullmatch(r"0?(?P[1-9])d", raw_format) + if matched is None: + raise LayoutTemplateError("Issue format must be a decimal width such as 02d or 03d") + return int(matched.group("width")) + + +def _token_value_pattern(semantic_name: str) -> str: + if semantic_name == "Issue": + value_pattern = r"[+-]?\d+(?:\.\d+)?[A-Za-z]*" + elif semantic_name == "Year": + value_pattern = r"(?:19|20)\d{2}" + elif semantic_name == "ComicVineId": + value_pattern = r"\d+" + elif semantic_name in _FREE_TEXT_TOKENS: + value_pattern = r".+?" + else: + raise LayoutTemplateError(f"Unsupported semantic token: {semantic_name}") + return value_pattern + + +def _escaped_literal_pattern(value: str) -> str: + parts: list[str] = [] + cursor = 0 + for matched in re.finditer(r"\s+", value): + parts.append(re.escape(value[cursor : matched.start()])) + parts.append(r"\s+") + cursor = matched.end() + parts.append(re.escape(value[cursor:])) + return "".join(parts) + + +def _normalize_token_value(semantic_name: str, raw_value: str) -> str: + value = unicodedata.normalize("NFC", raw_value.strip()) + if not value: + raise LayoutValueError(f"{semantic_name} cannot be empty") + if len(value.encode("utf-8")) > MAX_LAYOUT_SEGMENT_BYTES: + raise LayoutValueError(f"{semantic_name} exceeds the supported byte length") + if _CONTROL_RE.search(value) or "/" in value or "\\" in value: + raise LayoutValueError(f"{semantic_name} contains an unsafe character") + reserved_base = value.rstrip(" .").split(".", 1)[0].upper() + if reserved_base in _RESERVED_WINDOWS_NAMES: + raise LayoutValueError(f"{semantic_name} is a reserved filesystem name") + if semantic_name == "Issue": + return _normalize_issue_value(value) + if semantic_name in {"Year", "ComicVineId"}: + return str(int(value)) + return re.sub(r"\s+", " ", value) + + +def _normalize_issue_value(value: str) -> str: + matched = _ISSUE_RE.fullmatch(value.strip()) + if matched is None: + raise LayoutValueError("Issue is not a supported number") + try: + number = Decimal(matched.group("number")) + except InvalidOperation as exc: + raise LayoutValueError("Issue is not a supported number") from exc + normalized = format(number, "f") + if "." in normalized: + normalized = normalized.rstrip("0").rstrip(".") + if normalized == "-0": + normalized = "0" + return f"{normalized}{matched.group('suffix')}" + + +def _agreed_capture_values(captures: dict[str, list[str]]) -> dict[str, str] | None: + agreed: dict[str, str] = {} + for semantic_name, values in captures.items(): + first = values[0] + comparison = _comparison_value(semantic_name, first) + if any(_comparison_value(semantic_name, value) != comparison for value in values[1:]): + return None + agreed[semantic_name] = first + return agreed + + +def _comparison_value(semantic_name: str, value: str) -> str: + if semantic_name in {"Issue", "Year", "ComicVineId"}: + return value.casefold() + return re.sub(r"\s+", " ", value).casefold() + + +def _extend_captures( + target: dict[str, list[str]], + source: dict[str, list[str]], +) -> None: + for key, values in source.items(): + target.setdefault(key, []).extend(values) + + +def _validate_relative_input_path(raw_path: str) -> None: + if not raw_path or len(raw_path.encode("utf-8")) > MAX_LAYOUT_PATH_BYTES: + raise LayoutValueError("Relative source path is empty or too long") + if raw_path.startswith(("/", "\\")) or "\\" in raw_path: + raise LayoutValueError("Source path must be root-relative and use forward slashes") + if _CONTROL_RE.search(raw_path): + raise LayoutValueError("Source path contains a control character") + raw_parts = raw_path.split("/") + if any(part in {"", ".", ".."} for part in raw_parts): + raise LayoutValueError("Source path contains an unsafe segment") + + +def _optional_string(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def _optional_int(value: str | None) -> int | None: + return int(value) if value is not None else None + + +def _required_int(value: object, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, (int, str)): + raise LayoutTemplateError(f"{field_name} must be an integer") + try: + return int(value) + except ValueError as exc: + raise LayoutTemplateError(f"{field_name} must be an integer") from exc + + +def _required_bool(value: object, *, field_name: str) -> bool: + if not isinstance(value, bool): + raise LayoutTemplateError(f"{field_name} must be a boolean") + return value diff --git a/src/pullbox/core/library_leave_in_place.py b/src/pullbox/core/library_leave_in_place.py index 65c97992..d993456f 100644 --- a/src/pullbox/core/library_leave_in_place.py +++ b/src/pullbox/core/library_leave_in_place.py @@ -2,24 +2,20 @@ from __future__ import annotations -import asyncio -from pathlib import Path from typing import TYPE_CHECKING -import structlog - -from pullbox.core.library_naming import compute_target_filename, resolve_naming_issue_type -from pullbox.core.library_transfer import safe_move +from pullbox.core.exceptions import ConfigurationError +from pullbox.core.library_root_resolution import path_is_inside_root if TYPE_CHECKING: + from pathlib import Path + from sqlalchemy.ext.asyncio import AsyncSession from pullbox.core.library_policy import LibraryIngestPolicy from pullbox.models.issue import Issue from pullbox.models.library import LibraryRoot -logger = structlog.get_logger(__name__) - async def handle_leave_in_place( session: AsyncSession, @@ -30,32 +26,10 @@ async def handle_leave_in_place( ingest_policy: LibraryIngestPolicy, rename: bool, ) -> Path: - """Handle leave-in-place registration, with optional rename.""" - if not rename: - return source_path - - comics_dir = Path(root.path) - if not str(source_path).startswith(str(comics_dir)): - return source_path - - effective_issue_type = await resolve_naming_issue_type(session, issue) - new_name = compute_target_filename( - issue, - series, - source_path, - ingest_policy, - issue_type_override=effective_issue_type, - ) - target_path = source_path.parent / new_name - - if target_path == source_path: - return source_path - - await asyncio.to_thread(safe_move, source_path, target_path) - - logger.info( - "file_renamed_in_place", - source=str(source_path), - destination=str(target_path), - ) - return target_path + """Return a contained source path without ever mutating a referenced file.""" + _ = session, issue, series, ingest_policy + if not root.enabled or not path_is_inside_root(source_path, root): + raise ConfigurationError("Referenced files must be inside an enabled library root.") + if rename: + raise ConfigurationError("Referenced library files cannot rename source files.") + return source_path.resolve(strict=True) diff --git a/src/pullbox/core/library_naming.py b/src/pullbox/core/library_naming.py index 31f8a6f9..ee9e9e05 100644 --- a/src/pullbox/core/library_naming.py +++ b/src/pullbox/core/library_naming.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from typing import TYPE_CHECKING, Any, overload from sqlalchemy import func, select @@ -18,6 +19,27 @@ from pullbox.models.issue import Issue, IssueType, is_non_standard_issue_type from pullbox.models.series import Series +_SERIES_PATH_TOKEN_RE = re.compile(r"\{(?P[A-Za-z][A-Za-z0-9]*)\}") +_SERIES_PATH_TOKENS = frozenset({"Publisher", "Series", "Year", "ComicVineId", "Type"}) +_MAX_SERIES_PATH_TEMPLATE_BYTES = 1024 +_FILE_TEMPLATE_TOKEN_RE = re.compile(r"\{[^{}]+\}") +_FILE_TEMPLATE_TOKENS = frozenset( + { + "{Series}", + "{Year}", + "{Issue}", + "{Issue:03d}", + "{Volume}", + "{Volume:02d}", + "{Type}", + "{IssueTitle}", + "{Title}", + "{Publisher}", + "{Edition}", + } +) +_MAX_FILE_TEMPLATE_BYTES = 1024 + if TYPE_CHECKING: from pathlib import Path @@ -83,17 +105,42 @@ def build_naming_snapshot( "id": root.id, "path": root.path, }, + "root_policy": { + "id": getattr(naming_policy, "root_policy_id", None), + "source": str(getattr(naming_policy, "policy_source", "global_default")), + "revision": int(getattr(naming_policy, "policy_revision", 0)), + "source_import_job_id": getattr( + naming_policy, + "source_import_job_id", + None, + ), + }, "series": series_payload, "issue": { "id": issue.id, "comicvine_id": issue.comicvine_id, "issue_number": issue.issue_number, + "issue_number_text": issue.effective_issue_number_text, "title": issue.title, "raw_issue_type": raw_issue_type, "effective_issue_type": effective_issue_type, }, "template_key": template_key_for_issue_type(effective_issue_type), "templates": { + "series_path_template": policy_value( + naming_policy, + "series_path_template", + policy_value( + naming_policy, + "series_folder_template", + "{Series} ({Year})", + ), + ) + or policy_value( + naming_policy, + "series_folder_template", + "{Series} ({Year})", + ), "series_folder_template": policy_value( naming_policy, "series_folder_template", "{Series} ({Year})" ), @@ -157,6 +204,9 @@ async def resolve_naming_issue_type(session: AsyncSession, issue: Issue) -> str: def build_series_folder_name( series: object, naming_policy: LibraryIngestPolicy | dict[str, str], + *, + series_type_override: str | None = None, + comicvine_id_override: int | None = None, ) -> str: """Build a series folder name from a Series model and naming config.""" title = "" @@ -169,8 +219,14 @@ def build_series_folder_name( if isinstance(series, Series): title = series.title year = series.year_start - cv_id = series.comicvine_id - series_type_value = series.series_type.value if series.series_type else None + cv_id = comicvine_id_override or series.comicvine_id + series_type_value = ( + series_type_override + if series_type_override is not None + else series.series_type.value + if series.series_type + else None + ) if series.publisher is not None: publisher_name = series.publisher.name @@ -186,6 +242,84 @@ def build_series_folder_name( ) +def build_series_relative_path( + series: object, + naming_policy: LibraryIngestPolicy | dict[str, str], + *, + series_type_override: str | None = None, + comicvine_id_override: int | None = None, +) -> Path: + """Render and sanitize every segment in a root-relative series path.""" + from pathlib import Path + + template = policy_value( + naming_policy, + "series_path_template", + policy_value(naming_policy, "series_folder_template", "{Series} ({Year})"), + ) or policy_value(naming_policy, "series_folder_template", "{Series} ({Year})") + raw_segments = validate_series_path_template(template) + + rendered_segments = tuple( + build_series_folder_name( + series, + { + "series_folder_template": segment, + "replace_illegal_characters": str( + policy_bool(naming_policy, "replace_illegal_characters", True) + ), + "colon_replacement": policy_value( + naming_policy, + "colon_replacement", + "dash", + ), + }, + series_type_override=series_type_override, + comicvine_id_override=comicvine_id_override, + ) + for segment in raw_segments + ) + if any("/" in segment or "\\" in segment for segment in rendered_segments): + raise ValueError("Series path token rendered an unsafe path separator.") + if any(segment in {"", ".", ".."} for segment in rendered_segments): + raise ValueError("Series path template rendered an unsafe segment.") + return Path(*rendered_segments) + + +def validate_series_path_template(template: str) -> tuple[str, ...]: + """Validate a root-relative output template and return its raw segments.""" + if not template or len(template.encode("utf-8")) > _MAX_SERIES_PATH_TEMPLATE_BYTES: + raise ValueError("Series path template is empty or too long.") + if template.startswith(("/", "\\")) or "\\" in template: + raise ValueError("Series path template must be root-relative.") + + raw_segments = template.split("/") + if any(segment in {"", ".", ".."} for segment in raw_segments): + raise ValueError("Series path template contains an unsafe segment.") + + for matched in _SERIES_PATH_TOKEN_RE.finditer(template): + if matched.group("name") not in _SERIES_PATH_TOKENS: + raise ValueError(f"Unsupported series path token: {matched.group('name')}") + without_tokens = _SERIES_PATH_TOKEN_RE.sub("", template) + if "{" in without_tokens or "}" in without_tokens: + raise ValueError("Series path template contains an invalid token.") + return tuple(raw_segments) + + +def validate_library_file_template(template: str) -> None: + """Reject unsupported or path-producing tokens in a file-name template.""" + if not template or len(template.encode("utf-8")) > _MAX_FILE_TEMPLATE_BYTES: + raise ValueError("Library file template is empty or too long.") + if "/" in template or "\\" in template or any(ord(char) < 32 for char in template): + raise ValueError("Library file template must produce a single file name.") + + for token in _FILE_TEMPLATE_TOKEN_RE.findall(template): + if token not in _FILE_TEMPLATE_TOKENS: + raise ValueError(f"Unsupported library file token: {token}") + without_tokens = _FILE_TEMPLATE_TOKEN_RE.sub("", template) + if "{" in without_tokens or "}" in without_tokens: + raise ValueError("Library file template contains an invalid token.") + + def compute_target_filename( issue: Issue, series: object, @@ -239,7 +373,7 @@ def compute_target_filename( return format_comic_file( series=title, year=year, - issue=issue.issue_number, + issue=issue.effective_issue_number_text, volume=volume_number, issue_type=issue_type, title=issue.title, diff --git a/src/pullbox/core/library_policy.py b/src/pullbox/core/library_policy.py index aab15f48..4c12fb03 100644 --- a/src/pullbox/core/library_policy.py +++ b/src/pullbox/core/library_policy.py @@ -6,10 +6,12 @@ from __future__ import annotations -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field, replace from typing import TYPE_CHECKING, Any from pullbox.core.config_resolver import load_system_config_values, parse_bool +from pullbox.core.exceptions import ConfigurationError +from pullbox.models.library import LibraryRootPolicy, LibraryRootPolicySource if TYPE_CHECKING: from collections.abc import Mapping @@ -29,6 +31,14 @@ class LibraryNamingPolicy: single_non_standard_file_template: str replace_illegal_characters: bool colon_replacement: str + series_path_template: str = field(default="", kw_only=True) + policy_source: LibraryRootPolicySource = field( + default=LibraryRootPolicySource.GLOBAL_DEFAULT, + kw_only=True, + ) + root_policy_id: int | None = field(default=None, kw_only=True) + policy_revision: int = field(default=0, kw_only=True) + source_import_job_id: int | None = field(default=None, kw_only=True) @dataclass(frozen=True, slots=True) @@ -91,6 +101,20 @@ def library_ingest_policy_from_snapshot( ), replace_illegal_characters=parse_bool(snapshot["replace_illegal_characters"]), colon_replacement=_snapshot_text(snapshot, "colon_replacement"), + series_path_template=str( + snapshot["series_path_template"] + if "series_path_template" in snapshot + else snapshot["series_folder_template"] + ), + policy_source=LibraryRootPolicySource( + str(snapshot.get("policy_source", LibraryRootPolicySource.GLOBAL_DEFAULT)) + ), + root_policy_id=_optional_snapshot_int(snapshot, "root_policy_id"), + policy_revision=int(snapshot.get("policy_revision", 0)), + source_import_job_id=_optional_snapshot_int( + snapshot, + "source_import_job_id", + ), post_processing_method=_snapshot_text(snapshot, "post_processing_method"), torrent_import_strategy=_snapshot_text(snapshot, "torrent_import_strategy"), normalize_imported_archives_to_cbz=parse_bool( @@ -112,6 +136,11 @@ def _snapshot_text(snapshot: Mapping[str, Any], key: str) -> str: return str(value) +def _optional_snapshot_int(snapshot: Mapping[str, Any], key: str) -> int | None: + value = snapshot.get(key) + return None if value is None else int(value) + + def _library_naming_policy_from_configs(configs: Mapping[str, str]) -> LibraryNamingPolicy: """Build the naming-policy portion from already-loaded config values.""" return LibraryNamingPolicy( @@ -123,6 +152,7 @@ def _library_naming_policy_from_configs(configs: Mapping[str, str]) -> LibraryNa single_non_standard_file_template=configs["single_non_standard_file_template"], replace_illegal_characters=parse_bool(configs["replace_illegal_characters"]), colon_replacement=configs["colon_replacement"], + series_path_template=configs["series_folder_template"], ) @@ -154,7 +184,66 @@ async def load_library_ingest_policy(session: AsyncSession) -> LibraryIngestPoli update_embedded_comicinfo_from_match=parse_bool( configs["update_embedded_comicinfo_from_match_on_import"] ), + series_path_template=naming.series_path_template, + ) + + +async def load_effective_library_ingest_policy( + session: AsyncSession, + root: object, +) -> LibraryIngestPolicy: + """Load one root's complete naming policy plus global ingest behavior.""" + from sqlalchemy import select + + root_id = getattr(root, "id", root) + if not isinstance(root_id, int) or root_id < 1: + raise ConfigurationError("A persisted library root is required to resolve naming policy.") + + global_policy = await load_library_ingest_policy(session) + result = await session.execute( + select(LibraryRootPolicy).where(LibraryRootPolicy.library_root_id == root_id) + ) + stored = result.scalar_one_or_none() + if stored is None: + return global_policy + + _validate_stored_root_policy(stored) + return replace( + global_policy, + series_folder_template=stored.series_path_template, + comic_file_template=stored.comic_file_template, + annual_file_template=stored.annual_file_template, + non_standard_file_template=stored.non_standard_file_template, + single_non_standard_file_template=stored.single_non_standard_file_template, + replace_illegal_characters=stored.replace_illegal_characters, + colon_replacement=stored.colon_replacement, + series_path_template=stored.series_path_template, + policy_source=stored.source, + root_policy_id=stored.id, + policy_revision=stored.revision, + source_import_job_id=stored.source_import_job_id, + ) + + +def _validate_stored_root_policy(policy: LibraryRootPolicy) -> None: + """Reject incomplete or unsupported persisted policies on every read.""" + if policy.schema_version != 1: + raise ConfigurationError( + f"Unsupported library root policy schema version: {policy.schema_version}" + ) + templates = ( + policy.series_path_template, + policy.comic_file_template, + policy.annual_file_template, + policy.non_standard_file_template, + policy.single_non_standard_file_template, ) + if any(not value.strip() for value in templates): + raise ConfigurationError("Library root policy templates must be complete.") + if policy.colon_replacement not in {"dash", "space", "empty", "smart"}: + raise ConfigurationError("Library root policy has an invalid colon replacement.") + if policy.revision < 1: + raise ConfigurationError("Library root policy revision must be positive.") async def load_search_on_add_default(session: AsyncSession) -> bool: diff --git a/src/pullbox/core/library_root_resolution.py b/src/pullbox/core/library_root_resolution.py index e1a40cde..27501836 100644 --- a/src/pullbox/core/library_root_resolution.py +++ b/src/pullbox/core/library_root_resolution.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os from pathlib import Path from typing import TYPE_CHECKING @@ -10,7 +9,7 @@ from pullbox.core.exceptions import ConfigurationError from pullbox.models.config import SystemConfig -from pullbox.models.library import LibraryRoot +from pullbox.models.library import LibraryFileStorageMode, LibraryRoot from pullbox.models.series import Series if TYPE_CHECKING: @@ -19,6 +18,12 @@ from sqlalchemy.ext.asyncio import AsyncSession +def preferred_managed_root_id(series: object) -> int | None: + """Return only an explicitly selected future managed destination.""" + root_id = getattr(series, "preferred_library_root_id", None) + return root_id if isinstance(root_id, int) and not isinstance(root_id, bool) else None + + async def resolve_library_root( session: AsyncSession, source_path: Path, @@ -26,56 +31,101 @@ async def resolve_library_root( *, series: Series | None = None, ) -> LibraryRoot: - """Resolve the LibraryRoot for file registration.""" + """Resolve the writable destination root for a managed library file. + + ``source_path`` remains part of the public call signature for compatibility, + but an import source does not confer library ownership or choose where a + managed artifact is written. + """ if explicit_root_id is not None: root = await session.get(LibraryRoot, explicit_root_id) - if root is not None: - return root + return _require_managed_root(root, label="Selected library root") if series is not None: + if series.preferred_library_root_id is not None: + root = await session.get(LibraryRoot, series.preferred_library_root_id) + return _require_managed_root(root, label="Series preferred library root") if series.library_root_id is not None: root = await session.get(LibraryRoot, series.library_root_id) - if root is not None: + if root is None: + raise ConfigurationError("Series library root does not exist.") + if root.enabled and root.allow_managed_writes: return root if series.path: - series_path = str(Path(series.path).expanduser().resolve(strict=False)) - roots_result = await session.execute( - select(LibraryRoot).where(LibraryRoot.enabled.is_(True)) - ) - for root in roots_result.scalars().all(): - root_path = str(Path(root.path).expanduser().resolve(strict=False)) - if series_path == root_path or series_path.startswith(root_path + os.sep): - return root - - roots_result = await session.execute(select(LibraryRoot).where(LibraryRoot.enabled.is_(True))) - roots = list(roots_result.scalars().all()) - source_str = str(source_path) - for root in roots: - if source_str.startswith(root.path + "/") or source_str.startswith(root.path + "\\"): - return root + roots = list((await session.execute(select(LibraryRoot))).scalars().all()) + containing_roots = [ + root for root in roots if path_is_inside_root(Path(series.path), root) + ] + if len(containing_roots) > 1: + raise ConfigurationError("Series path matches multiple library roots.") + if containing_roots: + containing_root = containing_roots[0] + if containing_root.enabled and containing_root.allow_managed_writes: + return containing_root + elif series.library_root_id is None: + raise ConfigurationError( + "Series path does not belong to a configured library root." + ) + + default_roots_result = await session.execute( + select(LibraryRoot).where(LibraryRoot.is_default_managed_destination.is_(True)) + ) + default_roots = list(default_roots_result.scalars().all()) + if len(default_roots) > 1: + raise ConfigurationError( + "Multiple library roots are configured as the default destination." + ) + if default_roots: + return _require_managed_root( + default_roots[0], + label="Default managed library root", + ) config_result = await session.execute( select(SystemConfig).where(SystemConfig.key == "comics_directory") ) config = config_result.scalars().first() - if config is not None: - root_result = await session.execute( - select(LibraryRoot).where(LibraryRoot.path == config.value) - ) - root = root_result.scalars().first() - if root is not None: - return root + if config is not None and config.value.strip(): + configured_path = _resolved_path(config.value) + roots_result = await session.execute(select(LibraryRoot)) + matching_roots = [ + root + for root in roots_result.scalars().all() + if _resolved_path(root.path) == configured_path + ] + if len(matching_roots) > 1: + raise ConfigurationError("The legacy comics directory matches multiple library roots.") + if matching_roots: + return _require_managed_root( + matching_roots[0], + label="Legacy comics directory root", + ) + + raise ConfigurationError( + "No managed library destination is configured. Set a default root in Settings → Media." + ) - if roots: - return roots[0] - raise ConfigurationError("No comics directory configured. Set it in Settings → Media.") +def _require_managed_root(root: LibraryRoot | None, *, label: str) -> LibraryRoot: + """Require a resolved root to be available for managed placement.""" + if root is None: + raise ConfigurationError(f"{label} does not exist.") + if not root.enabled: + raise ConfigurationError(f"{label} is disabled.") + if not root.allow_managed_writes: + raise ConfigurationError(f"{label} does not allow managed writes.") + return root + + +def _resolved_path(path: str | Path) -> Path: + """Return one normalized path value for identity-safe comparisons.""" + return Path(path).expanduser().resolve(strict=False) def path_is_inside_root(path: Path, root: LibraryRoot) -> bool: """Return true when a candidate path is inside a library root.""" - root_path = Path(root.path).expanduser().resolve(strict=False) - candidate = path.expanduser().resolve(strict=False) + root_path = _resolved_path(root.path) + candidate = _resolved_path(path) return candidate == root_path or candidate.is_relative_to(root_path) @@ -109,7 +159,13 @@ def resolve_path_inside_roots( return candidate -def materialize_series_path(series: object, series_folder: Path, root: LibraryRoot) -> None: +def materialize_series_path( + series: object, + series_folder: Path, + root: LibraryRoot, + *, + storage_mode: LibraryFileStorageMode, +) -> None: """Persist the actual library folder once the first file lands there.""" if not isinstance(series, Series): return @@ -117,3 +173,5 @@ def materialize_series_path(series: object, series_folder: Path, root: LibraryRo series.path = str(series_folder) if series.library_root_id is None: series.library_root_id = root.id + if series.preferred_library_root_id is None and storage_mode == LibraryFileStorageMode.MANAGED: + series.preferred_library_root_id = root.id diff --git a/src/pullbox/core/library_target_paths.py b/src/pullbox/core/library_target_paths.py index b2b405b7..b7dcb3d4 100644 --- a/src/pullbox/core/library_target_paths.py +++ b/src/pullbox/core/library_target_paths.py @@ -3,13 +3,14 @@ from __future__ import annotations import asyncio +import os from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING -from pullbox.core.exceptions import ConfigurationError +from pullbox.core.exceptions import ConfigurationError, ImportDestinationValidationError from pullbox.core.library_naming import ( - build_series_folder_name, + build_series_relative_path, compute_target_filename, resolve_naming_issue_type, ) @@ -29,6 +30,8 @@ class ResolvedLibraryTarget: path: Path series_folder_created: bool + created_directory_paths: tuple[Path, ...] = () + directory_ownership_boundary_path: Path | None = None async def resolve_library_target_path( @@ -41,6 +44,8 @@ async def resolve_library_target_path( rename: bool, *, replace_existing_path: Path | None = None, + source_scan_root: Path | None = None, + strict_import: bool = False, ) -> ResolvedLibraryTarget: """Resolve the final library path before materializing the artifact.""" target_path = await predict_library_target_path( @@ -58,13 +63,30 @@ async def resolve_library_target_path( if not comics_dir.exists(): raise ConfigurationError(f"Comics directory does not exist: {comics_dir}") - series_folder_created = not series_folder.exists() - await asyncio.to_thread(series_folder.mkdir, parents=True, exist_ok=True) + if strict_import: + _validate_strict_import_target( + source_path, + target_path, + comics_dir=comics_dir, + source_scan_root=source_scan_root, + ) + + created_directory_paths = await asyncio.to_thread( + _create_target_directories, + series_folder, + comics_dir, + ) + series_folder_created = series_folder in created_directory_paths target_is_replaceable = replace_existing_path is not None and target_path.resolve( strict=False ) == replace_existing_path.resolve(strict=False) - if target_path.exists() and target_path != source_path and not target_is_replaceable: + if ( + not strict_import + and target_path.exists() + and target_path != source_path + and not target_is_replaceable + ): stem = target_path.stem suffix = target_path.suffix counter = 1 @@ -75,9 +97,114 @@ async def resolve_library_target_path( return ResolvedLibraryTarget( path=target_path, series_folder_created=series_folder_created, + created_directory_paths=created_directory_paths, + directory_ownership_boundary_path=comics_dir, + ) + + +def _create_target_directories(directory: Path, boundary: Path) -> tuple[Path, ...]: + """Create target directories one at a time and return only paths we created. + + ``mkdir(parents=True)`` cannot report which missing ancestors it created. A + rollback therefore could not distinguish an import-owned publisher folder + from an empty folder that existed before the import. Creating each segment + individually gives the rollback journal exact, race-aware ownership. + """ + try: + relative = directory.relative_to(boundary) + except ValueError: + # Existing series paths may be expressed through a lexical alias while + # resolving inside the configured root. Preserve compatibility without + # claiming ownership that cannot be proven from this path expression. + directory.mkdir(parents=True, exist_ok=True) + return () + + created: list[Path] = [] + current = boundary + for segment in relative.parts: + current /= segment + try: + current.mkdir() + except FileExistsError: + if not current.is_dir(): + raise + else: + created.append(current) + return tuple(created) + + +def _validate_strict_import_target( + source_path: Path, + target_path: Path, + *, + comics_dir: Path, + source_scan_root: Path | None, +) -> None: + """Fail closed when an import target is not a new, disjoint artifact path.""" + resolved_source = source_path.expanduser().resolve(strict=False) + resolved_target = target_path.expanduser().resolve(strict=False) + if resolved_source == resolved_target or _same_existing_file(source_path, target_path): + raise ImportDestinationValidationError( + "source_destination_same", + "Managed import source and destination resolve to the same file. " + "Choose Keep files in place or a different managed library root.", + ) + + if source_scan_root is not None: + resolved_source_root = source_scan_root.expanduser().resolve(strict=False) + target_inside_source = ( + resolved_target == resolved_source_root + or resolved_target.is_relative_to(resolved_source_root) + ) + root_aliases_source = _same_existing_file(comics_dir, source_scan_root) + if target_inside_source or root_aliases_source: + raise ImportDestinationValidationError( + "destination_inside_source", + "Managed library destination is inside the import source or aliases its " + "inventory boundary. Choose Keep files in place or a non-overlapping " + "library root.", + ) + + collision_name = _casefold_collision_name(target_path) + if collision_name is None: + return + if collision_name != target_path.name: + raise ImportDestinationValidationError( + "destination_case_collision", + f"Managed import target has a case-insensitive collision: {collision_name}. " + "Review the existing library artifact before retrying.", + ) + raise ImportDestinationValidationError( + "destination_collision", + "Managed import target already exists. Review the existing library artifact " + "before retrying.", ) +def _same_existing_file(first: Path, second: Path) -> bool: + try: + return os.path.samefile(first, second) + except (FileNotFoundError, OSError, ValueError): + return False + + +def _casefold_collision_name(target_path: Path) -> str | None: + parent = target_path.parent + if not parent.exists(): + return None + target_key = target_path.name.casefold() + try: + with os.scandir(parent) as entries: + for entry in entries: + if entry.name.casefold() == target_key: + return entry.name + except OSError as exc: + raise ConfigurationError( + f"Could not verify the managed import target directory: {parent}" + ) from exc + return None + + async def predict_library_target_path( session: AsyncSession, source_path: Path, @@ -92,11 +219,32 @@ async def predict_library_target_path( if not comics_dir.exists(): raise ConfigurationError(f"Comics directory does not exist: {comics_dir}") + use_existing_series_folder = False if isinstance(series, Series) and series.path: - series_folder = Path(series.path) - else: - folder_name = build_series_folder_name(series, ingest_policy) - series_folder = comics_dir / folder_name + current_series_folder = Path(series.path) + try: + current_series_folder.resolve(strict=False).relative_to( + comics_dir.resolve(strict=False) + ) + except ValueError as exc: + if series.library_root_id == root.id: + raise ConfigurationError( + "Existing series path is outside its library root." + ) from exc + else: + series_folder = current_series_folder + use_existing_series_folder = True + + if not use_existing_series_folder: + try: + relative_series_path = build_series_relative_path(series, ingest_policy) + except ValueError as exc: + raise ConfigurationError(str(exc)) from exc + series_folder = comics_dir / relative_series_path + try: + series_folder.resolve(strict=False).relative_to(comics_dir.resolve(strict=False)) + except ValueError as exc: + raise ConfigurationError("Rendered series path is outside its library root.") from exc if rename: effective_issue_type = await resolve_naming_issue_type(session, issue) diff --git a/src/pullbox/core/mylar3_path_mapping.py b/src/pullbox/core/mylar3_path_mapping.py new file mode 100644 index 00000000..248f1764 --- /dev/null +++ b/src/pullbox/core/mylar3_path_mapping.py @@ -0,0 +1,99 @@ +"""Validation helpers for frozen Mylar path-mapping snapshots.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from pullbox.core.filesystem_policy import is_invalid_path_text + +if TYPE_CHECKING: + from collections.abc import Iterable + +MAX_MYLAR3_PATH_MAPPINGS = 16 + + +def normalize_mylar3_path_map(path_map: dict[str, str]) -> dict[str, str]: + """Return a bounded canonical map or reject unsafe/ambiguous entries.""" + return normalize_mylar3_path_mapping_items(path_map.items()) + + +def normalize_mylar3_path_mapping_items( + mapping_items: Iterable[tuple[str, str]], +) -> dict[str, str]: + """Normalize ordered editor rows without silently collapsing duplicates.""" + items = list(mapping_items) + if len(items) > MAX_MYLAR3_PATH_MAPPINGS: + raise ValueError(f"Mylar path mapping supports at most {MAX_MYLAR3_PATH_MAPPINGS} entries.") + + normalized: dict[str, str] = {} + for stored_prefix, visible_prefix in items: + normalized_source = _normalize_mapping_path(stored_prefix, role="stored") + normalized_target = _normalize_mapping_path(visible_prefix, role="Pullbox-visible") + if normalized_source in normalized: + raise ValueError("Mylar path mapping contains a duplicate stored prefix.") + if normalized_source == normalized_target: + raise ValueError( + "Mylar path mapping must not contain an identity entry; remove that mapping." + ) + normalized[normalized_source] = normalized_target + + if has_conflicting_overlapping_mappings(normalized): + raise ValueError( + "Mylar path mapping contains overlapping entries that resolve to different paths." + ) + return normalized + + +def has_conflicting_overlapping_mappings(path_map: dict[str, str]) -> bool: + """Return whether nested stored prefixes translate inconsistently.""" + mappings = [(Path(source), Path(target)) for source, target in path_map.items()] + for index, (left_source, left_target) in enumerate(mappings): + for right_source, right_target in mappings[index + 1 :]: + if _nested_mapping_conflicts( + left_source, + left_target, + right_source, + right_target, + ) or _nested_mapping_conflicts( + right_source, + right_target, + left_source, + left_target, + ): + return True + return False + + +def ordered_mylar3_path_map_items(path_map: dict[str, str]) -> list[tuple[str, str]]: + """Return mappings in deterministic longest-complete-prefix order.""" + return sorted( + path_map.items(), + key=lambda item: (-len(Path(item[0]).parts), item[0], item[1]), + ) + + +def _normalize_mapping_path(value: str, *, role: str) -> str: + if not isinstance(value, str) or is_invalid_path_text(value): + raise ValueError(f"Mylar path mapping {role} prefix is invalid.") + path = Path(value) + if not path.is_absolute() or ".." in path.parts or path == Path("/"): + raise ValueError( + f"Mylar path mapping {role} prefix must be a safe absolute directory path." + ) + return str(path) + + +def _nested_mapping_conflicts( + parent_source: Path, + parent_target: Path, + child_source: Path, + child_target: Path, +) -> bool: + try: + relative = child_source.relative_to(parent_source) + except ValueError: + return False + if not relative.parts: + return parent_target != child_target + return parent_target / relative != child_target diff --git a/src/pullbox/core/mylar3_reader.py b/src/pullbox/core/mylar3_reader.py index af1a5480..a411e6b4 100644 --- a/src/pullbox/core/mylar3_reader.py +++ b/src/pullbox/core/mylar3_reader.py @@ -8,20 +8,184 @@ from __future__ import annotations import asyncio +import configparser +import os +import re import sqlite3 +import stat from dataclasses import dataclass from pathlib import Path +from typing import TYPE_CHECKING, cast import structlog from pullbox.core.collection_scanner import COMIC_EXTENSIONS, DiscoveredFile, DiscoveredSeries -from pullbox.core.exceptions import MylarReadError +from pullbox.core.exceptions import ConfigurationError, MylarReadError +from pullbox.core.filesystem_policy import is_invalid_path_text +from pullbox.core.issue_numbers import format_issue_number +from pullbox.core.library_file_ownership import build_file_identity_signature +from pullbox.core.library_layout import ( + ImportLayoutMode, + SourceLayoutMatch, + SourceLayoutSpec, + compile_source_layout, + resolve_source_layout_spec, +) +from pullbox.core.name_matcher import NameMatcher from pullbox.core.naming import parse_filename +from pullbox.core.naming_type_detection import detect_issue_type from pullbox.core.release_parser import normalize_issue_number +from pullbox.core.source_metadata import MetadataSignal, SourceMetadataExtractor from pullbox.models.issue import IssueType +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Sequence + logger = structlog.get_logger(__name__) +_GENERIC_FILENAME_SERIES = frozenset({"book", "comic", "issue", "scan", "unknown"}) + + +def _foreign_filename_series(parsed_series: str, mylar_series: str) -> bool: + """Return whether a filename clearly names a different series than its Mylar folder.""" + normalized = NameMatcher.normalize(parsed_series) + if ( + not normalized + or normalized in _GENERIC_FILENAME_SERIES + or not any(character.isalpha() for character in parsed_series) + ): + return False + return not NameMatcher().match(parsed_series, mylar_series).is_match + + +@dataclass(frozen=True, slots=True) +class Mylar3StoryArcEntrySnapshot: + """Immutable source evidence for one Mylar story-arc row.""" + + ordinal: int + reading_order: int | None + reading_order_raw: str | None + story_arc_id: str | None + story_arc_name: str | None + cv_arc_id: str | None + issue_arc_id: str | None + issue_id: str | None + comic_id: str | None + issue_number: str | None + comic_name: str | None + series_year: str | None + issue_year: str | None + status: str | None + location: str | None + release_date: str | None + issue_date: str | None + publisher: str | None + issue_publisher: str | None + issue_name: str | None + manual: str | None + date_added: str | None + digital_date: str | None + issue_type: str | None + aliases: str | None + total_issues: str | None + in_cache_dir: str | None + int_issue_number: str | None + dynamic_comic_name: str | None + volume: str | None + arc_image: str | None + + +@dataclass(frozen=True, slots=True) +class Mylar3StoryArcSnapshot: + """Immutable normalized grouping of Mylar story-arc rows.""" + + story_arc_id: str | None + cv_arc_id: str | None + name: str | None + entries: tuple[Mylar3StoryArcEntrySnapshot, ...] + + +@dataclass(frozen=True, slots=True) +class Mylar3ArcSettingValue: + """One allowlisted Mylar arc setting and its reviewable raw value.""" + + key: str + section: str + value: bool | str | None + raw_value: str | None + used_default: bool + + +@dataclass(frozen=True, slots=True) +class Mylar3ArcSettingsSnapshot: + """Bounded, secret-free snapshot of Mylar's story-arc settings.""" + + present: bool + parse_warnings: tuple[str, ...] + values: tuple[Mylar3ArcSettingValue, ...] + + +@dataclass(frozen=True, slots=True) +class Mylar3CollectionSnapshot: + """One read-only snapshot of Mylar series, arcs, and read-list inventory.""" + + series: tuple[DiscoveredSeries, ...] + story_arcs: tuple[Mylar3StoryArcSnapshot, ...] + storyarcs_present: bool + readlist_present: bool + readlist_count: int + arc_settings: Mylar3ArcSettingsSnapshot + + +@dataclass(frozen=True, slots=True) +class Mylar3ImportMetadataSnapshot: + """Small source-wide metadata shared by bounded Mylar import pages.""" + + storyarcs_present: bool + readlist_present: bool + readlist_count: int + arc_settings: Mylar3ArcSettingsSnapshot + series_count: int = 0 + + +@dataclass(frozen=True, slots=True) +class Mylar3StoryArcPreflightExample: + """One path-free Story Arc row suitable for a bounded Step 1 preview.""" + + story_arc: str | None + series: str | None + issue_number: str | None + issue_title: str | None + reading_order: str | None + status: str | None + + +@dataclass(frozen=True, slots=True) +class Mylar3StoryArcPreflightSnapshot: + """Exact aggregate counts plus bounded examples from Mylar Story Arc tables.""" + + storyarcs_present: bool + arcs_count: int + entries_count: int + missing_count: int + duplicate_count: int + existing_location_count: int + examples: tuple[Mylar3StoryArcPreflightExample, ...] + readlist_present: bool + readlist_count: int + arc_settings: Mylar3ArcSettingsSnapshot + warnings: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class _MylarArcSettingSpec: + """Static schema for one safe config value.""" + + key: str + section: str + kind: str + default: bool | str | None + @dataclass(frozen=True, slots=True) class _MylarIssueRecord: @@ -51,6 +215,17 @@ class _MylarReleaseSeries: files: list[DiscoveredFile] +@dataclass(frozen=True, slots=True) +class _ResolvedMylarPath: + """Safe resolution state for one Mylar ComicLocation.""" + + path: Path | None + status: str + mapping_applied: bool + reason: str | None = None + rejection_reason: str | None = None + + class Mylar3Reader: """Reads series data from a Mylar3 SQLite database. @@ -61,17 +236,100 @@ class Mylar3Reader: db_path: Path to mylar.db file. path_map: Optional dict of {container_prefix: host_prefix}. Applied to ComicLocation before checking disk. + config_path: Optional explicit Mylar config.ini path. When omitted, a + safe sibling config.ini is discovered without following links. + include_missing_files: Retain full recorded issue paths, including missing + files, for in-place source eligibility review. + reference_root_boundaries: Frozen lexical/resolved root pairs used by + in-place scans to mirror confirmed path semantics. """ MYLAR3_CV_PREFIX = "CV-" + MAX_CONFIG_BYTES = 1_048_576 + DEFAULT_IMPORT_PAGE_SIZE = 100 + MAX_IMPORT_PAGE_SIZE = 250 + ARC_SETTING_SPECS = ( + _MylarArcSettingSpec("STORYARCDIR", "StoryArc", "bool", False), + _MylarArcSettingSpec("STORYARC_LOCATION", "StoryArc", "str", None), + _MylarArcSettingSpec("COPY2ARCDIR", "StoryArc", "bool", False), + _MylarArcSettingSpec( + "ARC_FOLDERFORMAT", + "StoryArc", + "str", + "$arc ($spanyears)", + ), + _MylarArcSettingSpec("ARC_FILEOPS", "StoryArc", "str", "copy"), + _MylarArcSettingSpec( + "ARC_FILEOPS_SOFTLINK_RELATIVE", + "StoryArc", + "bool", + False, + ), + _MylarArcSettingSpec("UPCOMING_STORYARCS", "StoryArc", "bool", False), + _MylarArcSettingSpec("SEARCH_STORYARCS", "StoryArc", "bool", False), + _MylarArcSettingSpec("READ2FILENAME", "General", "bool", False), + ) + STORY_ARC_COLUMNS = ( + "StoryArcID", + "ComicName", + "IssueNumber", + "SeriesYear", + "IssueYEAR", + "StoryArc", + "TotalIssues", + "Status", + "inCacheDir", + "Location", + "IssueArcID", + "ReadingOrder", + "IssueID", + "ComicID", + "ReleaseDate", + "IssueDate", + "Publisher", + "IssuePublisher", + "IssueName", + "CV_ArcID", + "Int_IssueNumber", + "DynamicComicName", + "Volume", + "Manual", + "DateAdded", + "DigitalDate", + "Type", + "Aliases", + "ArcImage", + ) def __init__( self, db_path: str | Path, path_map: dict[str, str] | None = None, + source_layout: SourceLayoutSpec | None = None, + config_path: str | Path | None = None, + *, + include_missing_files: bool = False, + reference_root_boundaries: Sequence[tuple[Path, Path]] | None = None, ) -> None: self._db_path = Path(db_path) + self._config_path = Path(config_path) if config_path is not None else None self._path_map = path_map or {} + self._include_missing_files = include_missing_files + self.import_series_rows_read = 0 + self._reference_root_boundaries = ( + None + if reference_root_boundaries is None + else tuple( + (Path(lexical).expanduser().absolute(), Path(resolved)) + for lexical, resolved in reference_root_boundaries + ) + ) + self._source_layout = resolve_source_layout_spec(source_layout or SourceLayoutSpec()) + self._compiled_source_layout = ( + None + if self._source_layout.mode == ImportLayoutMode.AUTO + else compile_source_layout(self._source_layout) + ) async def read_series(self) -> list[DiscoveredSeries]: """Read all series from the Mylar3 database. @@ -83,22 +341,110 @@ async def read_series(self) -> list[DiscoveredSeries]: FileNotFoundError: If db_path does not exist. MylarReadError: If the file is not a valid Mylar3 database. """ + snapshot = await self.read_snapshot() + return list(snapshot.series) + + async def read_collection(self) -> Mylar3CollectionSnapshot: + """Read the complete normalized Mylar source collection.""" + return await self.read_snapshot() + + async def read_snapshot(self) -> Mylar3CollectionSnapshot: + """Read series, story arcs, and the read-list count in one DB snapshot.""" if not self._db_path.exists(): msg = f"Mylar3 database not found: {self._db_path}" raise FileNotFoundError(msg) - return await asyncio.to_thread(self._read_sync) + return await asyncio.to_thread(self._read_snapshot_sync) + + async def read_import_metadata(self) -> Mylar3ImportMetadataSnapshot: + """Read source-wide flags without retaining comics, issues, or arc rows.""" + self._require_database() + return await asyncio.to_thread(self._read_import_metadata_sync) + + async def iter_import_series_pages( + self, + *, + page_size: int = DEFAULT_IMPORT_PAGE_SIZE, + ) -> AsyncIterator[tuple[DiscoveredSeries, ...]]: + """Yield complete normalized series cohorts from bounded Comic row pages.""" + self._require_database() + self._require_import_page_size(page_size) + after_rowid = 0 + self.import_series_rows_read = 0 + seen_cv_ids: set[int] = set() + while True: + page, next_rowid, source_count = await asyncio.to_thread( + self._read_import_series_page_sync, + after_rowid, + page_size, + ) + if next_rowid == after_rowid: + break + after_rowid = next_rowid + self.import_series_rows_read += source_count + filtered: list[DiscoveredSeries] = [] + for series in page: + cv_id = series.mylar3_cv_id + if cv_id is not None and cv_id in seen_cv_ids: + logger.warning( + "mylar3_duplicate_cv_id", + cv_id=cv_id, + series_name=series.raw_series_name, + ) + continue + if cv_id is not None: + seen_cv_ids.add(cv_id) + filtered.append(series) + if filtered: + yield tuple(filtered) + + async def iter_import_story_arc_pages( + self, + *, + page_size: int = DEFAULT_IMPORT_PAGE_SIZE, + ) -> AsyncIterator[tuple[Mylar3StoryArcSnapshot, ...]]: + """Yield deterministic pages of complete Story Arc cohorts.""" + self._require_database() + self._require_import_page_size(page_size) + after_group: tuple[str, str] | None = None + while True: + page, after_group = await asyncio.to_thread( + self._read_import_story_arc_page_sync, + after_group, + page_size, + ) + if not page: + break + yield page + + async def read_story_arc_preflight( + self, + *, + max_examples: int = 5, + ) -> Mylar3StoryArcPreflightSnapshot: + """Read aggregate Story Arc evidence without loading the Mylar collection.""" + if max_examples < 1 or max_examples > 20: + raise ValueError("Mylar Story Arc preview examples must be between 1 and 20") + if not self._db_path.exists(): + msg = f"Mylar3 database not found: {self._db_path}" + raise FileNotFoundError(msg) + return await asyncio.to_thread(self._read_story_arc_preflight_sync, max_examples) def _read_sync(self) -> list[DiscoveredSeries]: - """Synchronous database read — runs in a thread.""" + """Retain the legacy synchronous series-only helper.""" + return list(self._read_snapshot_sync().series) + + def _read_snapshot_sync(self) -> Mylar3CollectionSnapshot: + """Read every supported Mylar source domain through one connection.""" try: - conn = sqlite3.connect(f"file:{self._db_path}?mode=ro", uri=True) + conn = sqlite3.connect(f"{self._db_path.resolve().as_uri()}?mode=ro", uri=True) conn.row_factory = sqlite3.Row except sqlite3.DatabaseError as exc: msg = f"Could not read Mylar3 database: {exc}" raise MylarReadError(msg) from exc try: + conn.execute("BEGIN") # Verify the comics table exists cursor = conn.execute( "SELECT name FROM sqlite_master WHERE type='table' AND name='comics'" @@ -112,13 +458,399 @@ def _read_sync(self) -> list[DiscoveredSeries]: "ComicLocation, Status, Total FROM comics" ).fetchall() issue_records = self._read_issue_records(conn) + story_arc_rows, storyarcs_present = self._read_story_arc_rows(conn) + readlist_present, readlist_count = self._read_readlist_count(conn) except sqlite3.DatabaseError as exc: msg = f"Could not read Mylar3 database: {exc}" raise MylarReadError(msg) from exc finally: conn.close() - return self._convert_rows(rows, issue_records) + return Mylar3CollectionSnapshot( + series=tuple(self._convert_rows(rows, issue_records)), + story_arcs=self._convert_story_arc_rows(story_arc_rows), + storyarcs_present=storyarcs_present, + readlist_present=readlist_present, + readlist_count=readlist_count, + arc_settings=self._read_arc_settings(), + ) + + def _require_database(self) -> None: + if not self._db_path.exists(): + msg = f"Mylar3 database not found: {self._db_path}" + raise FileNotFoundError(msg) + + def _require_import_page_size(self, page_size: int) -> None: + if page_size < 1 or page_size > self.MAX_IMPORT_PAGE_SIZE: + msg = f"Mylar import page size must be between 1 and {self.MAX_IMPORT_PAGE_SIZE}" + raise ValueError(msg) + + def _read_import_metadata_sync(self) -> Mylar3ImportMetadataSnapshot: + """Read bounded source-wide inventory in one explicit read transaction.""" + try: + conn = sqlite3.connect(f"{self._db_path.resolve().as_uri()}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + except sqlite3.DatabaseError as exc: + msg = f"Could not read Mylar3 database: {exc}" + raise MylarReadError(msg) from exc + + try: + conn.execute("BEGIN") + if not conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='comics'" + ).fetchone(): + msg = "Not a Mylar3 database: 'comics' table not found" + raise MylarReadError(msg) + storyarcs_present = self._table_exists(conn, "storyarcs") + readlist_present, readlist_count = self._read_readlist_count(conn) + series_count = int(conn.execute("SELECT COUNT(*) FROM comics").fetchone()[0]) + except sqlite3.DatabaseError as exc: + msg = f"Could not read Mylar3 database: {exc}" + raise MylarReadError(msg) from exc + finally: + conn.close() + + return Mylar3ImportMetadataSnapshot( + storyarcs_present=storyarcs_present, + readlist_present=readlist_present, + readlist_count=readlist_count, + arc_settings=self._read_arc_settings(), + series_count=series_count, + ) + + def _read_import_series_page_sync( + self, + after_rowid: int, + page_size: int, + ) -> tuple[tuple[DiscoveredSeries, ...], int, int]: + """Read and normalize one bounded rowid-keyset Comic cohort.""" + try: + conn = sqlite3.connect(f"{self._db_path.resolve().as_uri()}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + except sqlite3.DatabaseError as exc: + msg = f"Could not read Mylar3 database: {exc}" + raise MylarReadError(msg) from exc + + try: + conn.execute("BEGIN") + if not conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='comics'" + ).fetchone(): + msg = "Not a Mylar3 database: 'comics' table not found" + raise MylarReadError(msg) + cursor = conn.execute( + "SELECT ComicID, ComicName, ComicYear, ComicPublisher, " + "ComicLocation, Status, Total, rowid AS __source_rowid " + "FROM comics WHERE rowid > ? ORDER BY rowid LIMIT ?", + (after_rowid, page_size), + ) + rows = list(cursor) + if not rows: + return (), after_rowid, 0 + comic_ids = { + cv_id for row in rows if (cv_id := self._parse_cv_id(row["ComicID"])) is not None + } + issue_records = self._read_issue_records_for_comic_ids(conn, comic_ids) + release_ids = { + record.series_cv_id + for owning_comic_id, records in issue_records.items() + for record in records + if record.issue_type == IssueType.ANNUAL and record.series_cv_id != owning_comic_id + } + normal_release_ids = self._existing_comic_ids(conn, release_ids) + converted = self._convert_rows(rows, issue_records) + page_items = [ + series + for series in converted + if not ( + series.mylar3_cv_id in normal_release_ids + and series.mylar3_cv_id not in comic_ids + ) + ] + cross_release_records = self._read_cross_release_records(conn, comic_ids) + if cross_release_records: + owner_rows = self._read_comic_rows_by_cv_ids( + conn, + {owner_cv_id for owner_cv_id, _record in cross_release_records}, + ) + self._attach_cross_release_records( + page_items, + cross_release_records=cross_release_records, + owner_rows=owner_rows, + ) + page = tuple(page_items) + next_rowid = int(rows[-1]["__source_rowid"]) + except sqlite3.DatabaseError as exc: + msg = f"Could not read Mylar3 database: {exc}" + raise MylarReadError(msg) from exc + finally: + conn.close() + return page, next_rowid, len(rows) + + def _read_import_story_arc_page_sync( + self, + after_group: tuple[str, str] | None, + page_size: int, + ) -> tuple[tuple[Mylar3StoryArcSnapshot, ...], tuple[str, str] | None]: + """Read one keyset page of complete Story Arc identity cohorts.""" + try: + conn = sqlite3.connect(f"{self._db_path.resolve().as_uri()}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + except sqlite3.DatabaseError as exc: + msg = f"Could not read Mylar3 database: {exc}" + raise MylarReadError(msg) from exc + + try: + conn.execute("BEGIN") + if not self._table_exists(conn, "storyarcs"): + return (), after_group + available_columns = { + column.casefold(): column for column in self._table_columns(conn, "storyarcs") + } + + def expression(column_name: str) -> str: + actual_column = available_columns.get(column_name.casefold()) + if actual_column is None: + return "NULL" + return f'"{actual_column.replace(chr(34), chr(34) * 2)}"' + + story_arc_id = f"COALESCE(CAST({expression('StoryArcID')} AS TEXT), '')" + cv_arc_id = f"COALESCE(CAST({expression('CV_ArcID')} AS TEXT), '')" + story_arc_name = f"COALESCE(CAST({expression('StoryArc')} AS TEXT), '')" + group_kind = ( + f"CASE WHEN {story_arc_id} <> '' THEN 'story_arc_id' " + f"WHEN {cv_arc_id} <> '' THEN 'cv_arc_id' " + f"WHEN {story_arc_name} <> '' THEN 'story_arc_name' " + "ELSE 'unidentified_row' END" + ) + group_value = ( + f"CASE WHEN {story_arc_id} <> '' THEN {story_arc_id} " + f"WHEN {cv_arc_id} <> '' THEN {cv_arc_id} " + f"WHEN {story_arc_name} <> '' THEN {story_arc_name} " + "ELSE printf('%020d', rowid) END" + ) + key_params: list[object] = [] + key_where = "" + if after_group is not None: + key_where = "WHERE group_kind > ? OR (group_kind = ? AND group_value > ?)" + key_params.extend([after_group[0], after_group[0], after_group[1]]) + key_params.append(page_size) + key_query = ( + "SELECT group_kind, group_value FROM (" + f"SELECT {group_kind} AS group_kind, {group_value} AS group_value " + "FROM storyarcs" + ") AS normalized " + f"{key_where} " + "GROUP BY group_kind, group_value " + "ORDER BY group_kind, group_value LIMIT ?" + ) + key_cursor = conn.execute( + key_query, + tuple(key_params), + ) + group_rows = list(key_cursor) + if not group_rows: + return (), after_group + group_keys = [(str(row["group_kind"]), str(row["group_value"])) for row in group_rows] + + expressions: list[str] = [] + for expected_column in self.STORY_ARC_COLUMNS: + actual_column = available_columns.get(expected_column.casefold()) + if actual_column is None: + expressions.append(f'NULL AS "{expected_column}"') + continue + quoted_column = actual_column.replace('"', '""') + expressions.append(f'"{quoted_column}" AS "{expected_column}"') + predicates = " OR ".join( + f"({group_kind} = ? AND {group_value} = ?)" for _key in group_keys + ) + row_params = tuple(value for key in group_keys for value in key) + row_query = ( + f"SELECT {', '.join(expressions)}, rowid AS __source_rowid " + f"FROM storyarcs WHERE {predicates} " + f"ORDER BY {group_kind}, {group_value}, rowid" + ) + row_cursor = conn.execute(row_query, row_params) + rows = list(row_cursor) + page = self._convert_story_arc_rows(rows) + next_group = group_keys[-1] + except sqlite3.DatabaseError as exc: + msg = f"Could not read Mylar3 database: {exc}" + raise MylarReadError(msg) from exc + finally: + conn.close() + return page, next_group + + def _read_story_arc_preflight_sync( + self, + max_examples: int, + ) -> Mylar3StoryArcPreflightSnapshot: + """Read exact aggregate counts and a bounded row sample in one RO transaction.""" + try: + conn = sqlite3.connect(f"{self._db_path.resolve().as_uri()}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + except sqlite3.DatabaseError as exc: + msg = f"Could not read Mylar3 database: {exc}" + raise MylarReadError(msg) from exc + + warnings: list[str] = [] + try: + conn.execute("BEGIN") + cursor = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='comics'" + ) + if not cursor.fetchone(): + msg = "Not a Mylar3 database: 'comics' table not found" + raise MylarReadError(msg) + if not self._table_exists(conn, "storyarcs"): + readlist_present, readlist_count = self._read_readlist_count(conn) + return Mylar3StoryArcPreflightSnapshot( + storyarcs_present=False, + arcs_count=0, + entries_count=0, + missing_count=0, + duplicate_count=0, + existing_location_count=0, + examples=(), + readlist_present=readlist_present, + readlist_count=readlist_count, + arc_settings=self._read_arc_settings(), + ) + + available = { + column.casefold(): column for column in self._table_columns(conn, "storyarcs") + } + + def expression(column_name: str) -> str: + actual = available.get(column_name.casefold()) + if actual is None: + return "NULL" + return f'"{actual.replace(chr(34), chr(34) * 2)}"' + + story_arc_id = expression("StoryArcID") + cv_arc_id = expression("CV_ArcID") + story_arc_name = expression("StoryArc") + reading_order = expression("ReadingOrder") + status = expression("Status") + location = expression("Location") + story_arc_id_text = f"COALESCE(CAST({story_arc_id} AS TEXT), '')" + cv_arc_id_text = f"COALESCE(CAST({cv_arc_id} AS TEXT), '')" + story_arc_name_text = f"COALESCE(CAST({story_arc_name} AS TEXT), '')" + identity_predicate = ( + f"({story_arc_id_text} <> '' OR {cv_arc_id_text} <> '' " + f"OR {story_arc_name_text} <> '')" + ) + group_kind = ( + f"CASE WHEN {story_arc_id_text} <> '' THEN 'story_arc_id' " + f"WHEN {cv_arc_id_text} <> '' THEN 'cv_arc_id' " + "ELSE 'story_arc_name' END" + ) + group_value = ( + f"CASE WHEN {story_arc_id_text} <> '' THEN {story_arc_id_text} " + f"WHEN {cv_arc_id_text} <> '' THEN {cv_arc_id_text} " + f"ELSE {story_arc_name_text} END" + ) + has_identity_columns = any( + available.get(column.casefold()) + for column in ("StoryArcID", "CV_ArcID", "StoryArc") + ) + + entries_count = int( + conn.execute("SELECT COUNT(*) AS count_value FROM storyarcs").fetchone()[ + "count_value" + ] + ) + identified_arcs_query = ( + "SELECT COUNT(*) AS count_value FROM (" + "SELECT 1 FROM storyarcs WHERE " + f"{identity_predicate} GROUP BY {group_kind}, {group_value})" + ) + identified_arcs_count = int( + conn.execute(identified_arcs_query).fetchone()["count_value"] + ) + unidentified_arcs_query = ( + f"SELECT COUNT(*) AS count_value FROM storyarcs WHERE NOT {identity_predicate}" + ) + unidentified_arcs_count = int( + conn.execute(unidentified_arcs_query).fetchone()["count_value"] + ) + arcs_count = identified_arcs_count + unidentified_arcs_count + missing_query = ( + "SELECT COUNT(*) AS count_value FROM storyarcs WHERE " + f"LOWER(TRIM(COALESCE(CAST({status} AS TEXT), ''))) IN ('missing', 'wanted')" + ) + missing_count = int(conn.execute(missing_query).fetchone()["count_value"]) + existing_location_query = ( + "SELECT COUNT(*) AS count_value FROM storyarcs WHERE " + f"TRIM(COALESCE(CAST({location} AS TEXT), '')) <> ''" + ) + existing_location_count = int( + conn.execute(existing_location_query).fetchone()["count_value"] + ) + duplicate_query = ( + "SELECT COALESCE(SUM(group_count - 1), 0) AS count_value FROM (" + "SELECT COUNT(*) AS group_count FROM storyarcs WHERE " + f"{identity_predicate} AND " + f"TRIM(COALESCE(CAST({reading_order} AS TEXT), '')) <> '' " + f"GROUP BY {group_kind}, {group_value}, {reading_order} " + "HAVING COUNT(*) > 1)" + ) + duplicate_count = int(conn.execute(duplicate_query).fetchone()["count_value"]) + + selected = { + "story_arc": story_arc_name, + "series": expression("ComicName"), + "issue_number": expression("IssueNumber"), + "issue_title": expression("IssueName"), + "reading_order": reading_order, + "status": status, + } + select_list = ", ".join(f'{column} AS "{alias}"' for alias, column in selected.items()) + examples_query = ( + f"SELECT {select_list} FROM storyarcs " + f"ORDER BY {story_arc_name}, {reading_order} LIMIT ?" + ) + rows = conn.execute(examples_query, (max_examples,)).fetchall() + examples = tuple( + Mylar3StoryArcPreflightExample( + story_arc=self._bounded_preflight_text(row["story_arc"]), + series=self._bounded_preflight_text(row["series"]), + issue_number=self._bounded_preflight_text(row["issue_number"]), + issue_title=self._bounded_preflight_text(row["issue_title"]), + reading_order=self._bounded_preflight_text(row["reading_order"]), + status=self._bounded_preflight_text(row["status"]), + ) + for row in rows + ) + readlist_present, readlist_count = self._read_readlist_count(conn) + except sqlite3.DatabaseError as exc: + msg = f"Could not read Mylar3 database: {exc}" + raise MylarReadError(msg) from exc + finally: + conn.close() + + if not has_identity_columns: + warnings.append("story_arc_identity_columns_missing") + return Mylar3StoryArcPreflightSnapshot( + storyarcs_present=True, + arcs_count=arcs_count, + entries_count=entries_count, + missing_count=missing_count, + duplicate_count=duplicate_count, + existing_location_count=existing_location_count, + examples=examples, + readlist_present=readlist_present, + readlist_count=readlist_count, + arc_settings=self._read_arc_settings(), + warnings=tuple(warnings), + ) + + def _bounded_preflight_text(self, value: object, *, max_length: int = 200) -> str | None: + """Return one single-line display value without retaining paths or control text.""" + if value is None: + return None + text = " ".join(str(value).split()).strip() + return text[:max_length] or None def _convert_rows( self, @@ -148,27 +880,42 @@ def _convert_rows( location = row["ComicLocation"] issue_count_hint = self._parse_positive_int(row["Total"]) series_status = row["Status"] - file_count = self._get_file_count(location) + path_resolution = self._resolve_location_details(location) + resolved_location = ( + str(path_resolution.path) if path_resolution.path is not None else None + ) + file_count = self._get_resolved_file_count(path_resolution.path) has_files = file_count > 0 series_issue_records = issue_records.get(cv_id, []) if cv_id is not None else [] # Build sample paths and DiscoveredFile objects from the location directory sample_paths: list[str] = [] discovered_files: list[DiscoveredFile] = [] - resolved_location = self._resolve_location(location) - if resolved_location: - p = Path(resolved_location) - if p.is_dir(): - comic_paths = sorted( - f for f in p.iterdir() if f.suffix.lower() in COMIC_EXTENSIONS - ) - sample_paths = [str(f) for f in comic_paths[:5]] - discovered_files = self._build_files( - comic_paths, - issue_records=series_issue_records, - issue_count_hint=issue_count_hint, - series_status=series_status, - ) + comic_paths: list[Path] = [] + if resolved_location and Path(resolved_location).is_dir(): + comic_paths = sorted( + f + for f in Path(resolved_location).iterdir() + if f.suffix.lower() in COMIC_EXTENSIONS + ) + comic_paths = self._include_recorded_issue_paths( + comic_paths, + path_resolution, + series_issue_records, + ) + if comic_paths: + sample_paths = [str(f) for f in comic_paths[:5]] + discovered_files = self._build_files( + comic_paths, + source_location=location, + series_cv_id=cv_id, + series_name=row["ComicName"] or "Unknown", + series_year=year, + series_publisher=row["ComicPublisher"], + issue_records=series_issue_records, + issue_count_hint=issue_count_hint, + series_status=series_status, + ) if cv_id is not None and discovered_files: parent_files: list[DiscoveredFile] = [] @@ -212,11 +959,72 @@ def _convert_rows( file_count = len(discovered_files) has_files = bool(discovered_files) + recorded_path_resolutions = [ + resolution + for record in series_issue_records + if record.location + and ( + resolution := self._selected_recorded_issue_path_details( + record.location, + path_resolution, + ) + ) + is not None + and resolution.path is not None + and resolution.path in comic_paths + ] + recorded_paths_replace_series_location = bool( + path_resolution.path is None + and path_resolution.status in {"missing", "unmapped"} + and discovered_files + and recorded_path_resolutions + ) diagnostics: dict[str, object] = {} + if recorded_paths_replace_series_location: + recorded_mapping_applied = any( + resolution.mapping_applied for resolution in recorded_path_resolutions + ) + diagnostics["mylar3_path"] = { + "status": "mapped" if recorded_mapping_applied else "local", + "mapping_applied": recorded_mapping_applied, + } + diagnostics["mylar3_series_location"] = { + "status": path_resolution.status, + "mapping_applied": path_resolution.mapping_applied, + "reason": path_resolution.reason, + } + else: + diagnostics["mylar3_path"] = { + "status": path_resolution.status, + "mapping_applied": path_resolution.mapping_applied, + } + if path_resolution.reason is not None and not recorded_paths_replace_series_location: + diagnostics.update( + { + "kind": "mylar3_path_incompatible", + "reason": path_resolution.reason, + "rejection_reason": path_resolution.rejection_reason, + } + ) if series_status: diagnostics["series_status"] = series_status if issue_count_hint is not None: diagnostics["issue_count_hint"] = issue_count_hint + unrecorded_files = [ + file for file in discovered_files if "mylar3_issue" not in file.metadata_diagnostics + ] + folder_scope_conflicts = [ + file + for file in unrecorded_files + if "mylar3_folder_scope_conflict" in file.metadata_diagnostics + ] + if folder_scope_conflicts: + diagnostics["mylar3_folder_scope"] = { + "review_required": True, + "unrecorded_file_count": len(unrecorded_files), + "conflicting_file_count": len(folder_scope_conflicts), + "examples": [file.file_name for file in folder_scope_conflicts[:5]], + } results.append( DiscoveredSeries( @@ -325,19 +1133,213 @@ def _parse_positive_int(self, raw_value: object) -> int | None: def _resolve_location(self, location: str | None) -> str | None: """Apply path map translation to a ComicLocation string.""" + resolved = self._resolve_location_details(location) + return str(resolved.path) if resolved.path is not None else None + + def _resolve_location_details(self, location: str | None) -> _ResolvedMylarPath: + """Resolve one location without allowing path-map escape or silent misses.""" if not location: - return None - resolved = location - for container_prefix, host_prefix in self._path_map.items(): - if resolved.startswith(container_prefix): - resolved = host_prefix + resolved[len(container_prefix) :] - break - return resolved + return _ResolvedMylarPath( + path=None, + status="missing", + mapping_applied=False, + reason="missing_location", + rejection_reason="Mylar does not provide a comic folder for this series.", + ) + location_path = Path(location) + if not location_path.is_absolute(): + return _ResolvedMylarPath( + path=None, + status="invalid", + mapping_applied=False, + reason="invalid_location", + rejection_reason="The Mylar comic folder must be an absolute path.", + ) + if self._path_text_is_invalid(location): + return _ResolvedMylarPath( + path=None, + status="invalid", + mapping_applied=bool(self._path_map), + reason="invalid_path_text", + rejection_reason=( + "The Mylar comic folder contains unsupported control or formatting " + "characters, or exceeds the maximum path length." + ), + ) + if ".." in location_path.parts: + return _ResolvedMylarPath( + path=None, + status="invalid", + mapping_applied=bool(self._path_map), + reason="unsafe_path_mapping", + rejection_reason=( + "The Mylar comic folder resolves outside the configured mapped root." + ), + ) + + try: + resolved_local_path = location_path.resolve(strict=False) + except (OSError, RuntimeError, ValueError): + resolved_local_path = None + identity_outside_root = False + if resolved_local_path is not None and resolved_local_path.is_dir(): + identity_status = self._root_claim_status(location_path, resolved_local_path) + if identity_status == "allowed": + return _ResolvedMylarPath( + path=location_path if self._include_missing_files else resolved_local_path, + status="local", + mapping_applied=False, + ) + if identity_status == "ambiguous": + return _ResolvedMylarPath( + path=location_path if self._include_missing_files else None, + status="ambiguous", + mapping_applied=False, + reason="source_root_ambiguous", + rejection_reason=( + "The Mylar comic folder matches ambiguous reference-capable roots." + ), + ) + identity_outside_root = True + + for container_prefix, host_prefix in self._ordered_path_map_items(): + container_root = Path(container_prefix) + try: + relative = location_path.relative_to(container_root) + except ValueError: + continue + if ".." in relative.parts: + return _ResolvedMylarPath( + path=None, + status="invalid", + mapping_applied=True, + reason="unsafe_path_mapping", + rejection_reason=( + "The Mylar comic folder resolves outside the configured mapped root." + ), + ) + host_root = Path(host_prefix) + if not container_root.is_absolute() or not host_root.is_absolute(): + return _ResolvedMylarPath( + path=None, + status="invalid", + mapping_applied=True, + reason="invalid_path_mapping", + rejection_reason="Mylar path mappings must use absolute paths.", + ) + resolved_root = host_root.resolve(strict=False) + resolved_path = (host_root / relative).resolve(strict=False) + if resolved_path != resolved_root and resolved_root not in resolved_path.parents: + return _ResolvedMylarPath( + path=None, + status="invalid", + mapping_applied=True, + reason="unsafe_path_mapping", + rejection_reason=( + "The Mylar comic folder resolves outside the configured mapped root." + ), + ) + if not resolved_path.is_dir(): + return _ResolvedMylarPath( + path=None, + status="missing", + mapping_applied=True, + reason="mapped_path_missing", + rejection_reason=("The mapped Mylar comic folder is not available to Pullbox."), + ) + mapped_identity_status = self._root_claim_status(host_root / relative, resolved_path) + if mapped_identity_status == "ambiguous": + return _ResolvedMylarPath( + path=host_root / relative if self._include_missing_files else None, + status="ambiguous", + mapping_applied=True, + reason="source_root_ambiguous", + rejection_reason=( + "The mapped Mylar comic folder matches ambiguous reference-capable roots." + ), + ) + if mapped_identity_status == "outside": + return _ResolvedMylarPath( + path=host_root / relative if self._include_missing_files else None, + status="outside_root", + mapping_applied=True, + reason="source_outside_root", + rejection_reason=( + "The mapped Mylar comic folder is outside every enabled " + "reference-capable root." + ), + ) + return _ResolvedMylarPath( + path=host_root / relative if self._include_missing_files else resolved_path, + status="mapped", + mapping_applied=True, + ) + + if identity_outside_root: + return _ResolvedMylarPath( + path=location_path if self._include_missing_files else None, + status="outside_root", + mapping_applied=False, + reason="source_outside_root", + rejection_reason=( + "The Mylar comic folder is outside every enabled reference-capable root." + ), + ) + if self._path_map: + return _ResolvedMylarPath( + path=None, + status="unmapped", + mapping_applied=False, + reason="unmapped_path", + rejection_reason=( + "The Mylar comic folder is not available through the configured path mappings." + ), + ) + return _ResolvedMylarPath( + path=None, + status="missing", + mapping_applied=False, + reason="path_missing", + rejection_reason="The Mylar comic folder is not available to Pullbox.", + ) + + def _ordered_path_map_items(self) -> list[tuple[str, str]]: + """Return path mappings with the most-specific container prefix first.""" + return sorted( + self._path_map.items(), + key=lambda item: len(Path(item[0]).parts), + reverse=True, + ) + + def _root_claim_status(self, lexical_path: Path, resolved_path: Path) -> str: + """Match preflight's lexical-and-resolved root containment contract.""" + if self._reference_root_boundaries is None: + return "allowed" + try: + lexical = lexical_path.expanduser().absolute() + except (OSError, RuntimeError, ValueError): + return "outside" + claims = sum( + lexical.is_relative_to(root_lexical) and resolved_path.is_relative_to(root_resolved) + for root_lexical, root_resolved in self._reference_root_boundaries + ) + if claims == 1: + return "allowed" + return "ambiguous" if claims > 1 else "outside" + + @staticmethod + def _path_text_is_invalid(value: str) -> bool: + return is_invalid_path_text(value) def _build_files( self, comic_paths: list[Path], *, + source_location: str | None, + series_cv_id: int | None, + series_name: str, + series_year: int | None, + series_publisher: str | None, issue_records: list[_MylarIssueRecord], issue_count_hint: int | None, series_status: str | None, @@ -345,47 +1347,191 @@ def _build_files( """Build DiscoveredFile objects from a list of comic file paths.""" results: list[DiscoveredFile] = [] issue_by_file_name = self._issue_records_by_file_name(issue_records) + series_resolution = self._resolve_location_details(source_location) + issue_by_path = self._issue_records_by_path(issue_records, series_resolution) + reconciled = self._reconcile_missing_recorded_paths(comic_paths, issue_by_path) + for actual, recorded in reconciled.items(): + issue_by_path[str(actual)] = issue_by_path[str(recorded)] + replaced_paths = set(reconciled.values()) + comic_paths = [path for path in comic_paths if path not in replaced_paths] + extractor = SourceMetadataExtractor() + sidecars_by_folder = { + folder: extractor.read_sidecars(folder) + for folder in dict.fromkeys(path.parent for path in comic_paths) + } for fpath in comic_paths: + sidecar_data = sidecars_by_folder[fpath.parent] file_name = fpath.name file_format = fpath.suffix.lstrip(".").lower() - issue_record = issue_by_file_name.get(file_name.casefold()) + issue_record = issue_by_path.get(str(fpath)) or issue_by_file_name.get( + file_name.casefold() + ) try: - file_size = fpath.stat().st_size - except OSError: + source_signature = build_file_identity_signature(fpath) + file_size = int(source_signature["size"]) + except (OSError, RuntimeError, ValueError, ConfigurationError): + source_signature = {} file_size = 0 parsed = parse_filename(file_name) - parsed_series: str | None = None + release_series_candidate = ( + parsed.series if parsed else extractor.from_release_title(file_name).series_name + ) + parsed_series: str | None = series_name parsed_issue_number: float | None = None - parsed_year: int | None = None + parsed_year: int | None = series_year + parsed_publisher: str | None = series_publisher issue_type = IssueType.ISSUE issue_number_raw: str | None = None + folder_scope_conflict: dict[str, object] | None = None + metadata_signals: dict[str, str] = { + "series_name": MetadataSignal.MYLAR3.value, + } + if series_year is not None: + metadata_signals["year"] = MetadataSignal.MYLAR3.value + if series_publisher is not None: + metadata_signals["publisher"] = MetadataSignal.MYLAR3.value if parsed: - parsed_series = parsed.series parsed_issue_number = parsed.issue_number - parsed_year = parsed.year try: issue_type = IssueType(parsed.issue_type) except ValueError: issue_type = IssueType.ISSUE - if parsed.issue_number == int(parsed.issue_number): - issue_number_raw = str(int(parsed.issue_number)) - else: - issue_number_raw = str(parsed.issue_number) + issue_number_raw = format_issue_number(parsed.issue_number) + if parsed_issue_number is not None: + metadata_signals["issue_number"] = MetadataSignal.RELEASE_TITLE.value + if parsed.issue_type != IssueType.ISSUE.value: + metadata_signals["issue_type"] = MetadataSignal.RELEASE_TITLE.value + if ( + issue_record is None + and release_series_candidate + and _foreign_filename_series(release_series_candidate, series_name) + ): + parsed_series = release_series_candidate + metadata_signals["series_name"] = MetadataSignal.RELEASE_TITLE.value + folder_scope_conflict = { + "expected_series": series_name, + "parsed_series": release_series_candidate, + "recorded_issue": False, + } if issue_record is not None and issue_record.issue_number: issue_number_raw = issue_record.issue_number normalized_issue_number = normalize_issue_number(issue_record.issue_number) if normalized_issue_number is not None: parsed_issue_number = normalized_issue_number + metadata_signals["issue_number"] = MetadataSignal.MYLAR3.value + + normalized_sidecar = sidecar_data or {} + metadata_diagnostics: dict[str, object] = { + "sidecar_files_present": sorted( + str(name) for name in normalized_sidecar.get("files_present") or [] + ), + "archive_metadata_loaded": False, + "archive_metadata_deferred": True, + "has_comicinfo": False, + "mylar3_folder_metadata_scanned": True, + } + if issue_record is None: + metadata_diagnostics["mylar3_unrecorded_file"] = { + "expected_series": series_name, + } + if fpath in reconciled: + metadata_diagnostics["mylar3_path_reconciliation"] = { + "recorded_path": str(reconciled[fpath]), + "actual_path": str(fpath), + "method": "unique_same_folder_normalized_stem", + } + if folder_scope_conflict is not None: + metadata_diagnostics["mylar3_folder_scope_conflict"] = folder_scope_conflict + if sidecar_data is not None and ( + sidecar_data.get("files_present") + or sidecar_data.get("series_id") is not None + or sidecar_data.get("issue_id") is not None + or sidecar_data.get("booktype") is not None + or sidecar_data.get("series_status") is not None + or sidecar_data.get("issue_count") is not None + or sidecar_data.get("series_name") is not None + or sidecar_data.get("year") is not None + or sidecar_data.get("identity_conflicts") + ): + sidecar_booktype = sidecar_data.get("booktype") + metadata_diagnostics["sidecar_snapshot"] = { + "files_present": list(sidecar_data.get("files_present") or []), + "series_id": sidecar_data.get("series_id"), + "series_id_source": sidecar_data.get("series_id_source"), + "issue_id": sidecar_data.get("issue_id"), + "booktype": ( + sidecar_booktype.value + if isinstance(sidecar_booktype, IssueType) + else sidecar_booktype + ), + "series_status": sidecar_data.get("series_status"), + "issue_count": sidecar_data.get("issue_count"), + "series_name": sidecar_data.get("series_name"), + "year": sidecar_data.get("year"), + "identity_conflicts": list(sidecar_data.get("identity_conflicts") or []), + } + sidecar_identity: dict[str, object] = {} + sidecar_series_id = normalized_sidecar.get("series_id") + sidecar_issue_id = normalized_sidecar.get("issue_id") + if isinstance(sidecar_series_id, int): + sidecar_identity["comicvine_series_id"] = sidecar_series_id + else: + sidecar_series_id = None + if isinstance(sidecar_issue_id, int): + sidecar_identity["comicvine_issue_id"] = sidecar_issue_id + else: + sidecar_issue_id = None + if sidecar_identity: + metadata_diagnostics["sidecar_identity"] = sidecar_identity + raw_identity_conflicts = normalized_sidecar.get("identity_conflicts") + identity_conflicts = ( + [ + dict(conflict) + for conflict in raw_identity_conflicts + if isinstance(conflict, dict) + ] + if isinstance(raw_identity_conflicts, list) + else [] + ) + if ( + sidecar_series_id is not None + and series_cv_id is not None + and sidecar_series_id != series_cv_id + ): + identity_conflicts.append( + { + "field": "comicvine_series_id", + "mylar3": series_cv_id, + "sidecar": sidecar_series_id, + } + ) + if identity_conflicts: + metadata_diagnostics["identity_conflicts"] = identity_conflicts - metadata_signals: dict[str, str] = {} - metadata_diagnostics: dict[str, object] = {} - comicvine_issue_id: int | None = None - comicvine_series_id: int | None = None + comicvine_issue_id = sidecar_issue_id + comicvine_series_id = ( + sidecar_series_id + if folder_scope_conflict is not None + else series_cv_id or sidecar_series_id + ) + if comicvine_issue_id is not None: + metadata_signals["comicvine_issue_id"] = MetadataSignal.SIDECAR.value + if comicvine_series_id is not None: + metadata_signals["comicvine_series_id"] = ( + MetadataSignal.MYLAR3.value + if series_cv_id is not None and folder_scope_conflict is None + else MetadataSignal.SIDECAR.value + ) if issue_record is not None: if issue_record.issue_type != IssueType.ISSUE: issue_type = issue_record.issue_type + metadata_signals["issue_type"] = MetadataSignal.MYLAR3.value + if issue_record.series_name: + parsed_series = issue_record.series_name + if issue_record.issue_type == IssueType.ANNUAL: + parsed_year = self._release_year(issue_record.release_date, parsed_year) comicvine_issue_id = issue_record.issue_id comicvine_series_id = issue_record.series_cv_id metadata_signals["comicvine_issue_id"] = "mylar3" @@ -397,16 +1543,60 @@ def _build_files( "release_date": issue_record.release_date, } + layout_match, relative_path = self._match_selected_layout( + fpath, + source_location=source_location, + ) + if self._compiled_source_layout is not None and relative_path is not None: + layout_diagnostics: dict[str, object] = { + "fit": layout_match is not None, + "fallback_used": ( + layout_match is None and self._source_layout.fallback_to_auto + ), + "relative_path": relative_path, + } + if layout_match is None and not self._source_layout.fallback_to_auto: + layout_diagnostics.update( + { + "review_required": True, + "review_reason": "selected_layout_no_match", + } + ) + if layout_match is not None and layout_match.issue_title is not None: + layout_diagnostics["issue_title"] = layout_match.issue_title + metadata_diagnostics["source_layout"] = layout_diagnostics + + if layout_match is not None: + ( + parsed_series, + parsed_year, + parsed_publisher, + parsed_issue_number, + issue_number_raw, + issue_type, + ) = self._apply_layout_match( + layout_match, + parsed_series=parsed_series, + parsed_year=parsed_year, + parsed_publisher=parsed_publisher, + parsed_issue_number=parsed_issue_number, + issue_number_raw=issue_number_raw, + issue_type=issue_type, + metadata_signals=metadata_signals, + metadata_diagnostics=metadata_diagnostics, + ) + results.append( DiscoveredFile( file_path=str(fpath), file_name=file_name, file_size=file_size, file_format=file_format, + source_signature=source_signature, parsed_series=parsed_series, parsed_issue_number=parsed_issue_number, parsed_year=parsed_year, - parsed_publisher=None, + parsed_publisher=parsed_publisher, has_comicinfo=False, comicvine_issue_id=comicvine_issue_id, issue_number_raw=issue_number_raw, @@ -420,6 +1610,308 @@ def _build_files( ) return results + @staticmethod + def _reconcile_missing_recorded_paths( + comic_paths: list[Path], + issue_by_path: dict[str, _MylarIssueRecord], + ) -> dict[Path, Path]: + """Accept only unambiguous case/spacing/extension drift, never issue-number guesses.""" + existing: dict[tuple[Path, str], list[Path]] = {} + missing: dict[tuple[Path, str], list[Path]] = {} + for path in comic_paths: + normalized = " ".join(path.stem.casefold().split()) + normalized = re.sub(r"\s*([.()\[\]])\s*", r"\1", normalized) + key = (path.parent, normalized) + try: + if path.is_file() and not path.is_symlink(): + existing.setdefault(key, []).append(path) + elif not path.exists() and str(path) in issue_by_path: + missing.setdefault(key, []).append(path) + except OSError: + continue + reconciled: dict[Path, Path] = {} + for key, recorded in missing.items(): + candidates = existing.get(key, []) + if len(recorded) != 1 or len(candidates) != 1: + continue + actual = candidates[0] + if str(actual) in issue_by_path: + continue + reconciled[actual] = recorded[0] + return reconciled + + def _include_recorded_issue_paths( + self, + comic_paths: list[Path], + series_resolution: _ResolvedMylarPath, + issue_records: list[_MylarIssueRecord], + ) -> list[Path]: + """Merge safely resolved records, retaining missing paths only in-place.""" + paths = set(comic_paths) + for record in issue_records: + if not record.location: + continue + path = Path(record.location) + if path.suffix.lower() not in COMIC_EXTENSIONS: + continue + resolution = self._selected_recorded_issue_path_details( + record.location, + series_resolution, + ) + if resolution is not None and resolution.path is not None: + paths.add(resolution.path) + return sorted(paths) + + def _issue_records_by_path( + self, + issue_records: list[_MylarIssueRecord], + series_resolution: _ResolvedMylarPath, + ) -> dict[str, _MylarIssueRecord]: + """Index issue identities by their exact selected source path.""" + records: dict[str, _MylarIssueRecord] = {} + ambiguous: set[str] = set() + for record in issue_records: + if not record.location: + continue + resolution = self._selected_recorded_issue_path_details( + record.location, + series_resolution, + ) + if resolution is not None and resolution.path is not None: + path = str(resolution.path) + existing = records.get(path) + if existing is not None and existing.issue_id != record.issue_id: + ambiguous.add(path) + records[path] = record + for path in ambiguous: + records.pop(path, None) + return records + + def _selected_recorded_issue_path_details( + self, + location: str, + series_resolution: _ResolvedMylarPath, + ) -> _ResolvedMylarPath | None: + """Resolve one exact issue source under the confirmed series/path-map contract.""" + if self._path_text_is_invalid(location): + return None + path = Path(location) + if ".." in path.parts or path.suffix.lower() not in COMIC_EXTENSIONS: + return None + + if not path.is_absolute(): + if series_resolution.path is None: + return None + try: + resolved_series = series_resolution.path.resolve(strict=False) + resolved_path = (series_resolution.path / path).resolve(strict=False) + except (OSError, RuntimeError, ValueError): + return None + if not resolved_path.is_relative_to(resolved_series): + return None + if not self._include_missing_files and not resolved_path.is_file(): + return None + return _ResolvedMylarPath( + path=(series_resolution.path / path) + if self._include_missing_files + else resolved_path, + status=series_resolution.status, + mapping_applied=series_resolution.mapping_applied, + ) + + try: + identity_path = path.resolve(strict=False) + except (OSError, RuntimeError, ValueError): + identity_path = None + identity_exists = identity_path is not None and identity_path.is_file() + if identity_exists: + assert identity_path is not None + identity_status = self._root_claim_status(path, identity_path) + if identity_status == "allowed": + return _ResolvedMylarPath( + path=path if self._include_missing_files else identity_path, + status="local", + mapping_applied=False, + ) + if identity_status == "ambiguous": + return ( + _ResolvedMylarPath( + path=path, + status="ambiguous", + mapping_applied=False, + reason="source_root_ambiguous", + ) + if self._include_missing_files + else None + ) + + for container_prefix, host_prefix in self._ordered_path_map_items(): + container_root = Path(container_prefix) + try: + relative = path.relative_to(container_root) + except ValueError: + continue + host_root = Path(host_prefix) + if ( + not container_root.is_absolute() + or not host_root.is_absolute() + or ".." in relative.parts + ): + return None + try: + resolved_root = host_root.resolve(strict=False) + resolved_path = (host_root / relative).resolve(strict=False) + except (OSError, RuntimeError, ValueError): + return None + if not resolved_path.is_relative_to(resolved_root): + return None + if not self._include_missing_files and not resolved_path.is_file(): + return None + mapped_path = host_root / relative if self._include_missing_files else resolved_path + mapped_status = self._root_claim_status(host_root / relative, resolved_path) + if mapped_status == "allowed": + return _ResolvedMylarPath( + path=mapped_path, + status="mapped", + mapping_applied=True, + ) + return ( + _ResolvedMylarPath( + path=mapped_path, + status="ambiguous" if mapped_status == "ambiguous" else "outside_root", + mapping_applied=True, + reason=( + "source_root_ambiguous" + if mapped_status == "ambiguous" + else "source_outside_root" + ), + ) + if self._include_missing_files + else None + ) + if self._include_missing_files: + return _ResolvedMylarPath( + path=path, + status="outside_root" if identity_exists else "missing", + mapping_applied=False, + reason="source_outside_root" if identity_exists else "path_missing", + ) + return None + + def _match_selected_layout( + self, + path: Path, + *, + source_location: str | None, + ) -> tuple[SourceLayoutMatch | None, str | None]: + """Match one resolved Mylar file against the frozen source layout.""" + compiled = self._compiled_source_layout + if compiled is None: + return None, None + root = self._mapped_source_root(source_location) + if root is None: + resolved_location = self._resolve_location(source_location) + if resolved_location is None: + return None, None + root = Path(resolved_location) + for _segment in compiled.path_segments: + root = root.parent + try: + relative_path = path.relative_to(root).as_posix() + except ValueError: + return None, None + return compiled.match(relative_path), relative_path + + def _mapped_source_root(self, source_location: str | None) -> Path | None: + """Return the mapped host root that contains one Mylar series path.""" + if not source_location: + return None + location_path = Path(source_location) + if ".." not in location_path.parts and location_path.resolve(strict=False).is_dir(): + return None + for container_prefix, host_prefix in self._ordered_path_map_items(): + try: + location_path.relative_to(Path(container_prefix)) + except ValueError: + continue + return Path(host_prefix) + return None + + def _apply_layout_match( + self, + layout_match: SourceLayoutMatch, + *, + parsed_series: str | None, + parsed_year: int | None, + parsed_publisher: str | None, + parsed_issue_number: float | None, + issue_number_raw: str | None, + issue_type: IssueType, + metadata_signals: dict[str, str], + metadata_diagnostics: dict[str, object], + ) -> tuple[str | None, int | None, str | None, float | None, str | None, IssueType]: + """Apply lower-precedence layout evidence without replacing Mylar identity.""" + conflicts: dict[str, dict[str, object]] = {} + + def selected_value(field_name: str, current: object, selected: object | None) -> object: + if selected is None: + return current + current_signal = metadata_signals.get(field_name) + if current is not None and current_signal == MetadataSignal.MYLAR3.value: + if str(current).casefold() != str(selected).casefold(): + conflicts[field_name] = { + "selected": selected, + "preserved_signal": MetadataSignal.MYLAR3.value, + } + return current + metadata_signals[field_name] = MetadataSignal.SOURCE_LAYOUT.value + return selected + + parsed_series = cast( + "str | None", + selected_value("series_name", parsed_series, layout_match.series), + ) + parsed_year = cast( + "int | None", + selected_value("year", parsed_year, layout_match.year), + ) + parsed_publisher = cast( + "str | None", + selected_value( + "publisher", + parsed_publisher, + layout_match.publisher, + ), + ) + if layout_match.issue_number is not None: + normalized = normalize_issue_number(layout_match.issue_number) + parsed_issue_number = cast( + "float | None", + selected_value( + "issue_number", + parsed_issue_number, + normalized, + ), + ) + if metadata_signals.get("issue_number") == MetadataSignal.SOURCE_LAYOUT.value: + issue_number_raw = layout_match.issue_number + if layout_match.issue_type is not None: + selected_issue_type = IssueType(detect_issue_type(layout_match.issue_type)) + issue_type = cast( + "IssueType", + selected_value("issue_type", issue_type, selected_issue_type), + ) + if conflicts: + metadata_diagnostics["source_layout_conflicts"] = conflicts + return ( + parsed_series, + parsed_year, + parsed_publisher, + parsed_issue_number, + issue_number_raw, + issue_type, + ) + def _issue_records_by_file_name( self, issue_records: list[_MylarIssueRecord], @@ -441,10 +1933,162 @@ def _read_issue_records( self._append_annual_table_records(conn, records) return records + def _read_issue_records_for_comic_ids( + self, + conn: sqlite3.Connection, + comic_ids: set[int], + ) -> dict[int, list[_MylarIssueRecord]]: + """Read only issue identities owned by one bounded Comic page.""" + if not comic_ids: + return {} + records: dict[int, list[_MylarIssueRecord]] = {} + self._append_issue_table_records(conn, records, comic_ids=comic_ids) + self._append_annual_table_records(conn, records, comic_ids=comic_ids) + return records + + def _existing_comic_ids( + self, + conn: sqlite3.Connection, + comic_ids: set[int], + ) -> set[int]: + if not comic_ids: + return set() + where_clause, params = self._comic_id_filter(comic_ids) + query = f"SELECT ComicID FROM comics{where_clause}" + return { + parsed + for row in conn.execute(query, params) + if (parsed := self._parse_cv_id(row["ComicID"])) is not None + } + + def _read_cross_release_records( + self, + conn: sqlite3.Connection, + release_comic_ids: set[int], + ) -> list[tuple[int, _MylarIssueRecord]]: + """Read Annual records whose release identity is in the current Comic page.""" + if not release_comic_ids or not self._table_has_columns( + conn, + "annuals", + { + "IssueID", + "ComicID", + "Issue_Number", + "IssueName", + "IssueDate", + "Location", + "ReleaseComicID", + }, + ): + return [] + annual_columns = self._table_columns(conn, "annuals") + base_query = ( + "SELECT IssueID, ComicID, Issue_Number, IssueName, IssueDate, Location, " + "ReleaseComicID, ReleaseComicName FROM annuals" + if "ReleaseComicName" in annual_columns + else "SELECT IssueID, ComicID, Issue_Number, IssueName, IssueDate, Location, " + "ReleaseComicID, NULL AS ReleaseComicName FROM annuals" + ) + where_clause, params = self._comic_id_filter( + release_comic_ids, + column_name="ReleaseComicID", + ) + query = f"{base_query}{where_clause}" + records: list[tuple[int, _MylarIssueRecord]] = [] + for row in conn.execute(query, params): + parsed = self._annual_record_from_row(row) + if parsed is None or parsed[0] == parsed[1].series_cv_id: + continue + records.append(parsed) + return records + + def _read_comic_rows_by_cv_ids( + self, + conn: sqlite3.Connection, + comic_ids: set[int], + ) -> dict[int, sqlite3.Row]: + if not comic_ids: + return {} + where_clause, params = self._comic_id_filter(comic_ids) + rows: dict[int, sqlite3.Row] = {} + query = ( + "SELECT ComicID, ComicName, ComicYear, ComicPublisher, ComicLocation, " + f"Status, Total FROM comics{where_clause} ORDER BY rowid" + ) + cursor = conn.execute(query, params) + for row in cursor: + comic_id = self._parse_cv_id(row["ComicID"]) + if comic_id is not None: + rows.setdefault(comic_id, row) + return rows + + def _attach_cross_release_records( + self, + series_page: list[DiscoveredSeries], + *, + cross_release_records: list[tuple[int, _MylarIssueRecord]], + owner_rows: dict[int, sqlite3.Row], + ) -> None: + """Attach cross-page Annual files to their existing release series cohort.""" + target_by_cv_id = { + series.mylar3_cv_id: series for series in series_page if series.mylar3_cv_id is not None + } + for owning_comic_id, record in cross_release_records: + target = target_by_cv_id.get(record.series_cv_id) + owner_row = owner_rows.get(owning_comic_id) + if target is None or owner_row is None or not record.location: + continue + owner_location = owner_row["ComicLocation"] + owner_resolution = self._resolve_location_details(owner_location) + candidate_resolution = self._selected_recorded_issue_path_details( + record.location, + owner_resolution, + ) + candidate = candidate_resolution.path if candidate_resolution is not None else None + if candidate is None or candidate.suffix.lower() not in COMIC_EXTENSIONS: + continue + existing_paths = {item.file_path for item in target.files} + if str(candidate) in existing_paths: + continue + built_files = self._build_files( + [candidate], + source_location=owner_location, + series_cv_id=target.mylar3_cv_id, + series_name=target.raw_series_name, + series_year=target.raw_year, + series_publisher=target.raw_publisher, + issue_records=[record], + issue_count_hint=self._parse_positive_int(owner_row["Total"]), + series_status=owner_row["Status"], + ) + target.files.extend(built_files) + target.file_count = len(target.files) + target.sample_paths = [item.file_path for item in target.files[:5]] + target.has_files = bool(target.files) + target.diagnostics["source_issue_type"] = IssueType.ANNUAL.value + + def _comic_id_filter( + self, + comic_ids: set[int], + *, + column_name: str = "ComicID", + ) -> tuple[str, tuple[str, ...]]: + if column_name not in {"ComicID", "ReleaseComicID"}: + raise ValueError("Unsupported Mylar Comic ID filter column") + identifiers = tuple( + identifier + for comic_id in sorted(comic_ids) + for identifier in (str(comic_id), f"{self.MYLAR3_CV_PREFIX}{comic_id}") + ) + placeholders = ", ".join("?" for _identifier in identifiers) + return f" WHERE {column_name} IN ({placeholders})", identifiers + def _append_issue_table_records( self, conn: sqlite3.Connection, records: dict[int, list[_MylarIssueRecord]], + *, + comic_ids: set[int] | None = None, ) -> None: if not self._table_has_columns( conn, @@ -452,9 +2096,14 @@ def _append_issue_table_records( {"IssueID", "ComicID", "Issue_Number", "IssueName", "IssueDate", "Location"}, ): return - for row in conn.execute( + where_clause, params = ( + self._comic_id_filter(comic_ids) if comic_ids is not None else ("", ()) + ) + query = ( "SELECT IssueID, ComicID, Issue_Number, IssueName, IssueDate, Location FROM issues" - ).fetchall(): + f"{where_clause}" + ) + for row in conn.execute(query, params): comic_id = self._parse_cv_id(row["ComicID"]) issue_id = self._parse_positive_int(row["IssueID"]) if comic_id is None or issue_id is None: @@ -474,6 +2123,8 @@ def _append_annual_table_records( self, conn: sqlite3.Connection, records: dict[int, list[_MylarIssueRecord]], + *, + comic_ids: set[int] | None = None, ) -> None: if not self._table_has_columns( conn, @@ -490,55 +2141,408 @@ def _append_annual_table_records( ): return annual_columns = self._table_columns(conn, "annuals") - query = ( + base_query = ( "SELECT IssueID, ComicID, Issue_Number, IssueName, IssueDate, Location, " "ReleaseComicID, ReleaseComicName FROM annuals" if "ReleaseComicName" in annual_columns else "SELECT IssueID, ComicID, Issue_Number, IssueName, IssueDate, Location, " "ReleaseComicID, NULL AS ReleaseComicName FROM annuals" ) - for row in conn.execute(query).fetchall(): - owning_comic_id = self._parse_cv_id(row["ComicID"]) - issue_id = self._parse_positive_int(row["IssueID"]) - release_comic_id = self._parse_cv_id(row["ReleaseComicID"]) - if owning_comic_id is None or issue_id is None: + where_clause, params = ( + self._comic_id_filter(comic_ids) if comic_ids is not None else ("", ()) + ) + query = f"{base_query}{where_clause}" + for row in conn.execute(query, params): + parsed = self._annual_record_from_row(row) + if parsed is None: continue - records.setdefault(owning_comic_id, []).append( - _MylarIssueRecord( - issue_id=issue_id, - series_cv_id=release_comic_id or owning_comic_id, - issue_number=row["Issue_Number"], - title=row["IssueName"], - release_date=row["IssueDate"], - location=row["Location"], - issue_type=IssueType.ANNUAL, - series_name=row["ReleaseComicName"], + owning_comic_id, record = parsed + records.setdefault(owning_comic_id, []).append(record) + + def _annual_record_from_row( + self, + row: sqlite3.Row, + ) -> tuple[int, _MylarIssueRecord] | None: + owning_comic_id = self._parse_cv_id(row["ComicID"]) + issue_id = self._parse_positive_int(row["IssueID"]) + release_comic_id = self._parse_cv_id(row["ReleaseComicID"]) + if owning_comic_id is None or issue_id is None: + return None + return owning_comic_id, _MylarIssueRecord( + issue_id=issue_id, + series_cv_id=release_comic_id or owning_comic_id, + issue_number=row["Issue_Number"], + title=row["IssueName"], + release_date=row["IssueDate"], + location=row["Location"], + issue_type=IssueType.ANNUAL, + series_name=row["ReleaseComicName"], + ) + + def _read_arc_settings(self) -> Mylar3ArcSettingsSnapshot: + """Read only the allowlisted arc settings from a bounded config file.""" + config_path = self._config_path or self._db_path.with_name("config.ini") + content, present, warnings = self._read_bounded_config(config_path) + if content is None: + return Mylar3ArcSettingsSnapshot( + present=present, + parse_warnings=tuple(warnings), + values=self._arc_setting_values(None, warnings), + ) + + try: + config_text = content.decode("utf-8-sig") + except UnicodeDecodeError: + warnings.append("config_decode_failed") + return Mylar3ArcSettingsSnapshot( + present=True, + parse_warnings=tuple(warnings), + values=self._arc_setting_values(None, warnings), + ) + + parser = configparser.ConfigParser(interpolation=None, strict=False) + try: + parser.read_string(config_text) + except configparser.Error: + warnings.append("config_parse_failed") + return Mylar3ArcSettingsSnapshot( + present=True, + parse_warnings=tuple(warnings), + values=self._arc_setting_values(None, warnings), + ) + + values = self._arc_setting_values(parser, warnings) + return Mylar3ArcSettingsSnapshot( + present=True, + parse_warnings=tuple(warnings), + values=values, + ) + + def _read_bounded_config( + self, + config_path: Path, + ) -> tuple[bytes | None, bool, list[str]]: + """Open one regular config file without following a symlink.""" + try: + initial_stat = config_path.lstat() + except FileNotFoundError: + return None, False, [] + except OSError: + return None, True, ["config_stat_failed"] + if stat.S_ISLNK(initial_stat.st_mode): + return None, True, ["config_symlink_rejected"] + if not stat.S_ISREG(initial_stat.st_mode): + return None, True, ["config_not_regular_file"] + if initial_stat.st_size > self.MAX_CONFIG_BYTES: + return None, True, ["config_too_large"] + + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(config_path, flags) + except OSError: + return None, True, ["config_open_failed"] + + try: + opened_stat = os.fstat(descriptor) + if not stat.S_ISREG(opened_stat.st_mode): + return None, True, ["config_not_regular_file"] + if opened_stat.st_size > self.MAX_CONFIG_BYTES: + return None, True, ["config_too_large"] + if ( + initial_stat.st_ino + and opened_stat.st_ino + and (initial_stat.st_dev, initial_stat.st_ino) + != (opened_stat.st_dev, opened_stat.st_ino) + ): + return None, True, ["config_source_changed"] + + chunks: list[bytes] = [] + remaining = self.MAX_CONFIG_BYTES + 1 + while remaining > 0: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + content = b"".join(chunks) + except OSError: + return None, True, ["config_read_failed"] + finally: + os.close(descriptor) + + if len(content) > self.MAX_CONFIG_BYTES: + return None, True, ["config_too_large"] + return content, True, [] + + def _arc_setting_values( + self, + parser: configparser.ConfigParser | None, + warnings: list[str], + ) -> tuple[Mylar3ArcSettingValue, ...]: + """Normalize only known keys while retaining their raw values.""" + sections = ( + {section.casefold(): section for section in parser.sections()} + if parser is not None + else {} + ) + values: list[Mylar3ArcSettingValue] = [] + for spec in self.ARC_SETTING_SPECS: + section = sections.get(spec.section.casefold()) + raw_value = ( + parser.get(section, spec.key, raw=True) + if parser is not None + and section is not None + and parser.has_option(section, spec.key) + else None + ) + value, used_default = self._normalize_arc_setting(spec, raw_value, warnings) + values.append( + Mylar3ArcSettingValue( + key=spec.key, + section=spec.section, + value=value, + raw_value=raw_value, + used_default=used_default, + ) + ) + return tuple(values) + + def _normalize_arc_setting( + self, + spec: _MylarArcSettingSpec, + raw_value: str | None, + warnings: list[str], + ) -> tuple[bool | str | None, bool]: + if raw_value is None: + return spec.default, True + if spec.kind == "bool": + normalized = raw_value.strip().casefold() + if normalized in {"1", "yes", "true", "on"}: + return True, False + if normalized in {"0", "no", "false", "off"}: + return False, False + warnings.append(f"invalid_boolean:{spec.key}") + return spec.default, True + if spec.key == "STORYARC_LOCATION" and raw_value.strip().casefold() in {"", "none"}: + return None, False + if spec.key == "ARC_FILEOPS" and raw_value.strip().casefold() not in { + "copy", + "move", + "hardlink", + "softlink", + }: + warnings.append("unknown_value:ARC_FILEOPS") + return raw_value, False + + def _read_story_arc_rows( + self, + conn: sqlite3.Connection, + ) -> tuple[list[sqlite3.Row], bool]: + """Read available story-arc evidence without requiring a schema upgrade.""" + if not self._table_exists(conn, "storyarcs"): + return [], False + + available_columns = { + column.casefold(): column for column in self._table_columns(conn, "storyarcs") + } + expressions: list[str] = [] + for expected_column in self.STORY_ARC_COLUMNS: + actual_column = available_columns.get(expected_column.casefold()) + if actual_column is None: + expressions.append(f'NULL AS "{expected_column}"') + continue + quoted_column = actual_column.replace('"', '""') + expressions.append(f'"{quoted_column}" AS "{expected_column}"') + + select_columns = ", ".join(expressions) + query_with_rowid = f'SELECT {select_columns}, rowid AS "__source_rowid" FROM storyarcs' + try: + rows = conn.execute(query_with_rowid).fetchall() + except sqlite3.OperationalError: + query_without_rowid = ( + f'SELECT {select_columns}, NULL AS "__source_rowid" FROM storyarcs' + ) + rows = conn.execute(query_without_rowid).fetchall() + return rows, True + + def _read_readlist_count(self, conn: sqlite3.Connection) -> tuple[bool, int]: + """Inventory Mylar's distinct personal read list without importing it.""" + if not self._table_exists(conn, "readlist"): + return False, 0 + row = conn.execute("SELECT COUNT(*) AS row_count FROM readlist").fetchone() + return True, int(row["row_count"]) if row is not None else 0 + + def _convert_story_arc_rows( + self, + rows: list[sqlite3.Row], + ) -> tuple[Mylar3StoryArcSnapshot, ...]: + """Group raw Mylar rows while retaining unresolved and duplicate entries.""" + grouped: dict[tuple[str, str], list[tuple[int, sqlite3.Row]]] = {} + for source_index, row in enumerate(rows): + group_key = self._story_arc_group_key(row, source_index) + grouped.setdefault(group_key, []).append((source_index, row)) + + arcs: list[Mylar3StoryArcSnapshot] = [] + for grouped_rows in grouped.values(): + ordered_rows = sorted(grouped_rows, key=self._story_arc_row_sort_key) + entries = tuple( + self._story_arc_entry(row, ordinal=ordinal) + for ordinal, (_source_index, row) in enumerate(ordered_rows, start=1) + ) + arcs.append( + Mylar3StoryArcSnapshot( + story_arc_id=self._first_story_arc_value(entries, "story_arc_id"), + cv_arc_id=self._first_story_arc_value(entries, "cv_arc_id"), + name=self._first_story_arc_value(entries, "story_arc_name"), + entries=entries, ) ) + return tuple( + sorted( + arcs, + key=lambda arc: ( + arc.name is None, + (arc.name or "").casefold(), + arc.story_arc_id or "", + arc.cv_arc_id or "", + ), + ) + ) + + def _story_arc_group_key( + self, + row: sqlite3.Row, + source_index: int, + ) -> tuple[str, str]: + story_arc_id = self._optional_text(row["StoryArcID"]) + if story_arc_id: + return "story_arc_id", story_arc_id + cv_arc_id = self._optional_text(row["CV_ArcID"]) + if cv_arc_id: + return "cv_arc_id", cv_arc_id + story_arc_name = self._optional_text(row["StoryArc"]) + if story_arc_name: + return "story_arc_name", story_arc_name + source_rowid = self._parse_optional_int(row["__source_rowid"]) + return "unidentified_row", str(source_rowid if source_rowid is not None else source_index) + + def _story_arc_row_sort_key( + self, + item: tuple[int, sqlite3.Row], + ) -> tuple[object, ...]: + source_index, row = item + reading_order = self._parse_optional_int(row["ReadingOrder"]) + source_rowid = self._parse_optional_int(row["__source_rowid"]) + return ( + reading_order is None, + reading_order if reading_order is not None else 0, + source_rowid is None, + source_rowid if source_rowid is not None else 0, + self._optional_text(row["IssueArcID"]) or "", + self._optional_text(row["IssueID"]) or "", + self._optional_text(row["ComicID"]) or "", + self._optional_text(row["IssueNumber"]) or "", + source_index, + ) + + def _story_arc_entry( + self, + row: sqlite3.Row, + *, + ordinal: int, + ) -> Mylar3StoryArcEntrySnapshot: + reading_order_raw = self._optional_text(row["ReadingOrder"]) + return Mylar3StoryArcEntrySnapshot( + ordinal=ordinal, + reading_order=self._parse_optional_int(reading_order_raw), + reading_order_raw=reading_order_raw, + story_arc_id=self._optional_text(row["StoryArcID"]), + story_arc_name=self._optional_text(row["StoryArc"]), + cv_arc_id=self._optional_text(row["CV_ArcID"]), + issue_arc_id=self._optional_text(row["IssueArcID"]), + issue_id=self._optional_text(row["IssueID"]), + comic_id=self._optional_text(row["ComicID"]), + issue_number=self._optional_text(row["IssueNumber"]), + comic_name=self._optional_text(row["ComicName"]), + series_year=self._optional_text(row["SeriesYear"]), + issue_year=self._optional_text(row["IssueYEAR"]), + status=self._optional_text(row["Status"]), + location=self._optional_text(row["Location"]), + release_date=self._optional_text(row["ReleaseDate"]), + issue_date=self._optional_text(row["IssueDate"]), + publisher=self._optional_text(row["Publisher"]), + issue_publisher=self._optional_text(row["IssuePublisher"]), + issue_name=self._optional_text(row["IssueName"]), + manual=self._optional_text(row["Manual"]), + date_added=self._optional_text(row["DateAdded"]), + digital_date=self._optional_text(row["DigitalDate"]), + issue_type=self._optional_text(row["Type"]), + aliases=self._optional_text(row["Aliases"]), + total_issues=self._optional_text(row["TotalIssues"]), + in_cache_dir=self._optional_text(row["inCacheDir"]), + int_issue_number=self._optional_text(row["Int_IssueNumber"]), + dynamic_comic_name=self._optional_text(row["DynamicComicName"]), + volume=self._optional_text(row["Volume"]), + arc_image=self._optional_text(row["ArcImage"]), + ) + + def _first_story_arc_value( + self, + entries: tuple[Mylar3StoryArcEntrySnapshot, ...], + attribute_name: str, + ) -> str | None: + for entry in entries: + value = getattr(entry, attribute_name) + if isinstance(value, str) and value: + return value + return None + + def _optional_text(self, value: object) -> str | None: + return None if value is None else str(value) + + def _parse_optional_int(self, value: object) -> int | None: + if value is None: + return None + try: + return int(str(value).strip()) + except (TypeError, ValueError): + return None + + def _table_exists(self, conn: sqlite3.Connection, table_name: str) -> bool: + if table_name not in {"storyarcs", "readlist"}: + return False + row = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=? COLLATE NOCASE", + (table_name,), + ).fetchone() + return row is not None + def _table_has_columns( self, conn: sqlite3.Connection, table_name: str, required_columns: set[str], ) -> bool: - return required_columns.issubset(self._table_columns(conn, table_name)) + available = {column.casefold() for column in self._table_columns(conn, table_name)} + return {column.casefold() for column in required_columns}.issubset(available) def _table_columns(self, conn: sqlite3.Connection, table_name: str) -> set[str]: - if table_name == "issues": - rows = conn.execute("PRAGMA table_info(issues)").fetchall() - elif table_name == "annuals": - rows = conn.execute("PRAGMA table_info(annuals)").fetchall() - else: + queries = { + "issues": "PRAGMA table_info(issues)", + "annuals": "PRAGMA table_info(annuals)", + "storyarcs": "PRAGMA table_info(storyarcs)", + "readlist": "PRAGMA table_info(readlist)", + } + query = queries.get(table_name) + if query is None: return set() + rows = list(conn.execute(query)) return {str(row["name"]) for row in rows} - def _get_file_count(self, location: str | None) -> int: - """Count comic files in a directory if it exists. Returns 0 otherwise.""" - resolved = self._resolve_location(location) - if not resolved: - return 0 - p = Path(resolved) - if not p.exists() or not p.is_dir(): + def _get_resolved_file_count(self, resolved_path: Path | None) -> int: + """Count comic files in one validated resolved directory.""" + if resolved_path is None: return 0 - return sum(1 for f in p.iterdir() if f.suffix.lower() in COMIC_EXTENSIONS) + return sum(1 for f in resolved_path.iterdir() if f.suffix.lower() in COMIC_EXTENSIONS) diff --git a/src/pullbox/core/mylar_story_arc_policy.py b/src/pullbox/core/mylar_story_arc_policy.py new file mode 100644 index 00000000..3ec10d09 --- /dev/null +++ b/src/pullbox/core/mylar_story_arc_policy.py @@ -0,0 +1,300 @@ +"""Translate allowlisted Mylar story-arc settings into a review-only draft. + +The translator is deliberately data-only. It does not resolve paths, inspect +library roots, call providers, or activate a policy. A later confirmation +boundary must select an approved root and validate the proposed destination. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +from pullbox.core.story_arc_naming import ( + DEFAULT_STORY_ARC_FILE_TEMPLATE, + DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + validate_story_arc_folder_template, +) + +if TYPE_CHECKING: + from pullbox.core.mylar3_reader import Mylar3ArcSettingsSnapshot + +MYLAR_STORY_ARC_POLICY_DRAFT_SCHEMA_VERSION = 1 +MYLAR_READING_ORDER_FILE_TEMPLATE = DEFAULT_STORY_ARC_FILE_TEMPLATE +MYLAR_PLAIN_FILE_TEMPLATE = "{Series} {IssueNumber}{IssueTitleOptional}" + +_EXPECTED_SETTINGS = frozenset( + { + "STORYARCDIR", + "STORYARC_LOCATION", + "COPY2ARCDIR", + "ARC_FOLDERFORMAT", + "ARC_FILEOPS", + "ARC_FILEOPS_SOFTLINK_RELATIVE", + "UPCOMING_STORYARCS", + "SEARCH_STORYARCS", + "READ2FILENAME", + } +) +_DEFAULT_FOLDER_FORMAT = "$arc" +_DEFAULT_FILE_OPERATION = "copy" +_MAX_DESTINATION_BYTES = 1000 +_MYLAR_FOLDER_TOKEN_RE = re.compile(r"\$(?P[A-Za-z_][A-Za-z0-9_]*)") +_WARNING_CODE_RE = re.compile(r"[^A-Za-z0-9_.:-]+") +_FOLDER_TOKEN_MAP = { + "arc": "{StoryArc}", + "spanyears": "{SpanYears}", + "publisher": "{Publisher}", +} + + +def build_mylar_story_arc_policy_draft( + settings: Mylar3ArcSettingsSnapshot, +) -> dict[str, object]: + """Return one safe proposal that always requires explicit confirmation. + + ``placement_policy`` deliberately uses the exact keys accepted by the + placement-policy boundary. Its root ID remains unset and its destination is + only source evidence until the user approves and validates both values. + """ + warnings = [_warning_code(item) for item in settings.parse_warnings] + values: dict[str, bool | str | None] = {} + for setting in settings.values: + if setting.key not in _EXPECTED_SETTINGS: + warnings.append(f"unsupported_setting:{_warning_code(setting.key)}") + continue + if setting.key in values: + warnings.append(f"duplicate_setting:{_warning_code(setting.key)}") + continue + values[setting.key] = setting.value + + story_arc_directory = _bool_setting( + values, + "STORYARCDIR", + default=False, + warnings=warnings, + ) + destination_root = _optional_string_setting( + values, + "STORYARC_LOCATION", + default=None, + warnings=warnings, + ) + copy_to_arc_directory = _bool_setting( + values, + "COPY2ARCDIR", + default=False, + warnings=warnings, + ) + raw_folder_format = _string_setting( + values, + "ARC_FOLDERFORMAT", + default=_DEFAULT_FOLDER_FORMAT, + warnings=warnings, + ) + raw_file_operation = _string_setting( + values, + "ARC_FILEOPS", + default=_DEFAULT_FILE_OPERATION, + warnings=warnings, + ) + relative_symlink = _bool_setting( + values, + "ARC_FILEOPS_SOFTLINK_RELATIVE", + default=False, + warnings=warnings, + ) + include_upcoming = _bool_setting( + values, + "UPCOMING_STORYARCS", + default=False, + warnings=warnings, + ) + search_missing = _bool_setting( + values, + "SEARCH_STORYARCS", + default=False, + warnings=warnings, + ) + prefix_reading_order = _bool_setting( + values, + "READ2FILENAME", + default=False, + warnings=warnings, + ) + + mode = _placement_mode( + story_arc_directory=story_arc_directory, + copy_to_arc_directory=copy_to_arc_directory, + file_operation=raw_file_operation, + warnings=warnings, + ) + folder_template = _translate_folder_template(raw_folder_format, warnings=warnings) + file_template = ( + MYLAR_READING_ORDER_FILE_TEMPLATE if prefix_reading_order else MYLAR_PLAIN_FILE_TEMPLATE + ) + synchronize = mode in {"copy", "hardlink", "symlink"} + symlink_style = "relative" if relative_symlink else "absolute" if mode == "symlink" else None + if mode != "symlink": + symlink_style = None + + requires_destination = mode != "logical" + proposed_destination = _bounded_destination(destination_root, warnings=warnings) + if not requires_destination: + proposed_destination = None + elif proposed_destination is None: + warnings.append("destination_root_missing") + + return { + "schema_version": MYLAR_STORY_ARC_POLICY_DRAFT_SCHEMA_VERSION, + "source": "mylar3", + "activation": "requires_confirmation", + "settings_present": bool(settings.present), + # Explicit confirmation is required even when no warning was detected. + "review_required": True, + "review_warnings": _unique_warnings(warnings), + "monitored": search_missing or include_upcoming, + "search_missing": search_missing, + "include_upcoming": include_upcoming, + "sync_enabled": synchronize, + "placement_policy": { + "schema_version": 1, + "mode": mode, + "target_library_root_id": None, + "destination_root": proposed_destination, + "folder_template": folder_template, + "file_template": file_template, + "symlink_style": symlink_style, + "synchronize": synchronize, + }, + "confirmation": { + "target_library_root_required": requires_destination, + "destination_root_requires_approval": requires_destination, + "ready_for_activation": False, + }, + } + + +def _placement_mode( + *, + story_arc_directory: bool, + copy_to_arc_directory: bool, + file_operation: str, + warnings: list[str], +) -> str: + if not story_arc_directory: + if copy_to_arc_directory: + warnings.append("copy_without_story_arc_directory") + return "logical" + if not copy_to_arc_directory: + return "reference_only" + + normalized = file_operation.strip().casefold() + if normalized == "copy": + return "copy" + if normalized == "move": + warnings.append("legacy_move_mapped_to_copy") + return "copy" + if normalized == "hardlink": + return "hardlink" + if normalized == "softlink": + return "symlink" + warnings.append("unsupported_arc_fileops") + return "reference_only" + + +def _translate_folder_template(raw: str, *, warnings: list[str]) -> str: + unsupported: list[str] = [] + + def replace(match: re.Match[str]) -> str: + source_name = match.group("name") + replacement = _FOLDER_TOKEN_MAP.get(source_name.casefold()) + if replacement is None: + unsupported.append(_warning_code(source_name).casefold()) + return match.group(0) + return replacement + + translated = _MYLAR_FOLDER_TOKEN_RE.sub(replace, raw.strip()) + warnings.extend(f"unsupported_folder_token:{name}" for name in unsupported) + if unsupported or "$" in translated: + warnings.append("invalid_arc_folderformat") + return DEFAULT_STORY_ARC_FOLDER_TEMPLATE + try: + validate_story_arc_folder_template(translated) + except ValueError: + warnings.append("invalid_arc_folderformat") + return DEFAULT_STORY_ARC_FOLDER_TEMPLATE + return translated + + +def _bool_setting( + values: dict[str, bool | str | None], + key: str, + *, + default: bool, + warnings: list[str], +) -> bool: + if key not in values: + warnings.append(f"missing_setting:{key}") + return default + value = values[key] + if not isinstance(value, bool): + warnings.append(f"invalid_setting_type:{key}") + return default + return value + + +def _string_setting( + values: dict[str, bool | str | None], + key: str, + *, + default: str, + warnings: list[str], +) -> str: + if key not in values: + warnings.append(f"missing_setting:{key}") + return default + value = values[key] + if not isinstance(value, str): + warnings.append(f"invalid_setting_type:{key}") + return default + return value + + +def _optional_string_setting( + values: dict[str, bool | str | None], + key: str, + *, + default: str | None, + warnings: list[str], +) -> str | None: + if key not in values: + warnings.append(f"missing_setting:{key}") + return default + value = values[key] + if value is None: + return None + if not isinstance(value, str): + warnings.append(f"invalid_setting_type:{key}") + return default + normalized = value.strip() + return normalized or None + + +def _bounded_destination(value: str | None, *, warnings: list[str]) -> str | None: + if value is None: + return None + if len(value.encode("utf-8")) > _MAX_DESTINATION_BYTES: + warnings.append("destination_root_too_long") + return None + return value + + +def _warning_code(value: object) -> str: + bounded = str(value)[:200] + normalized = _WARNING_CODE_RE.sub("_", bounded).strip("_") + return normalized or "unknown" + + +def _unique_warnings(warnings: list[str]) -> list[str]: + return list(dict.fromkeys(warnings)) diff --git a/src/pullbox/core/name_matcher.py b/src/pullbox/core/name_matcher.py index 6bf8d6b8..74ae177e 100644 --- a/src/pullbox/core/name_matcher.py +++ b/src/pullbox/core/name_matcher.py @@ -23,6 +23,57 @@ _WHITESPACE_RE = re.compile(r"\s+") _ARTICLES = frozenset({"the", "a", "an"}) _PUBLICATION_FORM_SUFFIXES = frozenset({"magazine"}) +_TITLE_COMPAT_TRANSLATION = str.maketrans( + { + "Ø": "O", + "ø": "o", + "ẞ": "SS", + "ß": "ss", + "Ł": "L", + "ł": "l", + "Æ": "AE", + "æ": "ae", + "Œ": "OE", + "œ": "oe", + "Đ": "D", + "đ": "d", + "Ð": "D", + "ð": "d", + "Þ": "Th", + "þ": "th", + "ı": "i", # noqa: RUF001 - intentional dotless i + "Ə": "A", + "ə": "a", + "Ǝ": "E", + "ǝ": "e", + "♂": " male ", + "♀": " female ", + "×": "x", # noqa: RUF001 - intentional Unicode multiplication sign + "✕": "x", + "✖": "x", + "+": " plus ", + "@": " at ", + "%": " percent ", + "№": " number ", + "#": " number ", + "=": " equals ", + "$": " dollar ", + "€": " euro ", + "£": " pound ", + "¥": " yen ", + "∞": " infinity ", + "☆": " ", + "★": " ", + "●": " ", + "○": " ", + "•": " ", + "・": " ", + "·": " ", + "®": "", + "©": "", + "™": "", + } +) def _compact_normalized(name: str) -> str: @@ -62,8 +113,19 @@ def normalize(name: str) -> str: # Step 1: HTML entity decode (& → &) s = html.unescape(name) + # Preserve semantic title symbols before compatibility normalization + # can rewrite them (for example, ``№`` becomes ``No`` under NFKD). + s = s.translate(_TITLE_COMPAT_TRANSLATION) + # Step 2: Unicode normalization (NFKD) — decomposes accented chars s = unicodedata.normalize("NFKD", s) + # Remove decomposed combining marks before punctuation normalization. + # Otherwise each accent becomes a space and splits provider-query words + # (for example, ``Remède`` became ``reme de``). + s = "".join(char for char in s if not unicodedata.combining(char)) + # NFKD converts full-width operators to ASCII; give those the same + # stable semantics as their native-width forms. + s = s.translate(_TITLE_COMPAT_TRANSLATION) # Step 3: Replace unicode dashes with ASCII hyphen s = s.replace("\u2014", "-").replace("\u2013", "-").replace("\u2012", "-") diff --git a/src/pullbox/core/naming.py b/src/pullbox/core/naming.py index 1e22d941..4336a84d 100644 --- a/src/pullbox/core/naming.py +++ b/src/pullbox/core/naming.py @@ -12,6 +12,7 @@ import re from dataclasses import dataclass, field +from pullbox.core.issue_numbers import normalize_issue_number_text from pullbox.core.naming_type_detection import ( classify_series_type as classify_series_type, ) @@ -574,7 +575,7 @@ def normalize_issue_type_for_naming( def format_comic_file( series: str, year: int | None = None, - issue: float | None = None, + issue: str | float | None = None, volume: int | None = None, issue_type: str = "issue", title: str | None = None, @@ -590,12 +591,13 @@ def format_comic_file( Supported tokens: ``{Series}``, ``{Year}``, ``{Issue}``, ``{Issue:03d}``, - ``{Volume}``, ``{Volume:02d}``, ``{Type}``, ``{Title}``, ``{Publisher}`` + ``{Volume}``, ``{Volume:02d}``, ``{Type}``, ``{IssueTitle}``, + ``{Title}``, ``{Publisher}`` Args: series: Series title. year: Year (from series or release date). - issue: Issue number (may be ``None`` for one-shots/GNs). + issue: Exact issue designation or legacy number (``None`` for unnumbered items). volume: Volume number (for TPBs, omnibuses, etc.). issue_type: String value of ``IssueType`` enum. title: Issue/collection title. @@ -641,6 +643,7 @@ def format_comic_file( name = name.replace("{Series}", clean_series) name = name.replace("{Year}", year_str) name = name.replace("{Type}", type_display) + name = name.replace("{IssueTitle}", title_str) name = name.replace("{Title}", title_str) name = name.replace("{Publisher}", publisher_str) # Keep older saved templates from leaking the deprecated token literally. @@ -648,11 +651,15 @@ def format_comic_file( # Handle {Issue:03d} and {Issue} — leave blank if None if issue is not None: - issue_val = int(issue) if issue == int(issue) else issue - if "{Issue:03d}" in name: - name = name.replace("{Issue:03d}", f"{issue_val:03d}") - if "{Issue}" in name: - name = name.replace("{Issue}", str(issue_val)) + issue_text = normalize_issue_number_text(issue) + # Pad the integer component without rounding fractions or dropping suffixes. + padded_issue = re.sub( + r"^([+-]?)([0-9]+)", + lambda match: match.group(1) + match.group(2).zfill(3 - len(match.group(1))), + issue_text, + ) + name = name.replace("{Issue:03d}", padded_issue) + name = name.replace("{Issue}", issue_text) else: name = name.replace("#{Issue:03d}", "").replace("{Issue:03d}", "") name = name.replace("#{Issue}", "").replace("{Issue}", "") @@ -728,6 +735,9 @@ def format_filename( def get_naming_preview( template: str, template_type: str = "standard", + *, + replace_illegal: bool = True, + colon_replacement: str = "dash", ) -> list[dict[str, str]]: """Generate preview examples for a naming template. @@ -765,13 +775,23 @@ def get_naming_preview( if template_type == "folder": # Map issue_type to series_type for folder preview folder_type = ex.issue_type if ex.issue_type != "issue" else "standard" - output = format_series_folder( - title=ex.series, - year=ex.year, - publisher=ex.publisher, - comicvine_id=ex.comicvine_id, - series_type=folder_type, - template=template, + # The runtime treats each slash-delimited component as a separate, + # sanitized directory. Preserve that contract in the global preview + # instead of sanitizing the whole path as one folder name. + from pullbox.core.library_naming import validate_series_path_template + + output = "/".join( + format_series_folder( + title=ex.series, + year=ex.year, + publisher=ex.publisher, + comicvine_id=ex.comicvine_id, + series_type=folder_type, + template=segment, + replace_illegal=replace_illegal, + colon_replacement=colon_replacement, + ) + for segment in validate_series_path_template(template) ) else: output = format_comic_file( @@ -783,6 +803,8 @@ def get_naming_preview( title=ex.title, publisher=ex.publisher, template=template, + replace_illegal=replace_illegal, + colon_replacement=colon_replacement, ) # Build a human-readable input description diff --git a/src/pullbox/core/release_parser.py b/src/pullbox/core/release_parser.py index 1d15cbf5..ec2c737b 100644 --- a/src/pullbox/core/release_parser.py +++ b/src/pullbox/core/release_parser.py @@ -18,6 +18,8 @@ import re from dataclasses import dataclass +from pullbox.core.issue_numbers import parse_issue_number_text +from pullbox.core.name_matcher import NameMatcher from pullbox.models.issue import IssueType # --------------------------------------------------------------------------- @@ -135,11 +137,21 @@ _YEAR_PAREN_RE = re.compile(r"\((\d{4})\)") # Issue number with hash prefix: #045, #5, #5.1 -_ISSUE_HASH_RE = re.compile(r"#(\d+(?:\.\d+)?)") +_ISSUE_HASH_RE = re.compile(r"#(\d+(?:\.\d+)?[A-Za-z]*)") + +# DC's One Million event used the literal issue number 1,000,000 across +# multiple ongoing titles. Keep this exact exception narrow so arbitrary long +# numeric title tokens do not become issue numbers. +_DC_ONE_MILLION_ISSUE_RE = re.compile(r"(?<=\s)(1000000)(?=\s|$)") # Long-running UK weekly anthologies often label issue numbers as "Prog 2483". _PROG_ISSUE_RE = re.compile(r"\bProg(?:ramme)?\.?\s*#?\s*(\d+(?:\.\d+)?)\b", re.IGNORECASE) +# Bare four-digit issues are unambiguous only at the end of the stripped title. +_LONG_POSITIONAL_ISSUE_RE = re.compile(r"(?<=\s)(\d{4}(?:\.\d+)?[A-Za-z]*)\s*$") +_RANGE_OR_COUNT_PREFIX_RE = re.compile(r"(?:\d\s*[-\u2013\u2014]|\bof)\s*$", re.IGNORECASE) +_VOLUME_YEAR_RE = re.compile(r"\b(?:v|vol(?:ume)?\.?)\s*((?:19|20)\d{2})\b", re.IGNORECASE) + # Limited series marker: (of 05) _LIMITED_SERIES_RE = re.compile(r"\(of\s+\d+\)", re.IGNORECASE) _INLINE_LIMITED_SERIES_RE = re.compile( @@ -248,6 +260,8 @@ class ParsedRelease: file_format: str | None is_pack: bool pack_range: str | None + issue_number_text: str | None = None + volume_year: int | None = None # --------------------------------------------------------------------------- @@ -329,7 +343,9 @@ def issues_match(wanted: float, found: float | None, tolerance: float = 0.001) - # --------------------------------------------------------------------------- -def parse_release_title(title: str) -> ParsedRelease | None: +def parse_release_title( + title: str, *, expected_series: tuple[str, ...] = () +) -> ParsedRelease | None: """Parse an NZB release title or local filename into structured components. Handles all common naming conventions: @@ -399,6 +415,13 @@ def parse_release_title(title: str) -> ParsedRelease | None: working, ).strip() + # An explicit v1977 is series identity, not a collection volume or issue date. + volume_year = None + volume_year_match = _VOLUME_YEAR_RE.search(working) + if volume_year_match: + volume_year = int(volume_year_match[1]) + working = working[: volume_year_match.start()] + working[volume_year_match.end() :] + # Step e: Detect issue type issue_type = _detect_type(working) @@ -413,7 +436,9 @@ def parse_release_title(title: str) -> ParsedRelease | None: pre_issue_is_pack, pre_issue_pack_range = _detect_pack(working) # Step h: Extract issue number - issue_number, working = _extract_issue_number(working, issue_type) + issue_number, working, issue_number_text = _extract_issue_number( + working, issue_type, expected_series=expected_series + ) # Step h½: Reclassify as VOLUME when volume is present but no issue # number was found and no other type was explicitly detected. @@ -440,6 +465,8 @@ def parse_release_title(title: str) -> ParsedRelease | None: file_format=file_format, is_pack=is_pack, pack_range=pack_range, + issue_number_text=issue_number_text, + volume_year=volume_year, ) @@ -707,10 +734,15 @@ def _extract_volume(title: str) -> tuple[str | None, str]: ) -def _extract_issue_number(title: str, issue_type: IssueType) -> tuple[float | None, str]: +def _extract_issue_number( + title: str, + issue_type: IssueType, + *, + expected_series: tuple[str, ...] = (), +) -> tuple[float | None, str, str | None]: """Step h: Extract issue number from the title. - Returns (issue_number, remaining_title). + Returns (numeric issue number, remaining title, exact issue designation). """ # Remove limited series markers first: (of 05) clean = _LIMITED_SERIES_RE.sub("", title).strip() @@ -719,68 +751,98 @@ def _extract_issue_number(title: str, issue_type: IssueType) -> tuple[float | No m = _WORD_NUMBER_RE.search(clean) if m: num = float(_WORD_NUMBERS[m.group(1).lower()]) + _, exact_text = parse_issue_number_text(num) remaining = clean[: m.start()] + clean[m.end() :] - return num, remaining.strip() + return num, remaining.strip(), exact_text # Priority 1: Hash prefix — #045, #5, #5.1 m = _ISSUE_HASH_RE.search(clean) if m: - num = float(m.group(1)) + num, exact_text = parse_issue_number_text(m.group(1)) + remaining = clean[: m.start()] + clean[m.end() :] + return num, remaining.strip(), exact_text + + # Priority 2: DC One Million's literal issue 1,000,000 without a hash. + m = _DC_ONE_MILLION_ISSUE_RE.search(clean) + if m: + num, exact_text = parse_issue_number_text(m.group(1)) remaining = clean[: m.start()] + clean[m.end() :] - return num, remaining.strip() + return num, remaining.strip(), exact_text - # Priority 2: Anthology "Prog" marker — supports long-running series + # Priority 3: Anthology "Prog" marker — supports long-running series # whose issue numbers exceed the usual 2-3 digit positional heuristic. m = _PROG_ISSUE_RE.search(clean) if m: - num = float(m.group(1)) + num, exact_text = parse_issue_number_text(m.group(1)) remaining = clean[: m.start()] + clean[m.end() :] - return num, remaining.strip() + return num, remaining.strip(), exact_text - # Priority 3: "No." pattern from dot-separated titles + # Priority 4: "No." pattern from dot-separated titles m = _DOT_NO_ISSUE_RE.search(clean) if m: - num = float(m.group(1)) + num, exact_text = parse_issue_number_text(m.group(1)) remaining = clean[: m.start()] + clean[m.end() :] - return num, remaining.strip() + return num, remaining.strip(), exact_text - # Priority 4: Inline limited-series marker — "02 of 03". The first number + # Priority 5: Inline limited-series marker — "02 of 03". The first number # is the issue; the second is the total issue count. m = _INLINE_LIMITED_SERIES_RE.search(clean) if m: - num = float(m.group(1)) + num, exact_text = parse_issue_number_text(m.group(1)) remaining = clean[: m.start()] + clean[m.end() :] - return num, remaining.strip() + return num, remaining.strip(), exact_text + + # Priority 6: Long-running series may omit "Prog" or "#" before issue 2487. + # Require a known series prefix: a title such as Marvel 1602 is not issue 1602. + m = _LONG_POSITIONAL_ISSUE_RE.search(clean) + if m and expected_series: + token = m.group(1) + prefix = clean[: m.start()].rstrip() + prefix_names = { + NameMatcher.normalize(name).replace(" ", "") + for name in (prefix, _clean_series_name(prefix, issue_type) or "") + if name + } + if ( + any( + NameMatcher.normalize(name).replace(" ", "") in prefix_names + for name in expected_series + ) + and not _RESOLUTION_TAG_RE.fullmatch(token) + and not _RANGE_OR_COUNT_PREFIX_RE.search(prefix) + ): + num, exact_text = parse_issue_number_text(token) + return num, prefix, exact_text - # Priority 5: Positional number — a 2-3 digit number that sits between + # Priority 7: Positional number — a 2-3 digit number that sits between # the series name and metadata (year/brackets) # Match a number preceded by space (or after series-name text) # but NOT part of an alphanumeric word like "D4VE2" or "Spider-Man 2099" positional_matches = list( re.finditer( - r"(?<=\s)(\d{2,3})(?:\.\d+)?(?=\s|$)", + r"(?<=\s)(\d{2,3}(?:\.\d+)?[A-Za-z]*)(?=\s|$)", clean, ) ) m = _select_positional_issue_match(positional_matches) if m: num_str = m.group(0) - num = float(num_str) + num, exact_text = parse_issue_number_text(num_str) # Avoid treating large numbers that could be years as issue numbers if num < 500: remaining = clean[: m.start()] + clean[m.end() :] - return num, remaining.strip() + return num, remaining.strip(), exact_text # If num >= 500, check if it might still be a valid issue (e.g., long-running series) # but only if there's no year already found and the number doesn't look like a year if num >= 1900: - return None, clean + return None, clean, None # Numbers 500-1899 — treat as issue for very long-running series remaining = clean[: m.start()] + clean[m.end() :] - return num, remaining.strip() + return num, remaining.strip(), exact_text - # Priority 6: Single digit number at word boundary after text + # Priority 8: Single digit number at word boundary after text # Must NOT be followed by a word (e.g. "4 Covers" is a count, not issue #4) - m = re.search(r"(?<=\s)(\d)(?=\s|$)", clean) + m = re.search(r"(?<=\s)(\d[A-Za-z]*)(?=\s|$)", clean) if m: # Check the word after the digit — if it's alphabetic, this is likely # a count or descriptor (e.g. "4 Covers", "3 Stories"), not an issue number @@ -788,11 +850,11 @@ def _extract_issue_number(title: str, issue_type: IssueType) -> tuple[float | No if after and after[0].isalpha(): pass # skip — looks like "N ", not an issue number else: - num = float(m.group(1)) + num, exact_text = parse_issue_number_text(m.group(1)) remaining = clean[: m.start()] + clean[m.end() :] - return num, remaining.strip() + return num, remaining.strip(), exact_text - return None, clean + return None, clean, None def _select_positional_issue_match(matches: list[re.Match[str]]) -> re.Match[str] | None: diff --git a/src/pullbox/core/release_year_matching.py b/src/pullbox/core/release_year_matching.py new file mode 100644 index 00000000..f5b15546 --- /dev/null +++ b/src/pullbox/core/release_year_matching.py @@ -0,0 +1,59 @@ +"""Date evidence for release matching, separate from provider query years.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime + +from pullbox.models.issue import IssueType + + +@dataclass(frozen=True, slots=True) +class ReleaseYearContext: + series_year: int | None = None + publication_years: tuple[int, ...] = () + series_continuing: bool = False + + +@dataclass(frozen=True, slots=True) +class ReleaseYearEvidence: + matches: bool | None + basis: str + weak: bool = False + + +def match_release_year( + year: int | None, + *, + wanted_year: int | None, + volume_year: int | None = None, + context: ReleaseYearContext | None = None, + issue_type: IssueType = IssueType.ISSUE, + tolerance: int = 1, +) -> ReleaseYearEvidence: + if context is not None: + explicit_series_match = volume_year is not None and context.series_year is not None + if explicit_series_match and volume_year != context.series_year: + return ReleaseYearEvidence(False, "explicit_series_year_mismatch") + if year is not None and context.publication_years: + matches = any( + abs(year - candidate) <= tolerance for candidate in context.publication_years + ) + return ReleaseYearEvidence( + matches, "publication_year" if matches else "publication_year_mismatch" + ) + if year is None and explicit_series_match: + return ReleaseYearEvidence(True, "explicit_series_year") + if ( + year is not None + and issue_type is IssueType.ISSUE + and context.series_continuing + and context.series_year is not None + ): + plausible = context.series_year <= year <= datetime.now(UTC).year + return ReleaseYearEvidence( + plausible, "series_window" if plausible else "outside_series_window", weak=True + ) + if year is None or wanted_year is None: + return ReleaseYearEvidence(None, "unknown") + return ReleaseYearEvidence(abs(year - wanted_year) <= tolerance, "target_year") diff --git a/src/pullbox/core/scheduler.py b/src/pullbox/core/scheduler.py index 502a43ec..2fc7d007 100644 --- a/src/pullbox/core/scheduler.py +++ b/src/pullbox/core/scheduler.py @@ -79,6 +79,8 @@ is_missing_import_jobs_table_error, ) +_IMPORT_PROTECTION_ALLOWED_TASK_IDS = frozenset({"sync_story_arc_placements"}) + if TYPE_CHECKING: from collections.abc import Callable from pathlib import Path @@ -575,6 +577,9 @@ async def _defer_task_for_import(self, task_id: str, log: Any, *, trigger_type: ) return True + if task_id in _IMPORT_PROTECTION_ALLOWED_TASK_IDS: + return False + if self._import_protection_check_disabled: return False diff --git a/src/pullbox/core/source_metadata.py b/src/pullbox/core/source_metadata.py index e6ca8f63..780ceddc 100644 --- a/src/pullbox/core/source_metadata.py +++ b/src/pullbox/core/source_metadata.py @@ -3,28 +3,24 @@ from __future__ import annotations import enum -import json import re from collections import Counter from dataclasses import dataclass, field, replace +from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, Any, TypedDict, cast +from typing import Any, TypedDict, cast from pullbox.core.archive import ArchiveError, ArchiveReader +from pullbox.core.comicinfo import ComicInfoData +from pullbox.core.issue_numbers import format_issue_number, parse_issue_number_text from pullbox.core.naming import detect_issue_type from pullbox.core.release_parser import ParsedRelease, normalize_issue_number, parse_release_title +from pullbox.core.source_sidecars import parse_source_sidecar from pullbox.core.type_semantics import TypeFamily, issue_type_family from pullbox.models.issue import IssueType -if TYPE_CHECKING: - from pullbox.core.comicinfo import ComicInfoData - _CV_ISSUE_URL_RE = re.compile(r"comicvine\.gamespot\.com/.*?/4000-(\d+)", re.IGNORECASE) _CV_SERIES_URL_RE = re.compile(r"comicvine\.gamespot\.com/.*?/4050-(\d+)", re.IGNORECASE) -_CV_ANY_ID_RE = re.compile( - r"\b(?:comicid|comicvineid|cv_vol_id|cvid|issueid)\s*[:=]\s*(\d+)\b", - re.I, -) _CV_NOTES_PATTERNS = ( re.compile(r"\[cv_vol_id:(\d+)\]", re.IGNORECASE), re.compile(r"\[cvid:(\d+)\]", re.IGNORECASE), @@ -35,6 +31,10 @@ re.compile(r"\bissueid[:\s]+(\d+)\b", re.IGNORECASE), ) _YEAR_RE = re.compile(r"\b(19|20)\d{2}\b") +_EXPLICIT_STANDARD_ISSUE_RE = re.compile( + r"(?:^|[\s._-])issues?\s*(?:#|no\.?)?\s*[+-]?\d+(?:\.\d+)?[A-Za-z]?\b", + re.IGNORECASE, +) _ORDINAL_SUFFIX_WORDS: dict[str, int] = { "one": 1, "two": 2, @@ -178,6 +178,11 @@ def volume_subtitle_hint_from_filename( base_series, flags=re.IGNORECASE, ) + base_series = re.sub( + r"\s*(?:\((?:19|20)\d{2}\)|\[(?:19|20)\d{2}\]|(?:19|20)\d{2})\s*$", + "", + base_series, + ) base_series = re.sub(r"\s+", " ", base_series).strip(" -:") base_series = _VOLUME_HINT_PUBLISHER_PREFIX_RE.sub("", base_series).strip(" -:") subtitle = _clean_volume_subtitle_hint(normalized_stem[match.end() :]) @@ -205,6 +210,19 @@ def archive_entry_issue_hint_from_names( if len(image_entries) < _ARCHIVE_HINT_MIN_PARSEABLE_IMAGES: return None + @lru_cache(maxsize=256) + def parse_candidate(title: str) -> tuple[str, float, int | None] | None: + # Page suffixes often reduce hundreds of filenames to one release title. + parsed = parse_release_title(title) + if parsed is None or parsed.issue_number is None or not parsed.series_name: + return None + if expected_series_name and not _archive_entry_series_matches( + parsed.series_name, + expected_series_name, + ): + return None + return parsed.series_name, float(parsed.issue_number), parsed.year + parsed_entries: list[tuple[str, float, int | None, str]] = [] for entry in image_entries: file_name = entry.replace("\\", "/").rsplit("/", 1)[-1] @@ -212,15 +230,9 @@ def archive_entry_issue_hint_from_names( candidate_title = _ARCHIVE_PAGE_SUFFIX_RE.sub("", stem).strip() if not candidate_title or candidate_title == stem: continue - parsed = parse_release_title(candidate_title) - if parsed is None or parsed.issue_number is None or not parsed.series_name: - continue - if expected_series_name and not _archive_entry_series_matches( - parsed.series_name, - expected_series_name, - ): - continue - parsed_entries.append((parsed.series_name, float(parsed.issue_number), parsed.year, entry)) + identity = parse_candidate(candidate_title) + if identity is not None: + parsed_entries.append((*identity, entry)) total_image_entries = len(image_entries) parseable_image_entries = len(parsed_entries) @@ -282,6 +294,7 @@ class MetadataSignal(enum.StrEnum): FOLDER_HINT = "folder_hint" PULLBOX_FOLDER = "pullbox_folder" MYLAR3 = "mylar3" + SOURCE_LAYOUT = "source_layout" @dataclass(frozen=True, slots=True) @@ -292,6 +305,7 @@ class SourceMetadata: source_path: str | None = None series_name: str | None = None issue_number: float | None = None + issue_number_text: str | None = None year: int | None = None volume: str | None = None issue_type: IssueType = IssueType.ISSUE @@ -320,6 +334,7 @@ def from_path( include_archive_comicinfo: bool = True, include_archive_entry_issue_hint: bool = True, sidecar_data: dict[str, Any] | None = None, + archive_member_evidence: dict[str, Any] | None = None, ) -> SourceMetadata: """Build metadata from filename, folder sidecars, and optional archive ComicInfo.""" path = Path(archive_path) @@ -329,20 +344,27 @@ def from_path( source_path=str(path), folder_name=folder_name, ) - comicinfo = self._read_archive_comicinfo(path) if include_archive_comicinfo else None - archive_entry_issue_hint = ( - self.archive_entry_issue_hint_from_path( - path, - expected_series_name=metadata.series_name, - ) + comicinfo: ComicInfoData | None = None + archive_entry_issue_hint: ArchiveEntryIssueHint | None = None + if include_archive_comicinfo: if ( - include_archive_comicinfo and include_archive_entry_issue_hint and comicinfo is None - ) - else None - ) + isinstance(archive_member_evidence, dict) + and archive_member_evidence.get("member_index_scanned") is True + ): + comicinfo = self._comicinfo_from_member_evidence(archive_member_evidence) + if comicinfo is None and include_archive_entry_issue_hint: + archive_entry_issue_hint = self._archive_hint_from_member_evidence( + archive_member_evidence + ) + else: + comicinfo, archive_entry_issue_hint = self._read_archive_evidence( + path, + expected_series_name=metadata.series_name, + include_archive_entry_issue_hint=include_archive_entry_issue_hint, + ) sidecar = sidecar_data if sidecar_data is not None else self._read_sidecars(path.parent) - return self._merge_path_metadata( + merged = self._merge_path_metadata( metadata=metadata, folder_name=folder_name, comicinfo=comicinfo, @@ -351,6 +373,29 @@ def from_path( include_archive_comicinfo=include_archive_comicinfo, include_archive_entry_issue_hint=include_archive_entry_issue_hint, ) + if ( + include_archive_comicinfo + and isinstance(archive_member_evidence, dict) + and archive_member_evidence.get("member_index_scanned") is True + ): + evidence_diagnostics: dict[str, object] = { + "archive_member_index_reused": True, + "comicinfo_entry_count": int( + archive_member_evidence.get("comicinfo_entry_count") or 0 + ), + } + comicinfo_error = archive_member_evidence.get("comicinfo_error") + if isinstance(comicinfo_error, str): + evidence_diagnostics["comicinfo_error"] = comicinfo_error + merged = merged.model_copy( + update={ + "diagnostics": { + **merged.diagnostics, + **evidence_diagnostics, + } + } + ) + return merged def from_release_title( self, @@ -358,22 +403,39 @@ def from_release_title( *, source_path: str | None = None, folder_name: str | None = None, + expected_series: tuple[str, ...] = (), ) -> SourceMetadata: - parsed = parse_release_title(title) + parsed = parse_release_title(title, expected_series=expected_series) folder_issue_type = self._folder_issue_type(folder_name) issue_type = IssueType.ISSUE if parsed is not None: issue_type = parsed.issue_type - if folder_issue_type is not None and (issue_type == IssueType.ISSUE or parsed is None): + if ( + parsed.issue_number is not None + and _EXPLICIT_STANDARD_ISSUE_RE.search(Path(title).stem) is not None + ): + # ``Issue 2 - Special Selection`` describes a numbered issue + # whose title contains a type word; it is not a Special. A + # sidecar BookType can still explicitly override this later. + issue_type = IssueType.ISSUE + folder_issue_type_applies = folder_issue_type is not None and ( + parsed is None or (issue_type == IssueType.ISSUE and parsed.issue_number is None) + ) + if folder_issue_type_applies and folder_issue_type is not None: issue_type = folder_issue_type signals: dict[str, MetadataSignal] = {} - if parsed and parsed.issue_type != IssueType.ISSUE: + if parsed and issue_type != IssueType.ISSUE: signals["issue_type"] = MetadataSignal.RELEASE_TITLE - elif folder_issue_type is not None: + elif parsed and parsed.issue_number is not None: + # A numbered issue is stronger evidence than a type keyword embedded + # in the series-folder title (for example, "Should be Special"). + signals["issue_type"] = MetadataSignal.RELEASE_TITLE + elif folder_issue_type_applies: signals["issue_type"] = MetadataSignal.FOLDER_HINT series_name = parsed.series_name if parsed is not None else None issue_number = parsed.issue_number if parsed is not None else None + issue_number_text = parsed.issue_number_text if parsed is not None else None year = parsed.year if parsed is not None else None volume = parsed.volume if parsed is not None else None diagnostics: dict[str, object] = {} @@ -385,13 +447,16 @@ def from_release_title( issue_number=volume_issue_number, ) volume_issue_applies = issue_type_family(issue_type) == TypeFamily.COLLECTION - if volume_issue_applies and volume_hint is not None and issue_number is None: + if volume_issue_applies and volume_hint is not None: series_name = str(volume_hint["base_series"]) - issue_number = volume_hint["issue_number"] - signals["issue_number"] = MetadataSignal.RELEASE_TITLE diagnostics["volume_subtitle_hint"] = volume_hint + if issue_number is None: + issue_number = volume_hint["issue_number"] + issue_number_text = format_issue_number(issue_number) + signals["issue_number"] = MetadataSignal.RELEASE_TITLE elif volume_issue_applies and issue_number is None and volume_issue_number is not None: issue_number = volume_issue_number + issue_number_text = format_issue_number(issue_number) signals["issue_number"] = MetadataSignal.RELEASE_TITLE return SourceMetadata( @@ -399,6 +464,7 @@ def from_release_title( source_path=source_path, series_name=series_name, issue_number=issue_number, + issue_number_text=issue_number_text, year=year, volume=volume, issue_type=issue_type, @@ -438,6 +504,7 @@ def _merge_path_metadata( ) -> SourceMetadata: series_name = metadata.series_name issue_number = metadata.issue_number + issue_number_text = metadata.issue_number_text year = metadata.year volume = metadata.volume publisher = metadata.publisher @@ -506,6 +573,12 @@ def _merge_path_metadata( } elif normalized_comicinfo_issue is not None: issue_number = normalized_comicinfo_issue + try: + _, issue_number_text = parse_issue_number_text( + comicinfo.number.strip().lstrip("#") + ) + except ValueError: + issue_number_text = format_issue_number(normalized_comicinfo_issue) signals["issue_number"] = MetadataSignal.COMICINFO else: diagnostics["comicinfo_issue_number_ignored"] = { @@ -544,6 +617,8 @@ def _merge_path_metadata( "publisher": comicinfo.publisher, "web": comicinfo.web, "notes": comicinfo.notes, + "story_arc": comicinfo.story_arc, + "story_arc_number": comicinfo.story_arc_number, } else: diagnostics["has_comicinfo"] = False @@ -598,6 +673,7 @@ def _merge_path_metadata( source_path=metadata.source_path, series_name=series_name, issue_number=issue_number, + issue_number_text=issue_number_text, year=year, volume=volume, issue_type=issue_type, @@ -631,6 +707,110 @@ def _read_archive_comicinfo(path: Path) -> ComicInfoData | None: except ArchiveError: return None + @staticmethod + def _read_archive_evidence( + path: Path, + *, + expected_series_name: str | None, + include_archive_entry_issue_hint: bool, + ) -> tuple[ComicInfoData | None, ArchiveEntryIssueHint | None]: + """Read ComicInfo or page-name evidence with one archive member listing.""" + try: + reader = ArchiveReader(path) + entries = reader.list_files() + comicinfo = reader.read_comicinfo(entries=entries) + except ArchiveError: + return None, None + if comicinfo is not None or not include_archive_entry_issue_hint: + return comicinfo, None + return ( + None, + archive_entry_issue_hint_from_names( + entries, + expected_series_name=expected_series_name, + ), + ) + + @staticmethod + def _comicinfo_from_member_evidence( + evidence: dict[str, Any], + ) -> ComicInfoData | None: + raw = evidence.get("comicinfo") + if not isinstance(raw, dict): + return None + + def optional_text(name: str) -> str | None: + value = raw.get(name) + return value if isinstance(value, str) else None + + def optional_int(name: str) -> int | None: + value = raw.get(name) + return value if isinstance(value, int) and not isinstance(value, bool) else None + + return ComicInfoData( + series=optional_text("series"), + number=optional_text("number"), + volume=optional_text("volume"), + title=optional_text("title"), + year=optional_int("year"), + month=optional_int("month"), + day=optional_int("day"), + publisher=optional_text("publisher"), + notes=optional_text("notes"), + summary=optional_text("summary"), + writer=optional_text("writer"), + penciller=optional_text("penciller"), + inker=optional_text("inker"), + colorist=optional_text("colorist"), + letterer=optional_text("letterer"), + cover_artist=optional_text("cover_artist"), + editor=optional_text("editor"), + page_count=optional_int("page_count"), + genre=optional_text("genre"), + web=optional_text("web"), + story_arc=optional_text("story_arc"), + story_arc_number=optional_text("story_arc_number"), + series_group=optional_text("series_group"), + language=optional_text("language"), + ) + + @staticmethod + def _archive_hint_from_member_evidence( + evidence: dict[str, Any], + ) -> ArchiveEntryIssueHint | None: + raw = evidence.get("archive_entry_issue_hint") + if not isinstance(raw, dict) or raw.get("confidence") != "strong": + return None + series_name = raw.get("series_name") + issue_number = normalize_issue_number(raw.get("issue_number")) + if not isinstance(series_name, str) or issue_number is None: + return None + year = raw.get("year") + sample_entries = raw.get("sample_entries") + count_fields = ( + "total_image_entries", + "parseable_image_entries", + "matching_entry_count", + ) + if year is not None and not isinstance(year, int): + return None + if not isinstance(sample_entries, list) or not all( + isinstance(entry, str) for entry in sample_entries + ): + return None + if not all(isinstance(raw.get(field), int) for field in count_fields): + return None + return { + "series_name": series_name, + "issue_number": issue_number, + "year": year, + "confidence": "strong", + "total_image_entries": int(raw["total_image_entries"]), + "parseable_image_entries": int(raw["parseable_image_entries"]), + "matching_entry_count": int(raw["matching_entry_count"]), + "sample_entries": list(sample_entries), + } + @staticmethod def archive_entry_issue_hint_from_path( path: str | Path, @@ -668,11 +848,8 @@ def _read_sidecars(self, folder: Path) -> dict[str, Any]: payload["files_present"].append(sidecar_name) raw_text = path.read_text(errors="replace") sidecar_data = self._parse_sidecar(raw_text) - sidecar_series_id = _as_int( - sidecar_data.get("comicid") - or sidecar_data.get("comicvine_id") - or sidecar_data.get("series_id") - ) + payload["identity_conflicts"].extend(sidecar_data.get("_identity_conflicts", [])) + sidecar_series_id = _as_int(sidecar_data.get("comicid")) if ( sidecar_series_id is not None and payload["series_id"] is not None @@ -713,29 +890,7 @@ def _read_sidecars(self, folder: Path) -> dict[str, Any]: @staticmethod def _parse_sidecar(raw_text: str) -> dict[str, Any]: - raw_text = raw_text.strip() - if not raw_text: - return {} - try: - parsed = json.loads(raw_text) - except json.JSONDecodeError: - parsed = None - if isinstance(parsed, dict): - return {str(key).lower(): value for key, value in parsed.items()} - data: dict[str, Any] = {} - for line in raw_text.splitlines(): - if ":" in line: - key, value = line.split(":", 1) - elif "=" in line: - key, value = line.split("=", 1) - else: - continue - data[key.strip().lower()] = value.strip() - if not data: - match = _CV_ANY_ID_RE.search(raw_text) - if match: - data["comicid"] = match.group(1) - return data + return parse_source_sidecar(raw_text) def _extract_issue_id_from_web(web: str | None) -> int | None: @@ -920,6 +1075,7 @@ def _serialize_parsed_release(parsed: ParsedRelease | None) -> dict[str, object] return { "series_name": parsed.series_name if parsed is not None else None, "issue_number": parsed.issue_number if parsed is not None else None, + "issue_number_text": parsed.issue_number_text if parsed is not None else None, "year": parsed.year if parsed is not None else None, "volume": parsed.volume if parsed is not None else None, "issue_type": parsed.issue_type.value if parsed is not None else None, diff --git a/src/pullbox/core/source_sidecars.py b/src/pullbox/core/source_sidecars.py new file mode 100644 index 00000000..42701802 --- /dev/null +++ b/src/pullbox/core/source_sidecars.py @@ -0,0 +1,104 @@ +"""Read explicit ComicVine identities from legacy and Mylar sidecars.""" + +from __future__ import annotations + +import json +import re +from typing import Any +from urllib.parse import urlsplit + +_CV_HOSTS = frozenset( + {"comicvine.gamespot.com", "www.comicvine.gamespot.com", "comicvine.com", "www.comicvine.com"} +) +_SERIES_KEYS = ("comicid", "comicvine_id", "comicvineid", "cv_vol_id", "cvid") +_URL_KEYS = ("url", "web", "comicvine_url") + + +def _volume_url_id(value: str) -> int | None: + try: + url = urlsplit(value.strip()) + if url.scheme not in {"http", "https"} or url.hostname not in _CV_HOSTS: + return None + except ValueError: + return None + match = re.search(r"(?:^|/)4050-([0-9]{1,15})(?:/|$)", url.path) + return int(match[1]) if match and int(match[1]) > 0 else None + + +def _volume_id(value: object) -> int | None: + if isinstance(value, bool): + return None + match = re.fullmatch(r"(?:4050-)?([0-9]{1,15})", str(value).strip()) + return int(match[1]) if match and int(match[1]) > 0 else None + + +def parse_source_sidecar(raw_text: str) -> dict[str, Any]: + """Keep local IDs out of the ComicVine namespace and retain true conflicts.""" + raw_text = raw_text.strip() + if not raw_text: + return {} + try: + parsed = json.loads(raw_text) + except json.JSONDecodeError: + parsed = None + candidates: list[tuple[str, int]] = [] + if isinstance(parsed, dict): + data = {str(key).lower(): value for key, value in parsed.items()} + nested = data.pop("metadata", None) + layers = [("root", data)] + if isinstance(nested, dict): + metadata = {str(key).lower(): value for key, value in nested.items()} + layers.append(("metadata", metadata)) + data = {**data, **metadata} + for scope, layer in layers: + for key in _SERIES_KEYS: + cv_id = _volume_id(layer.get(key)) + if cv_id is not None: + candidates.append((f"{scope}.{key}", cv_id)) + for key in _URL_KEYS: + cv_id = _volume_url_id(str(layer.get(key) or "")) + if cv_id is not None: + candidates.append((f"{scope}.{key}", cv_id)) + else: + data = {} + for line in raw_text.splitlines(): + cv_id = _volume_url_id(line) + if cv_id is not None: + candidates.append(("comicvine_volume_url", cv_id)) + continue + if ":" in line: + key, value = line.split(":", 1) + elif "=" in line: + key, value = line.split("=", 1) + else: + continue + key = key.strip().lower() + data[key] = value.strip() + cv_id = ( + _volume_id(value) + if key in _SERIES_KEYS + else _volume_url_id(value) + if key in _URL_KEYS + else None + ) + if cv_id is not None: + candidates.append((key, cv_id)) + # An unqualified series_id may belong to another application, not ComicVine. + data.pop("series_id", None) + data.pop("comicid", None) + data["_identity_conflicts"] = [] + if candidates: + first_source, first_id = candidates[0] + data["comicid"] = first_id + for source, cv_id in candidates[1:]: + if cv_id != first_id: + data["_identity_conflicts"].append( + { + "field": "comicvine_series_id", + "first": first_id, + "conflicting": cv_id, + "first_source": first_source, + "conflicting_source": source, + } + ) + return data diff --git a/src/pullbox/core/sqlite_lock.py b/src/pullbox/core/sqlite_lock.py index 23a94781..9b72eebe 100644 --- a/src/pullbox/core/sqlite_lock.py +++ b/src/pullbox/core/sqlite_lock.py @@ -2,6 +2,14 @@ from __future__ import annotations +import asyncio +from typing import TYPE_CHECKING, Any + +from sqlalchemy.exc import OperationalError + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + SQLITE_LOCK_RETRY_ATTEMPTS = 8 SQLITE_LOCK_RETRY_BASE_DELAY_SECONDS = 0.25 @@ -15,3 +23,34 @@ def is_sqlite_locked_error(exc: BaseException) -> bool: def sqlite_lock_retry_delay(attempt: int) -> float: """Return the linear backoff delay for a SQLite lock retry attempt.""" return SQLITE_LOCK_RETRY_BASE_DELAY_SECONDS * attempt + + +async def run_sqlite_transaction_with_retry[T]( + session: Any, + operation: Callable[[], Awaitable[T]], + *, + event_name: str, + logger: Any, + retry_delay: Callable[[int], float] = sqlite_lock_retry_delay, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + attempts: int = SQLITE_LOCK_RETRY_ATTEMPTS, +) -> T: + """Commit a short transaction, retrying only transient SQLite lock failures.""" + for attempt in range(1, attempts + 1): + try: + result = await operation() + await session.commit() + return result + except OperationalError as exc: + await session.rollback() + if not is_sqlite_locked_error(exc) or attempt == attempts: + raise + delay_seconds = retry_delay(attempt) + logger.warning( + f"{event_name}_retrying_after_sqlite_lock", + attempt=attempt, + max_attempts=attempts, + delay_seconds=delay_seconds, + ) + await sleep(delay_seconds) + raise RuntimeError("SQLite transaction retry loop ended unexpectedly") diff --git a/src/pullbox/core/story_arc_identity.py b/src/pullbox/core/story_arc_identity.py new file mode 100644 index 00000000..ddb0398b --- /dev/null +++ b/src/pullbox/core/story_arc_identity.py @@ -0,0 +1,23 @@ +"""Conservative story-arc identity normalization.""" + +from __future__ import annotations + +import re +import unicodedata + +_WHITESPACE = re.compile(r"\s+") + + +def normalize_story_arc_name(name: str) -> str: + """Return a stable duplicate-detection key without fuzzy title rewriting. + + Story-arc names are user-visible collection identities. Canonical Unicode + normalization, case folding, and whitespace collapse are safe identity + operations; punctuation and articles remain significant. + """ + normalized = unicodedata.normalize("NFC", name) + normalized = _WHITESPACE.sub(" ", normalized).strip().casefold() + if not normalized: + msg = "Story-arc name must not be blank" + raise ValueError(msg) + return normalized diff --git a/src/pullbox/core/story_arc_naming.py b/src/pullbox/core/story_arc_naming.py new file mode 100644 index 00000000..f7ff604d --- /dev/null +++ b/src/pullbox/core/story_arc_naming.py @@ -0,0 +1,227 @@ +"""Safe, data-only naming for optional story-arc placements.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path, PureWindowsPath + +from pullbox.core.naming import sanitize_for_filesystem + +DEFAULT_STORY_ARC_FOLDER_TEMPLATE = "{StoryArc}" +DEFAULT_STORY_ARC_FILE_TEMPLATE = "{ReadingOrder:03d} - {Series} {IssueNumber}{IssueTitleOptional}" +# Explicit opt-in templates leave existing saved/default metadata policies unchanged. +ORIGINAL_STORY_ARC_FILE_TEMPLATE = "{OriginalFilename}" +ORDERED_ORIGINAL_STORY_ARC_FILE_TEMPLATE = "{ReadingOrder:02d} - {OriginalFilename}" + +_MAX_TEMPLATE_BYTES = 1024 +_TOKEN_RE = re.compile(r"\{(?P[A-Za-z][A-Za-z0-9]*)(?::(?P[^{}]+))?\}") +_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f]") +_FOLDER_TOKENS = frozenset({"StoryArc", "Publisher", "StartYear", "EndYear", "SpanYears"}) +_FILE_TOKENS = frozenset( + { + "ReadingOrder", + "Series", + "IssueNumber", + "IssueTitle", + "IssueTitleOptional", + "Year", + "Extension", + "OriginalFilename", + } +) +_READING_ORDER_FORMAT_RE = re.compile(r"0?[2-6]d") +_WINDOWS_DEVICE_NAME_RE = re.compile( + r"(?:CON|PRN|AUX|NUL|CONIN\$|CONOUT\$|(?:COM|LPT)[1-9¹²³])", re.IGNORECASE +) + + +@dataclass(frozen=True, slots=True) +class StoryArcNamingValues: + """Bounded values used to preview or render one arc placement path.""" + + story_arc: str + reading_order: int + series: str + issue_number: str + extension: str + issue_title: str | None = None + year: int | None = None + start_year: int | None = None + end_year: int | None = None + publisher: str | None = None + original_filename: str | None = None + + +class StoryArcOriginalFilenameError(ValueError): + """A canonical basename cannot be preserved safely at the destination.""" + + +def validate_story_arc_folder_template(template: str) -> None: + """Reject path-producing or executable story-arc folder templates.""" + _validate_template(template, allowed_tokens=_FOLDER_TOKENS, kind="folder") + + +def validate_story_arc_file_template(template: str) -> None: + """Reject path-producing or unbounded story-arc file templates.""" + tokens = _validate_template(template, allowed_tokens=_FILE_TOKENS, kind="file") + names = [name for name, _format_spec in tokens] + if "OriginalFilename" in names and ( + names.count("OriginalFilename") != 1 + or "Extension" in names + or not template.endswith("{OriginalFilename}") + ): + msg = "OriginalFilename must appear once at the end, without an Extension token" + raise ValueError(msg) + for name, format_spec in tokens: + if name == "ReadingOrder": + if format_spec is not None and not _READING_ORDER_FORMAT_RE.fullmatch(format_spec): + msg = "ReadingOrder format must be a bounded width from 2d through 6d" + raise ValueError(msg) + elif format_spec is not None: + msg = f"Formatting is not supported for story-arc token: {name}" + raise ValueError(msg) + + +def render_story_arc_relative_path( + values: StoryArcNamingValues, + *, + folder_template: str = DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + file_template: str = DEFAULT_STORY_ARC_FILE_TEMPLATE, + replace_illegal: bool = True, + colon_replacement: str = "dash", +) -> Path: + """Render a sanitized relative folder/file path without touching disk.""" + validate_story_arc_folder_template(folder_template) + validate_story_arc_file_template(file_template) + if values.reading_order < 0: + msg = "Story-arc reading order must not be negative" + raise ValueError(msg) + + issue_number = values.issue_number.strip() + if not issue_number or _CONTROL_RE.search(issue_number): + msg = "Story-arc issue number must be a non-empty exact value" + raise ValueError(msg) + + extension = values.extension.strip().lstrip(".").lower() + if not re.fullmatch(r"[a-z0-9]{1,10}", extension): + msg = "Story-arc source extension is invalid" + raise ValueError(msg) + + def safe(value: str) -> str: + return sanitize_for_filesystem( + value, + replace_illegal=replace_illegal, + colon_replacement=colon_replacement, + ) + + start_year = str(values.start_year) if values.start_year is not None else "" + end_year = str(values.end_year) if values.end_year is not None else "" + if start_year and end_year and start_year != end_year: + span_years = f"{start_year} - {end_year}" + else: + span_years = start_year or end_year + + folder_values = { + "StoryArc": safe(values.story_arc), + "Publisher": safe(values.publisher) if values.publisher else "", + "StartYear": start_year, + "EndYear": end_year, + "SpanYears": span_years, + } + file_values = { + "Series": safe(values.series), + "IssueNumber": safe(issue_number), + "IssueTitle": safe(values.issue_title) if values.issue_title else "", + "IssueTitleOptional": f" - {safe(values.issue_title)}" if values.issue_title else "", + "Year": str(values.year) if values.year is not None else "", + "Extension": extension, + } + + folder = _render_template(folder_template, folder_values, values.reading_order) + safe_folder = safe(folder) + if "{OriginalFilename}" in file_template: + original = _validated_original_filename(values.original_filename, extension) + prefix = _render_template( + file_template.removesuffix("{OriginalFilename}"), file_values, values.reading_order + ) + # Sanitize template-produced prefix text, not the preserved basename. + # The sentinel retains an intentional trailing space in e.g. "01 - ". + safe_prefix = safe(prefix + "x")[:-1] + safe_file = _validated_original_filename(safe_prefix + original, extension) + else: + rendered_file = _render_template(file_template, file_values, values.reading_order) + if "{Extension}" not in file_template: + rendered_file = f"{rendered_file}.{extension}" + safe_file = safe(rendered_file) + if safe_folder in {"", ".", ".."} or safe_file in {"", ".", ".."}: + msg = "Story-arc template rendered an unsafe path" + raise ValueError(msg) + return Path(safe_folder, safe_file) + + +def _validated_original_filename(filename: str | None, extension: str) -> str: + """Preserve safe spelling, spacing and extension case; never silently rename.""" + if not filename: + raise StoryArcOriginalFilenameError("Original filename requires a canonical file") + try: + encoded = filename.encode("utf-8") + except UnicodeEncodeError as exc: + raise StoryArcOriginalFilenameError("Original filename is not valid UTF-8") from exc + normalized_spaces = " ".join(filename.split()) + if ( + len(encoded) > 240 + or PureWindowsPath(filename).name != filename + or re.search(r"[\x00-\x1f\x7f-\x9f]", filename) + or filename.endswith((".", " ")) + or _WINDOWS_DEVICE_NAME_RE.fullmatch(filename.split(".", 1)[0].rstrip(" ")) + or sanitize_for_filesystem(normalized_spaces) != normalized_spaces + or Path(filename).suffix.casefold() != f".{extension}" + ): + raise StoryArcOriginalFilenameError( + "Original filename cannot be preserved safely with the canonical extension" + ) + return filename + + +def _validate_template( + template: str, + *, + allowed_tokens: frozenset[str], + kind: str, +) -> tuple[tuple[str, str | None], ...]: + if not template or len(template.encode("utf-8")) > _MAX_TEMPLATE_BYTES: + msg = f"Story-arc {kind} template is empty or too long" + raise ValueError(msg) + if "/" in template or "\\" in template or _CONTROL_RE.search(template): + msg = f"Story-arc {kind} template must produce one safe path segment" + raise ValueError(msg) + + tokens: list[tuple[str, str | None]] = [] + for match in _TOKEN_RE.finditer(template): + name = match.group("name") + if name not in allowed_tokens: + msg = f"Unsupported story-arc {kind} token: {name}" + raise ValueError(msg) + tokens.append((name, match.group("format"))) + + without_tokens = _TOKEN_RE.sub("", template) + if "{" in without_tokens or "}" in without_tokens: + msg = f"Story-arc {kind} template contains an invalid token" + raise ValueError(msg) + return tuple(tokens) + + +def _render_template( + template: str, + values: dict[str, str], + reading_order: int, +) -> str: + def replace(match: re.Match[str]) -> str: + name = match.group("name") + if name == "ReadingOrder": + format_spec = match.group("format") + return format(reading_order, format_spec) if format_spec else str(reading_order) + return values[name] + + return _TOKEN_RE.sub(replace, template) diff --git a/src/pullbox/core/story_arc_ordering.py b/src/pullbox/core/story_arc_ordering.py new file mode 100644 index 00000000..a5b07a17 --- /dev/null +++ b/src/pullbox/core/story_arc_ordering.py @@ -0,0 +1,49 @@ +"""Conservative filename evidence for story-arc reading order.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +_ORDER_PREFIX = re.compile(r"^(?P\d{1,9})\s*-\s*(?P\S(?:.*\S)?)$") + + +@dataclass(frozen=True, slots=True) +class StoryArcOrderPrefix: + """One anchored order prefix without any inferred series identity.""" + + reading_order: int + reading_order_raw: str + residual_file_name: str + + @property + def sort_key(self) -> tuple[int, str, str, str]: + """Return stable numeric-first ordering for review evidence.""" + return ( + self.reading_order, + self.reading_order_raw, + self.residual_file_name.casefold(), + self.residual_file_name, + ) + + +def extract_story_arc_order_prefix(file_name: str) -> StoryArcOrderPrefix | None: + """Extract a leading ``NNN-`` order as weak evidence only. + + This deliberately does not parse the residual text as a series or issue. + Callers must keep an unconfirmed prefix reviewable rather than rewriting + canonical identity from it. + """ + if not file_name or len(file_name) > 500 or "/" in file_name or "\\" in file_name: + return None + match = _ORDER_PREFIX.fullmatch(file_name.strip()) + if match is None: + return None + reading_order = int(match.group("order")) + if reading_order <= 0: + return None + return StoryArcOrderPrefix( + reading_order=reading_order, + reading_order_raw=match.group("order"), + residual_file_name=match.group("residual"), + ) diff --git a/src/pullbox/core/torrent_metadata.py b/src/pullbox/core/torrent_metadata.py new file mode 100644 index 00000000..475f6cdd --- /dev/null +++ b/src/pullbox/core/torrent_metadata.py @@ -0,0 +1,89 @@ +"""Bounded bencode validation and exact torrent metadata hashes.""" + +from __future__ import annotations + +import hashlib + + +def torrent_info_hashes(content: bytes) -> frozenset[str]: + """Validate one descriptor and hash its original info bytes without re-encoding.""" + info_start, info_end = _top_level_info_span(content) + info = content[info_start:info_end] + return frozenset( + ( + hashlib.sha1(info, usedforsecurity=False).hexdigest(), + hashlib.sha256(info).hexdigest(), + ) + ) + + +def _top_level_info_span(content: bytes) -> tuple[int, int]: + if not content or content[0] != ord("d"): + raise ValueError("torrent descriptor must be a dictionary") + index = 1 + info_span: tuple[int, int] | None = None + while index < len(content) and content[index] != ord("e"): + key, index = _parse_bencoded_bytes(content, index) + value_start = index + index = _skip_bencoded_value(content, index, depth=1) + if key == b"info": + if info_span is not None: + raise ValueError("torrent descriptor has duplicate info dictionaries") + info_span = (value_start, index) + if index >= len(content) or content[index] != ord("e") or index + 1 != len(content): + raise ValueError("torrent descriptor is truncated or has trailing data") + if info_span is None or content[info_span[0]] != ord("d"): + raise ValueError("torrent descriptor has no info dictionary") + return info_span + + +def _skip_bencoded_value(content: bytes, index: int, *, depth: int) -> int: + if depth > 100 or index >= len(content): + raise ValueError("invalid bencode nesting") + marker = content[index] + if 48 <= marker <= 57: + _, end = _parse_bencoded_bytes(content, index) + return end + if marker == ord("i"): + end = content.find(b"e", index + 1) + if end < 0: + raise ValueError("unterminated bencoded integer") + value = content[index + 1 : end] + digits = value[1:] if value.startswith(b"-") else value + if ( + not digits + or not digits.isdigit() + or (len(digits) > 1 and digits.startswith(b"0")) + or value == b"-0" + ): + raise ValueError("invalid bencoded integer") + return end + 1 + if marker not in {ord("l"), ord("d")}: + raise ValueError("invalid bencoded value") + cursor = index + 1 + while cursor < len(content) and content[cursor] != ord("e"): + if marker == ord("d"): + _, cursor = _parse_bencoded_bytes(content, cursor) + cursor = _skip_bencoded_value(content, cursor, depth=depth + 1) + if cursor >= len(content): + raise ValueError("unterminated bencoded collection") + return cursor + 1 + + +def _parse_bencoded_bytes(content: bytes, index: int) -> tuple[bytes, int]: + colon = content.find(b":", index) + if colon < 0: + raise ValueError("invalid bencoded byte string") + raw_length = content[index:colon] + if ( + not raw_length + or not raw_length.isdigit() + or (len(raw_length) > 1 and raw_length.startswith(b"0")) + ): + raise ValueError("invalid bencoded byte-string length") + length = int(raw_length) + start = colon + 1 + end = start + length + if end > len(content): + raise ValueError("truncated bencoded byte string") + return content[start:end], end diff --git a/src/pullbox/docker_entrypoint.py b/src/pullbox/docker_entrypoint.py index 1b532d2a..834fa7e0 100644 --- a/src/pullbox/docker_entrypoint.py +++ b/src/pullbox/docker_entrypoint.py @@ -6,6 +6,7 @@ import signal import subprocess import sys +import threading from pathlib import Path from typing import Any, TextIO, cast @@ -22,6 +23,11 @@ DEFAULT_COMMAND = ("python", "-m", "pullbox") +# Uvicorn gets five seconds to drain active requests. Keep the entrypoint's +# supervisor deadline below Docker's default ten-second stop grace period while +# leaving enough time for Uvicorn to finish its lifespan shutdown. +CHILD_SHUTDOWN_TIMEOUT_SECONDS = 8.0 + class _TeeStream: """Mirror writes to the original stream and the rotating startup log.""" @@ -116,10 +122,26 @@ def _run_process(command: list[str]) -> int: return 127 previous_handlers: dict[signal.Signals, Any] = {} + forwarded_signal: signal.Signals | None = None + shutdown_timer: threading.Timer | None = None + + def _force_kill_after_timeout() -> None: + if process.poll() is None: + process.kill() def _forward(signum: int, _frame: object | None) -> None: + nonlocal forwarded_signal, shutdown_timer if process.poll() is None: - process.send_signal(signum) + forwarded = signal.Signals(signum) + process.send_signal(forwarded) + if forwarded_signal is None: + forwarded_signal = forwarded + shutdown_timer = threading.Timer( + CHILD_SHUTDOWN_TIMEOUT_SECONDS, + _force_kill_after_timeout, + ) + shutdown_timer.daemon = True + shutdown_timer.start() for signum in (signal.Signals.SIGINT, signal.Signals.SIGTERM): previous_handlers[signum] = signal.getsignal(signum) @@ -129,11 +151,26 @@ def _forward(signum: int, _frame: object | None) -> None: if process.stdout is not None: for line in process.stdout: print(line, end="", flush=True) - return process.wait() + exit_code = process.wait() finally: + if shutdown_timer is not None: + shutdown_timer.cancel() for signum, handler in previous_handlers.items(): signal.signal(signum, cast("Any", handler)) + # Uvicorn intentionally re-raises the captured shutdown signal after its + # graceful lifecycle completes. A SIGTERM that this entrypoint forwarded is + # therefore an expected container stop, not an application failure. Raising + # here also prevents a stop received during migrations from launching the + # application after the migration child exits. + if forwarded_signal is signal.Signals.SIGTERM and exit_code in { + 0, + -int(signal.Signals.SIGTERM), + }: + raise SystemExit(0) + + return exit_code + def main(argv: list[str] | None = None) -> None: """Run migrations, launch Pullbox, and honor graceful restart requests.""" diff --git a/src/pullbox/models/__init__.py b/src/pullbox/models/__init__.py index 7ba4b054..4f16f6c6 100644 --- a/src/pullbox/models/__init__.py +++ b/src/pullbox/models/__init__.py @@ -42,6 +42,7 @@ ImportedFile, ImportedFileStatus, ImportedSeries, + ImportFileHandlingMode, ImportJob, ImportJobLog, ImportJobStatus, @@ -50,7 +51,14 @@ ) from pullbox.models.indexer import IndexerConfig, IndexerSource, IndexerType from pullbox.models.issue import Issue, IssueStatus, IssueType -from pullbox.models.library import FileFormat, LibraryFile, LibraryRoot, MatchConfidence +from pullbox.models.library import ( + FileFormat, + LibraryFile, + LibraryRoot, + LibraryRootPolicy, + LibraryRootPolicySource, + MatchConfidence, +) from pullbox.models.matching_suggestion import MatchingSuggestion, SuggestionStatus from pullbox.models.operation_progress import ( OperationProgress, @@ -72,7 +80,26 @@ SeriesStatusOverride, SeriesType, ) -from pullbox.models.story_arc import IssueStoryArc, StoryArc +from pullbox.models.story_arc import ( + ImportedStoryArcStatus, + IssueStoryArc, + StoryArc, + StoryArcExternalIdentity, + StoryArcLifecycle, + StoryArcPlacement, + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcPlacementState, + StoryArcResolutionState, + StoryArcSourceKind, + StoryArcSymlinkStyle, +) +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.models.story_arc_sync import ( + StoryArcSyncReason, + StoryArcSyncWork, + StoryArcSyncWorkState, +) from pullbox.models.user import APIKey, User from pullbox.models.whats_new import WhatsNewCacheKind, WhatsNewReleaseCache from pullbox.utilities.models import ( @@ -124,6 +151,7 @@ "HealthIncident", "HealthStatus", "IdentityMixin", + "ImportFileHandlingMode", "ImportJob", "ImportJobLog", "ImportJobStatus", @@ -132,6 +160,9 @@ "ImportedFile", "ImportedFileStatus", "ImportedSeries", + "ImportedStoryArc", + "ImportedStoryArcEntry", + "ImportedStoryArcStatus", "IndexerConfig", "IndexerSource", "IndexerType", @@ -147,6 +178,8 @@ "JobType", "LibraryFile", "LibraryRoot", + "LibraryRootPolicy", + "LibraryRootPolicySource", "LogLevel", "MatchConfidence", "MatchingSuggestion", @@ -167,6 +200,18 @@ "SeriesStatusOverride", "SeriesType", "StoryArc", + "StoryArcExternalIdentity", + "StoryArcLifecycle", + "StoryArcPlacement", + "StoryArcPlacementMode", + "StoryArcPlacementOwnership", + "StoryArcPlacementState", + "StoryArcResolutionState", + "StoryArcSourceKind", + "StoryArcSymlinkStyle", + "StoryArcSyncReason", + "StoryArcSyncWork", + "StoryArcSyncWorkState", "SuggestionStatus", "SystemConfig", "TimestampMixin", diff --git a/src/pullbox/models/audit_log.py b/src/pullbox/models/audit_log.py index 57170095..7410c32b 100644 --- a/src/pullbox/models/audit_log.py +++ b/src/pullbox/models/audit_log.py @@ -25,6 +25,10 @@ class AuditEventType(StrEnum): API_RATE_LIMITED = "api_rate_limited" SECURITY_CONFIG_CHANGED = "security_config_changed" LOCAL_BYPASS_TOGGLED = "local_bypass_toggled" + IMPORT_SAFETY_BULK_OVERRIDE = "import_safety_bulk_override" + IMPORT_SAFETY_SOURCE_TRASH = "import_safety_source_trash" + IMPORT_MISPLACED_SOURCE_CLEANUP = "import_misplaced_source_cleanup" + IMPORT_RECOVERY_BULK_ACTION = "import_recovery_bulk_action" class AuditLog(Base, IdentityMixin): diff --git a/src/pullbox/models/config.py b/src/pullbox/models/config.py index e419deac..3640ccb9 100644 --- a/src/pullbox/models/config.py +++ b/src/pullbox/models/config.py @@ -67,6 +67,21 @@ "rename_on_import": ("true", "bool"), "replace_illegal_characters": ("true", "bool"), "colon_replacement": ("dash", "string"), + # Story Arc Files — defaults captured only when adding a new arc. + "story_arc_files_enabled": ("false", "bool"), + "story_arc_files_method": ("copy", "string"), + "story_arc_files_library_root_id": ("", "string"), + "story_arc_files_destination": ("", "string"), + "story_arc_files_folder_template": ("{StoryArc}", "string"), + "story_arc_files_filename_style": ("original", "string"), + "story_arc_files_prefix_reading_order": ("false", "bool"), + "story_arc_files_reading_order_width": ("2", "int"), + "story_arc_files_file_template": ( + "{ReadingOrder:03d} - {Series} {IssueNumber}{IssueTitleOptional}", + "string", + ), + "story_arc_files_symlink_style": ("relative", "string"), + "story_arc_files_synchronize": ("true", "bool"), # Naming — Folder Management "create_empty_series_folders": ("false", "bool"), "delete_empty_folders": ("true", "bool"), diff --git a/src/pullbox/models/import_job.py b/src/pullbox/models/import_job.py index fcf15699..3a5a4a5d 100644 --- a/src/pullbox/models/import_job.py +++ b/src/pullbox/models/import_job.py @@ -30,6 +30,7 @@ if TYPE_CHECKING: from pullbox.models.library import LibraryRoot from pullbox.models.series import Series + from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry class ImportSourceType(enum.StrEnum): @@ -39,6 +40,13 @@ class ImportSourceType(enum.StrEnum): MYLAR3 = "mylar3" +class ImportFileHandlingMode(enum.StrEnum): + """How files selected by an import become Pullbox library files.""" + + MANAGED_COPY = "managed_copy" + IN_PLACE = "in_place" + + class ImportJobStatus(enum.StrEnum): """Lifecycle status of an import job.""" @@ -122,6 +130,21 @@ class ImportJob(Base, IdentityMixin, TimestampMixin): """ __tablename__ = "import_jobs" + __table_args__ = ( + Index( + "ix_import_jobs_story_arc_followup", + "status", + "story_arc_placement_followup_pending", + "id", + ), + Index( + "ix_import_jobs_story_arc_rollback_waiting", + "status", + "story_arc_rollback_waiting_work_id", + "id", + ), + Index("ix_import_jobs_archived_created", "archived_at", "created_at"), + ) # Source source_path: Mapped[str] = mapped_column(String(1000), nullable=False) @@ -167,6 +190,7 @@ class ImportJob(Base, IdentityMixin, TimestampMixin): match_completed_at: Mapped[datetime | None] = mapped_column(UTCDateTime) import_started_at: Mapped[datetime | None] = mapped_column(UTCDateTime) import_completed_at: Mapped[datetime | None] = mapped_column(UTCDateTime) + archived_at: Mapped[datetime | None] = mapped_column(UTCDateTime, index=False) # Error error_message: Mapped[str | None] = mapped_column(Text) @@ -180,11 +204,20 @@ class ImportJob(Base, IdentityMixin, TimestampMixin): server_default=ImportControlRequest.NONE.value, nullable=False, ) + story_arc_placement_followup_pending: Mapped[bool] = mapped_column( + default=False, + server_default="0", + nullable=False, + ) + story_arc_rollback_waiting_work_id: Mapped[int | None] = mapped_column( + ForeignKey("story_arc_sync_work.id", ondelete="SET NULL") + ) # Import settings (captured from wizard) target_library_root_id: Mapped[int | None] = mapped_column( - ForeignKey("library_roots.id", ondelete="SET NULL") + ForeignKey("library_roots.id", ondelete="RESTRICT") ) + removed_library_root_snapshot: Mapped[dict | None] = mapped_column(JSON(none_as_null=True)) # type: ignore[type-arg] monitored: Mapped[bool] = mapped_column(default=False) search_on_add: Mapped[bool] = mapped_column(default=False) move_to_library: Mapped[bool] = mapped_column(default=True) @@ -204,6 +237,52 @@ class ImportJob(Base, IdentityMixin, TimestampMixin): ingest_policy_snapshot: Mapped[dict] = mapped_column( # type: ignore[type-arg] JSON, default=dict, server_default="{}" ) + file_handling_mode: Mapped[ImportFileHandlingMode] = mapped_column( + SQLAlchemyEnum( + ImportFileHandlingMode, + values_callable=_enum_values, + native_enum=False, + create_constraint=True, + ), + default=ImportFileHandlingMode.MANAGED_COPY, + server_default=ImportFileHandlingMode.MANAGED_COPY.value, + nullable=False, + ) + source_layout_snapshot: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, + default=lambda: { + "schema_version": 1, + "mode": "auto", + "preset": None, + "series_path_template": None, + "issue_filename_template": None, + "selected_cluster_id": None, + "fallback_to_auto": True, + }, + server_default=( + '{"schema_version":1,"mode":"auto","preset":null,' + '"series_path_template":null,"issue_filename_template":null,' + '"selected_cluster_id":null,"fallback_to_auto":true}' + ), + nullable=False, + ) + future_layout_requested: Mapped[bool] = mapped_column( + default=False, + server_default="0", + nullable=False, + ) + future_root_policy_snapshot: Mapped[dict | None] = mapped_column(JSON, nullable=True) # type: ignore[type-arg] + future_root_policy_applied_at: Mapped[datetime | None] = mapped_column(UTCDateTime) + story_arc_import_requested: Mapped[bool] = mapped_column( + default=False, + server_default="0", + nullable=False, + ) + story_arc_materialization_requested: Mapped[bool] = mapped_column( + default=False, + server_default="0", + nullable=False, + ) # Per-job configuration cv_match_threshold: Mapped[float] = mapped_column(Float, default=0.70) @@ -216,25 +295,42 @@ class ImportJob(Base, IdentityMixin, TimestampMixin): mylar3_path_map: Mapped[dict] = mapped_column( # type: ignore[type-arg] JSON, default=dict, server_default="{}" ) + mylar3_path_map_confirmed: Mapped[bool] = mapped_column( + default=False, + server_default="0", + nullable=False, + ) # Relationships series: Mapped[list[ImportedSeries]] = relationship( - back_populates="import_job", cascade="all, delete-orphan" + back_populates="import_job", + cascade="all, delete-orphan", + passive_deletes=True, ) files: Mapped[list[ImportedFile]] = relationship( - back_populates="import_job", cascade="all, delete-orphan" + back_populates="import_job", + cascade="all, delete-orphan", + passive_deletes=True, ) target_library_root: Mapped[LibraryRoot | None] = relationship() logs: Mapped[list[ImportJobLog]] = relationship( back_populates="import_job", cascade="all, delete-orphan", + passive_deletes=True, order_by="ImportJobLog.logged_at", ) actions: Mapped[list[ImportJobAction]] = relationship( back_populates="import_job", cascade="all, delete-orphan", + passive_deletes=True, order_by="ImportJobAction.sequence_no", ) + story_arcs: Mapped[list[ImportedStoryArc]] = relationship( + back_populates="import_job", + cascade="all, delete-orphan", + passive_deletes=True, + order_by="ImportedStoryArc.id", + ) class ImportJobLog(Base, IdentityMixin): @@ -272,6 +368,7 @@ class ImportJobAction(Base, IdentityMixin, TimestampMixin): __table_args__ = ( Index("ix_import_job_actions_job_seq", "import_job_id", "sequence_no"), Index("ix_import_job_actions_job_status", "import_job_id", "status"), + Index("ix_import_job_actions_job_id_keyset", "import_job_id", "id"), ) import_job_id: Mapped[int] = mapped_column( @@ -362,7 +459,9 @@ class ImportedSeries(Base, IdentityMixin, TimestampMixin): import_job: Mapped[ImportJob] = relationship(back_populates="series") series: Mapped[Series | None] = relationship() files: Mapped[list[ImportedFile]] = relationship( - back_populates="import_series", cascade="all, delete-orphan" + back_populates="import_series", + cascade="all, delete-orphan", + passive_deletes=True, ) @@ -374,7 +473,19 @@ class ImportedFile(Base, IdentityMixin, TimestampMixin): """ __tablename__ = "import_files" - __table_args__ = (Index("ix_import_files_job_series", "import_job_id", "import_series_id"),) + __table_args__ = ( + Index("ix_import_files_job_series", "import_job_id", "import_series_id"), + Index("ix_import_files_import_series_id", "import_series_id"), + Index("ix_import_files_matched_issue_id", "matched_issue_id"), + Index("ix_import_files_duplicate_of_file_id", "duplicate_of_file_id"), + Index( + "ix_import_files_job_cohort_order", + "import_job_id", + "source_folder_cohort_key", + "source_ordinal", + "id", + ), + ) # Parent references import_job_id: Mapped[int] = mapped_column( @@ -424,6 +535,11 @@ class ImportedFile(Base, IdentityMixin, TimestampMixin): is_preferred: Mapped[bool] = mapped_column(default=False) include_in_import: Mapped[bool] = mapped_column(default=False, server_default="0") content_hash: Mapped[str | None] = mapped_column(String(64)) + source_signature: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + source_folder_cohort_key: Mapped[str | None] = mapped_column(String(1000)) + source_ordinal: Mapped[int | None] = mapped_column(Integer) # Import outcome library_file_id: Mapped[int | None] = mapped_column( @@ -437,3 +553,7 @@ class ImportedFile(Base, IdentityMixin, TimestampMixin): # Relationships import_job: Mapped[ImportJob] = relationship(back_populates="files") import_series: Mapped[ImportedSeries] = relationship(back_populates="files") + story_arc_entries: Mapped[list[ImportedStoryArcEntry]] = relationship( + back_populates="import_file", + foreign_keys="ImportedStoryArcEntry.import_file_id", + ) diff --git a/src/pullbox/models/issue.py b/src/pullbox/models/issue.py index 25b924f7..536c3c45 100644 --- a/src/pullbox/models/issue.py +++ b/src/pullbox/models/issue.py @@ -14,20 +14,24 @@ Integer, String, Text, - UniqueConstraint, ) from sqlalchemy import ( Enum as SQLAlchemyEnum, ) -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates +from pullbox.core.issue_numbers import ( + format_issue_number, + issue_number_text_matches_numeric, + normalize_issue_number_text, +) from pullbox.models.base import Base, IdentityMixin, TimestampMixin if TYPE_CHECKING: from pullbox.models.creator import Creator from pullbox.models.library import LibraryFile from pullbox.models.series import Series - from pullbox.models.story_arc import StoryArc + from pullbox.models.story_arc import IssueStoryArc, StoryArc class IssueType(enum.StrEnum): @@ -106,7 +110,19 @@ class IssueStatus(enum.StrEnum): class Issue(Base, IdentityMixin, TimestampMixin): __tablename__ = "issues" __table_args__ = ( - UniqueConstraint("series_id", "issue_number", name="uq_series_issue"), + Index( + "uq_series_issue_number_text", + "series_id", + "issue_number_text", + unique=True, + ), + Index( + "ix_issues_series_number_order", + "series_id", + "issue_number", + "issue_number_text", + "id", + ), Index("ix_issues_status", "status"), Index("ix_issues_release_date", "release_date"), ) @@ -116,6 +132,7 @@ class Issue(Base, IdentityMixin, TimestampMixin): ForeignKey("series.id", ondelete="CASCADE"), nullable=False ) issue_number: Mapped[float] = mapped_column(Float, nullable=False) + issue_number_text: Mapped[str | None] = mapped_column(String(320), nullable=True) title: Mapped[str | None] = mapped_column(String(500)) description: Mapped[str | None] = mapped_column(Text) release_date: Mapped[date | None] = mapped_column(Date) @@ -151,5 +168,44 @@ class Issue(Base, IdentityMixin, TimestampMixin): secondary="issue_creators", back_populates="issues" ) story_arcs: Mapped[list[StoryArc]] = relationship( - secondary="issue_story_arcs", back_populates="issues" + secondary="issue_story_arcs", back_populates="issues", viewonly=True ) + story_arc_memberships: Mapped[list[IssueStoryArc]] = relationship(back_populates="issue") + + @validates("issue_number") + def _dual_write_issue_number(self, _key: str, value: float) -> float: + """Keep numeric-only callers compatible with exact-text storage.""" + previous_number = self.__dict__.get("issue_number") + existing_text = self.__dict__.get("issue_number_text") + if existing_text is not None and ( + previous_number is None or float(previous_number) == float(value) + ): + if not issue_number_text_matches_numeric(value, str(existing_text)): + raise ValueError("issue number text must match the numeric issue number") + return value + + self.__dict__["_dual_writing_issue_number_text"] = True + try: + self.issue_number_text = format_issue_number(value) + finally: + self.__dict__.pop("_dual_writing_issue_number_text", None) + return value + + @validates("issue_number_text") + def _normalize_issue_number_text(self, _key: str, value: str | None) -> str | None: + if value is None: + return None + normalized = normalize_issue_number_text(value) + current_number = self.__dict__.get("issue_number") + if ( + current_number is not None + and not self.__dict__.get("_dual_writing_issue_number_text", False) + and not issue_number_text_matches_numeric(float(current_number), normalized) + ): + raise ValueError("issue number text must match the numeric issue number") + return normalized + + @property + def effective_issue_number_text(self) -> str: + """Return exact text, deriving it for rows written by an older image.""" + return self.issue_number_text or format_issue_number(self.issue_number) diff --git a/src/pullbox/models/library.py b/src/pullbox/models/library.py index 588bc55a..23b181e6 100644 --- a/src/pullbox/models/library.py +++ b/src/pullbox/models/library.py @@ -9,11 +9,13 @@ from sqlalchemy import ( JSON, BigInteger, + Boolean, Float, ForeignKey, Index, Integer, String, + text, ) from sqlalchemy import ( Enum as SQLAlchemyEnum, @@ -44,6 +46,21 @@ class MatchConfidence(enum.StrEnum): MANUAL = "manual" +class LibraryFileStorageMode(enum.StrEnum): + """Whether Pullbox owns an artifact or references a user-owned file.""" + + MANAGED = "managed" + REFERENCED = "referenced" + + +class LibraryRootPolicySource(enum.StrEnum): + """Origin of an explicit complete naming policy for one library root.""" + + GLOBAL_DEFAULT = "global_default" + IMPORT_ADOPTION = "import_adoption" + MANUAL = "manual" + + class LibraryFile(Base, IdentityMixin, TimestampMixin): __tablename__ = "library_files" __table_args__ = ( @@ -76,10 +93,26 @@ class LibraryFile(Base, IdentityMixin, TimestampMixin): naming_snapshot: Mapped[dict] = mapped_column( # type: ignore[type-arg] JSON, default=dict, server_default="{}", nullable=False ) + storage_mode: Mapped[LibraryFileStorageMode] = mapped_column( + SQLAlchemyEnum( + LibraryFileStorageMode, + values_callable=lambda enum_cls: [member.value for member in enum_cls], + native_enum=False, + create_constraint=True, + ), + default=LibraryFileStorageMode.MANAGED, + server_default=LibraryFileStorageMode.MANAGED.value, + nullable=False, + ) + source_signature: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) # Foreign keys issue_id: Mapped[int | None] = mapped_column(ForeignKey("issues.id", ondelete="SET NULL")) - library_root_id: Mapped[int] = mapped_column(ForeignKey("library_roots.id", ondelete="CASCADE")) + library_root_id: Mapped[int] = mapped_column( + ForeignKey("library_roots.id", ondelete="RESTRICT") + ) # Relationships issue: Mapped[Issue | None] = relationship(back_populates="library_file") @@ -88,16 +121,93 @@ class LibraryFile(Base, IdentityMixin, TimestampMixin): class LibraryRoot(Base, IdentityMixin, TimestampMixin): __tablename__ = "library_roots" + __table_args__ = ( + Index( + "uq_library_roots_default_managed_destination", + "is_default_managed_destination", + unique=True, + sqlite_where=text("is_default_managed_destination = 1"), + postgresql_where=text("is_default_managed_destination"), + ), + ) name: Mapped[str] = mapped_column(String(255), nullable=False) path: Mapped[str] = mapped_column(String(1000), nullable=False, unique=True) - enabled: Mapped[bool] = mapped_column(default=True) + enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + allow_referenced_registrations: Mapped[bool] = mapped_column( + Boolean, + default=True, + server_default="1", + nullable=False, + ) + allow_managed_writes: Mapped[bool] = mapped_column( + Boolean, + default=True, + server_default="1", + nullable=False, + ) + is_default_managed_destination: Mapped[bool] = mapped_column( + Boolean, + default=False, + server_default="0", + nullable=False, + ) last_scan_at: Mapped[datetime | None] = mapped_column(UTCDateTime) last_scan_duration_seconds: Mapped[float | None] = mapped_column(Float) last_scan_files_found: Mapped[int | None] = mapped_column(Integer) # Relationships files: Mapped[list[LibraryFile]] = relationship( - back_populates="library_root", cascade="all, delete-orphan" + back_populates="library_root", passive_deletes="all" + ) + series: Mapped[list[Series]] = relationship( + back_populates="library_root", + foreign_keys="Series.library_root_id", + passive_deletes="all", + ) + preferred_series: Mapped[list[Series]] = relationship( + back_populates="preferred_library_root", + foreign_keys="Series.preferred_library_root_id", + passive_deletes="all", ) - series: Mapped[list[Series]] = relationship(back_populates="library_root") + naming_policy: Mapped[LibraryRootPolicy | None] = relationship( + back_populates="library_root", + cascade="all, delete-orphan", + uselist=False, + ) + + +class LibraryRootPolicy(Base, IdentityMixin, TimestampMixin): + """Complete explicit naming policy owned by exactly one library root.""" + + __tablename__ = "library_root_policies" + + library_root_id: Mapped[int] = mapped_column( + ForeignKey("library_roots.id", ondelete="CASCADE"), + nullable=False, + unique=True, + ) + schema_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + series_path_template: Mapped[str] = mapped_column(String(1024), nullable=False) + comic_file_template: Mapped[str] = mapped_column(String(1024), nullable=False) + annual_file_template: Mapped[str] = mapped_column(String(1024), nullable=False) + non_standard_file_template: Mapped[str] = mapped_column(String(1024), nullable=False) + single_non_standard_file_template: Mapped[str] = mapped_column(String(1024), nullable=False) + replace_illegal_characters: Mapped[bool] = mapped_column(Boolean, nullable=False) + colon_replacement: Mapped[str] = mapped_column(String(16), nullable=False) + source: Mapped[LibraryRootPolicySource] = mapped_column( + SQLAlchemyEnum( + LibraryRootPolicySource, + values_callable=lambda enum_cls: [member.value for member in enum_cls], + native_enum=False, + create_constraint=True, + ), + nullable=False, + ) + source_import_job_id: Mapped[int | None] = mapped_column( + ForeignKey("import_jobs.id", ondelete="SET NULL"), + nullable=True, + ) + revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + + library_root: Mapped[LibraryRoot] = relationship(back_populates="naming_policy") diff --git a/src/pullbox/models/series.py b/src/pullbox/models/series.py index 41956429..9caa638e 100644 --- a/src/pullbox/models/series.py +++ b/src/pullbox/models/series.py @@ -146,12 +146,22 @@ class Series(Base, IdentityMixin, TimestampMixin): ForeignKey("publishers.id", ondelete="SET NULL") ) library_root_id: Mapped[int | None] = mapped_column( - ForeignKey("library_roots.id", ondelete="SET NULL") + ForeignKey("library_roots.id", ondelete="RESTRICT") + ) + preferred_library_root_id: Mapped[int | None] = mapped_column( + ForeignKey("library_roots.id", ondelete="RESTRICT") ) # Relationships publisher: Mapped[Publisher | None] = relationship(back_populates="series") - library_root: Mapped[LibraryRoot | None] = relationship(back_populates="series") + library_root: Mapped[LibraryRoot | None] = relationship( + back_populates="series", + foreign_keys=[library_root_id], + ) + preferred_library_root: Mapped[LibraryRoot | None] = relationship( + back_populates="preferred_series", + foreign_keys=[preferred_library_root_id], + ) issues: Mapped[list[Issue]] = relationship( back_populates="series", cascade="all, delete-orphan" ) diff --git a/src/pullbox/models/story_arc.py b/src/pullbox/models/story_arc.py index d62c3179..3363fd14 100644 --- a/src/pullbox/models/story_arc.py +++ b/src/pullbox/models/story_arc.py @@ -1,44 +1,464 @@ -"""StoryArc ORM model and IssueStoryArc junction table.""" +"""First-class story-arc, membership, identity, and placement ORM models.""" from __future__ import annotations +import enum +from datetime import datetime # noqa: TC003 - SQLAlchemy needs this at runtime from typing import TYPE_CHECKING -from sqlalchemy import ForeignKey, Integer, String, Text -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy import ( + JSON, + CheckConstraint, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, + event, +) +from sqlalchemy import Enum as SQLAlchemyEnum +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates -from pullbox.models.base import Base, IdentityMixin, TimestampMixin +from pullbox.core.story_arc_identity import normalize_story_arc_name +from pullbox.models.base import Base, IdentityMixin, TimestampMixin, UTCDateTime if TYPE_CHECKING: + from sqlalchemy.engine import Connection + from sqlalchemy.orm import Mapper + + from pullbox.models.import_job import ImportJob, ImportJobAction from pullbox.models.issue import Issue + from pullbox.models.library import LibraryFile, LibraryRoot + from pullbox.models.story_arc_import import ImportedStoryArc + + +class StoryArcSourceKind(enum.StrEnum): + """Durable provenance for an arc, membership, import row, or placement.""" + + LEGACY = "legacy" + PULLBOX = "pullbox" + MYLAR3 = "mylar3" + FOLDER = "folder" + COMICINFO = "comicinfo" + PROVIDER = "provider" + + +class StoryArcLifecycle(enum.StrEnum): + """User-visible lifecycle of a canonical story arc.""" + + ACTIVE = "active" + ARCHIVED = "archived" + + +class StoryArcResolutionState(enum.StrEnum): + """Canonical issue-resolution state for an ordered arc entry.""" + + PENDING = "pending" + RESOLVED = "resolved" + MISSING = "missing" + AMBIGUOUS = "ambiguous" + CONFLICT = "conflict" + SKIPPED = "skipped" + + +class ImportedStoryArcStatus(enum.StrEnum): + """Review and execution state for staged story-arc evidence.""" + + DETECTED = "detected" + NEEDS_REVIEW = "needs_review" + READY = "ready" + CONFIRMED = "confirmed" + SKIPPED = "skipped" + IMPORTED = "imported" + FAILED = "failed" + + +class StoryArcPlacementMode(enum.StrEnum): + """How an optional arc placement represents the canonical file.""" + + COPY = "copy" + HARDLINK = "hardlink" + SYMLINK = "symlink" + REFERENCE_ONLY = "reference_only" + + +class StoryArcSymlinkStyle(enum.StrEnum): + """How a future managed symlink target is rendered.""" + + ABSOLUTE = "absolute" + RELATIVE = "relative" + + +class StoryArcPlacementOwnership(enum.StrEnum): + """Whether Pullbox owns a story-arc placement artifact.""" + + MANAGED = "managed" + REFERENCED = "referenced" + + +class StoryArcPlacementState(enum.StrEnum): + """Observed synchronization state of an arc placement.""" + + CURRENT = "current" + MISSING = "missing" + DRIFTED = "drifted" + FAILED = "failed" + + +def story_arc_enum_values(enum_cls: type[enum.Enum]) -> list[str]: + """Persist lowercase enum values instead of Python member names.""" + return [str(member.value) for member in enum_cls] + + +def story_arc_enum(enum_cls: type[enum.Enum]) -> SQLAlchemyEnum: + """Build a portable constrained VARCHAR enum for SQLite and PostgreSQL.""" + return SQLAlchemyEnum( + enum_cls, + values_callable=story_arc_enum_values, + native_enum=False, + create_constraint=True, + validate_strings=True, + ) class StoryArc(Base, IdentityMixin, TimestampMixin): + """A first-class ordered collection that references canonical issues.""" + __tablename__ = "story_arcs" + __table_args__ = ( + Index("ix_story_arcs_normalized_id", "normalized_name", "id"), + Index( + "ix_story_arcs_lifecycle_monitored_id", + "lifecycle", + "monitored", + "id", + ), + Index("ix_story_arcs_source_job_id", "source_import_job_id", "id"), + ) comicvine_id: Mapped[int | None] = mapped_column(unique=True, index=True) name: Mapped[str] = mapped_column(String(500), nullable=False, index=True) + normalized_name: Mapped[str] = mapped_column( + String(500), nullable=False, server_default="__legacy__" + ) description: Mapped[str | None] = mapped_column(Text) publisher_id: Mapped[int | None] = mapped_column( ForeignKey("publishers.id", ondelete="SET NULL") ) comicvine_url: Mapped[str | None] = mapped_column(String(500)) + cover_path: Mapped[str | None] = mapped_column(String(500)) + cover_url: Mapped[str | None] = mapped_column(String(500)) + source_kind: Mapped[StoryArcSourceKind] = mapped_column( + story_arc_enum(StoryArcSourceKind), + default=StoryArcSourceKind.LEGACY, + server_default=StoryArcSourceKind.LEGACY.value, + nullable=False, + ) + lifecycle: Mapped[StoryArcLifecycle] = mapped_column( + story_arc_enum(StoryArcLifecycle), + default=StoryArcLifecycle.ACTIVE, + server_default=StoryArcLifecycle.ACTIVE.value, + nullable=False, + ) + monitored: Mapped[bool] = mapped_column(default=False, server_default="0", nullable=False) + # Legacy import/API preferences remain round-trippable for rollback and + # compatibility. Acquisition and provider discovery use monitored alone. + search_missing: Mapped[bool] = mapped_column(default=False, server_default="0", nullable=False) + include_upcoming: Mapped[bool] = mapped_column( + default=False, server_default="0", nullable=False + ) + sync_enabled: Mapped[bool] = mapped_column(default=False, server_default="0", nullable=False) + target_library_root_id: Mapped[int | None] = mapped_column( + ForeignKey("library_roots.id", ondelete="RESTRICT") + ) + policy_schema_version: Mapped[int | None] = mapped_column(Integer) + policy_snapshot: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + source_import_job_id: Mapped[int | None] = mapped_column( + ForeignKey("import_jobs.id", ondelete="SET NULL") + ) + revision: Mapped[int] = mapped_column(Integer, default=1, server_default="1", nullable=False) + diagnostics: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) - # Relationships + memberships: Mapped[list[IssueStoryArc]] = relationship( + back_populates="story_arc", + cascade="all, delete-orphan", + order_by=lambda: ( + IssueStoryArc.sequence_number, + IssueStoryArc.source_ordinal, + IssueStoryArc.id, + ), + ) issues: Mapped[list[Issue]] = relationship( - secondary="issue_story_arcs", back_populates="story_arcs" + secondary="issue_story_arcs", + back_populates="story_arcs", + viewonly=True, + ) + external_identities: Mapped[list[StoryArcExternalIdentity]] = relationship( + back_populates="story_arc", cascade="all, delete-orphan" + ) + target_library_root: Mapped[LibraryRoot | None] = relationship( + foreign_keys=[target_library_root_id] + ) + source_import_job: Mapped[ImportJob | None] = relationship(foreign_keys=[source_import_job_id]) + proposed_imports: Mapped[list[ImportedStoryArc]] = relationship( + back_populates="proposed_story_arc", + foreign_keys="ImportedStoryArc.proposed_story_arc_id", ) + materialized_imports: Mapped[list[ImportedStoryArc]] = relationship( + back_populates="materialized_story_arc", + foreign_keys="ImportedStoryArc.materialized_story_arc_id", + ) + + @validates("name") + def _normalize_name(self, _key: str, value: str) -> str: + self.normalized_name = normalize_story_arc_name(value) + return value -class IssueStoryArc(Base): - """Junction table: Issue <-> StoryArc with sequence.""" +class IssueStoryArc(Base, IdentityMixin, TimestampMixin): + """One deterministic ordered arc entry, resolved or unresolved.""" __tablename__ = "issue_story_arcs" + __table_args__ = ( + UniqueConstraint( + "story_arc_id", + "issue_id", + name="uq_issue_story_arcs_arc_issue", + ), + Index( + "ix_issue_story_arcs_order", + "story_arc_id", + "sequence_number", + "source_ordinal", + "id", + ), + Index( + "ix_issue_story_arcs_review", + "story_arc_id", + "resolution_state", + "sequence_number", + "source_ordinal", + "id", + ), + Index("ix_issue_story_arcs_issue", "issue_id", "story_arc_id", "id"), + ) + + issue_id: Mapped[int | None] = mapped_column( + ForeignKey("issues.id", ondelete="SET NULL"), nullable=True + ) + story_arc_id: Mapped[int] = mapped_column( + ForeignKey("story_arcs.id", ondelete="CASCADE"), nullable=False + ) + sequence_number: Mapped[int] = mapped_column(Integer, nullable=False) + source_ordinal: Mapped[int] = mapped_column(Integer, nullable=False) + legacy_sequence_was_null: Mapped[bool] = mapped_column( + default=False, server_default="0", nullable=False + ) + resolution_state: Mapped[StoryArcResolutionState] = mapped_column( + story_arc_enum(StoryArcResolutionState), + default=StoryArcResolutionState.PENDING, + server_default=StoryArcResolutionState.PENDING.value, + nullable=False, + ) + source_kind: Mapped[StoryArcSourceKind] = mapped_column( + story_arc_enum(StoryArcSourceKind), + default=StoryArcSourceKind.LEGACY, + server_default=StoryArcSourceKind.LEGACY.value, + nullable=False, + ) + source_entry_id: Mapped[str | None] = mapped_column(String(255)) + source_arc_id: Mapped[str | None] = mapped_column(String(255)) + source_issue_id: Mapped[str | None] = mapped_column(String(255)) + source_series_id: Mapped[str | None] = mapped_column(String(255)) + source_issue_number_text: Mapped[str | None] = mapped_column(String(320)) + source_series_name: Mapped[str | None] = mapped_column(String(500)) + source_issue_title: Mapped[str | None] = mapped_column(String(500)) + source_publisher: Mapped[str | None] = mapped_column(String(255)) + source_release_date_text: Mapped[str | None] = mapped_column(String(50)) + source_issue_date_text: Mapped[str | None] = mapped_column(String(50)) + resolution_confidence: Mapped[float | None] = mapped_column(Float) + resolution_method: Mapped[str | None] = mapped_column(String(50)) + evidence: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + sync_eligible: Mapped[bool] = mapped_column(default=False, server_default="0", nullable=False) + last_materialization_result: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + + story_arc: Mapped[StoryArc] = relationship(back_populates="memberships") + issue: Mapped[Issue | None] = relationship(back_populates="story_arc_memberships") + placements: Mapped[list[StoryArcPlacement]] = relationship( + back_populates="issue_story_arc", cascade="all, delete-orphan" + ) - issue_id: Mapped[int] = mapped_column( - ForeignKey("issues.id", ondelete="CASCADE"), primary_key=True + +class StoryArcExternalIdentity(Base, IdentityMixin, TimestampMixin): + """Provider-neutral external identity attached to a canonical story arc.""" + + __tablename__ = "story_arc_external_identities" + __table_args__ = ( + UniqueConstraint( + "source", + "namespace", + "external_id", + name="uq_story_arc_external_identity", + ), + Index("ix_story_arc_external_identities_arc_id", "story_arc_id", "id"), ) + story_arc_id: Mapped[int] = mapped_column( - ForeignKey("story_arcs.id", ondelete="CASCADE"), primary_key=True + ForeignKey("story_arcs.id", ondelete="CASCADE"), nullable=False ) - sequence_number: Mapped[int | None] = mapped_column(Integer) + source: Mapped[str] = mapped_column(String(50), nullable=False) + namespace: Mapped[str] = mapped_column(String(100), nullable=False) + external_id: Mapped[str] = mapped_column(String(255), nullable=False) + source_url: Mapped[str | None] = mapped_column(String(1000)) + evidence: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + + story_arc: Mapped[StoryArc] = relationship(back_populates="external_identities") + + +class StoryArcPlacement(Base, IdentityMixin, TimestampMixin): + """Optional representation of a canonical issue in an arc location. + + Database constraints are authoritative for the complete placement matrix. + ORM writes also validate the final combination immediately before insert or + update so keyword and attribute assignment order cannot change the result. + """ + + __tablename__ = "story_arc_placements" + __table_args__ = ( + UniqueConstraint("placement_path", name="uq_story_arc_placements_path"), + CheckConstraint( + "((mode = 'reference_only' AND ownership = 'referenced') OR " + "(mode IN ('copy', 'hardlink', 'symlink') AND ownership = 'managed'))", + name="ck_story_arc_placements_mode_ownership", + ), + CheckConstraint( + "((mode = 'symlink' AND symlink_style IS NOT NULL) OR " + "(mode != 'symlink' AND symlink_style IS NULL))", + name="ck_story_arc_placements_symlink_style", + ), + Index("ix_story_arc_placements_membership", "issue_story_arc_id", "id"), + Index("ix_story_arc_placements_library_file", "library_file_id", "id"), + Index("ix_story_arc_placements_state", "state", "id"), + Index( + "ix_story_arc_placements_creating_action", + "creating_action_id", + "id", + ), + ) + + issue_story_arc_id: Mapped[int] = mapped_column( + ForeignKey("issue_story_arcs.id", ondelete="CASCADE"), nullable=False + ) + library_file_id: Mapped[int | None] = mapped_column( + ForeignKey("library_files.id", ondelete="SET NULL") + ) + library_root_id: Mapped[int | None] = mapped_column( + ForeignKey("library_roots.id", ondelete="RESTRICT") + ) + placement_path: Mapped[str] = mapped_column(String(1000), nullable=False) + mode: Mapped[StoryArcPlacementMode] = mapped_column( + story_arc_enum(StoryArcPlacementMode), + default=StoryArcPlacementMode.REFERENCE_ONLY, + server_default=StoryArcPlacementMode.REFERENCE_ONLY.value, + nullable=False, + ) + ownership: Mapped[StoryArcPlacementOwnership] = mapped_column( + story_arc_enum(StoryArcPlacementOwnership), + default=StoryArcPlacementOwnership.REFERENCED, + server_default=StoryArcPlacementOwnership.REFERENCED.value, + nullable=False, + ) + symlink_style: Mapped[StoryArcSymlinkStyle | None] = mapped_column( + story_arc_enum(StoryArcSymlinkStyle) + ) + source_kind: Mapped[StoryArcSourceKind] = mapped_column( + story_arc_enum(StoryArcSourceKind), + default=StoryArcSourceKind.LEGACY, + server_default=StoryArcSourceKind.LEGACY.value, + nullable=False, + ) + source_import_job_id: Mapped[int | None] = mapped_column( + ForeignKey("import_jobs.id", ondelete="SET NULL") + ) + creating_action_id: Mapped[int | None] = mapped_column( + ForeignKey("import_job_actions.id", ondelete="SET NULL") + ) + rendered_reading_order: Mapped[int | None] = mapped_column(Integer) + policy_schema_version: Mapped[int | None] = mapped_column(Integer) + operation_token: Mapped[str | None] = mapped_column(String(32)) + source_fingerprint: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + state: Mapped[StoryArcPlacementState] = mapped_column( + story_arc_enum(StoryArcPlacementState), + default=StoryArcPlacementState.CURRENT, + server_default=StoryArcPlacementState.CURRENT.value, + nullable=False, + ) + last_result: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + last_checked_at: Mapped[datetime | None] = mapped_column(UTCDateTime) + + issue_story_arc: Mapped[IssueStoryArc] = relationship(back_populates="placements") + library_file: Mapped[LibraryFile | None] = relationship(foreign_keys=[library_file_id]) + library_root: Mapped[LibraryRoot | None] = relationship(foreign_keys=[library_root_id]) + source_import_job: Mapped[ImportJob | None] = relationship(foreign_keys=[source_import_job_id]) + creating_action: Mapped[ImportJobAction | None] = relationship( + foreign_keys=[creating_action_id] + ) + + def validate_configuration(self) -> None: + """Validate one complete placement combination independent of assignment order.""" + raw_mode = self.__dict__.get("mode", StoryArcPlacementMode.REFERENCE_ONLY) + raw_ownership = self.__dict__.get( + "ownership", + StoryArcPlacementOwnership.REFERENCED, + ) + raw_symlink_style = self.__dict__.get("symlink_style") + try: + mode = StoryArcPlacementMode(raw_mode) + ownership = StoryArcPlacementOwnership(raw_ownership) + symlink_style = ( + StoryArcSymlinkStyle(raw_symlink_style) if raw_symlink_style is not None else None + ) + except (TypeError, ValueError) as exc: + raise ValueError("Story arc placement uses an unsupported mode or style") from exc + + expected_ownership = ( + StoryArcPlacementOwnership.REFERENCED + if mode is StoryArcPlacementMode.REFERENCE_ONLY + else StoryArcPlacementOwnership.MANAGED + ) + if ownership is not expected_ownership: + raise ValueError( + f"Story arc placement mode {mode.value} requires " + f"{expected_ownership.value} ownership" + ) + if mode is StoryArcPlacementMode.SYMLINK and symlink_style is None: + raise ValueError("A symlink story arc placement requires a symlink style") + if mode is not StoryArcPlacementMode.SYMLINK and symlink_style is not None: + raise ValueError("Only a symlink story arc placement may specify a symlink style") + + +@event.listens_for(StoryArcPlacement, "before_insert") +@event.listens_for(StoryArcPlacement, "before_update") +def _validate_story_arc_placement_before_write( + _mapper: Mapper[StoryArcPlacement], + _connection: Connection, + target: StoryArcPlacement, +) -> None: + """Fail ORM writes before SQL while leaving database checks authoritative.""" + target.validate_configuration() diff --git a/src/pullbox/models/story_arc_import.py b/src/pullbox/models/story_arc_import.py new file mode 100644 index 00000000..6bdfb685 --- /dev/null +++ b/src/pullbox/models/story_arc_import.py @@ -0,0 +1,183 @@ +"""Review-only import staging models for story-arc evidence.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import JSON, Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship, validates + +from pullbox.core.story_arc_identity import normalize_story_arc_name +from pullbox.models.base import Base, IdentityMixin, TimestampMixin +from pullbox.models.story_arc import ( + ImportedStoryArcStatus, + IssueStoryArc, + StoryArc, + StoryArcResolutionState, + StoryArcSourceKind, + story_arc_enum, +) + +if TYPE_CHECKING: + from pullbox.models.import_job import ImportedFile, ImportJob + from pullbox.models.issue import Issue + + +class ImportedStoryArc(Base, IdentityMixin, TimestampMixin): + """One detected arc held for review before Step 4 materialization.""" + + __tablename__ = "import_story_arcs" + __table_args__ = ( + UniqueConstraint( + "import_job_id", + "source_key", + name="uq_import_story_arcs_job_source_key", + ), + Index("ix_import_story_arcs_job_status_id", "import_job_id", "status", "id"), + Index( + "ix_import_story_arcs_job_normalized_id", + "import_job_id", + "normalized_name", + "id", + ), + ) + + import_job_id: Mapped[int] = mapped_column( + ForeignKey("import_jobs.id", ondelete="CASCADE"), nullable=False + ) + source_kind: Mapped[StoryArcSourceKind] = mapped_column( + story_arc_enum(StoryArcSourceKind), nullable=False + ) + source_key: Mapped[str] = mapped_column(String(255), nullable=False) + source_arc_id: Mapped[str | None] = mapped_column(String(255)) + source_ordinal: Mapped[int] = mapped_column(Integer, nullable=False) + name: Mapped[str | None] = mapped_column(String(500)) + normalized_name: Mapped[str | None] = mapped_column(String(500)) + description: Mapped[str | None] = mapped_column(Text) + status: Mapped[ImportedStoryArcStatus] = mapped_column( + story_arc_enum(ImportedStoryArcStatus), + default=ImportedStoryArcStatus.DETECTED, + server_default=ImportedStoryArcStatus.DETECTED.value, + nullable=False, + ) + selected_for_import: Mapped[bool] = mapped_column( + default=False, server_default="0", nullable=False + ) + proposed_story_arc_id: Mapped[int | None] = mapped_column( + ForeignKey("story_arcs.id", ondelete="SET NULL") + ) + materialized_story_arc_id: Mapped[int | None] = mapped_column( + ForeignKey("story_arcs.id", ondelete="SET NULL") + ) + proposed_policy_snapshot: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + source_settings_snapshot: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + diagnostics: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + + import_job: Mapped[ImportJob] = relationship(back_populates="story_arcs") + entries: Mapped[list[ImportedStoryArcEntry]] = relationship( + back_populates="imported_story_arc", + cascade="all, delete-orphan", + order_by=lambda: ( + ImportedStoryArcEntry.source_ordinal, + ImportedStoryArcEntry.id, + ), + ) + proposed_story_arc: Mapped[StoryArc | None] = relationship( + back_populates="proposed_imports", + foreign_keys=[proposed_story_arc_id], + ) + materialized_story_arc: Mapped[StoryArc | None] = relationship( + back_populates="materialized_imports", + foreign_keys=[materialized_story_arc_id], + ) + + @validates("name") + def _normalize_name(self, _key: str, value: str | None) -> str | None: + self.normalized_name = normalize_story_arc_name(value) if value is not None else None + return value + + +class ImportedStoryArcEntry(Base, IdentityMixin, TimestampMixin): + """One staged ordered entry with review evidence and no final side effect.""" + + __tablename__ = "import_story_arc_entries" + __table_args__ = ( + UniqueConstraint( + "imported_story_arc_id", + "source_ordinal", + name="uq_import_story_arc_entries_arc_ordinal", + ), + Index( + "ix_import_story_arc_entries_arc_resolution_order", + "imported_story_arc_id", + "resolution_state", + "reading_order", + "source_ordinal", + "id", + ), + Index("ix_import_story_arc_entries_import_file_id", "import_file_id"), + Index("ix_import_story_arc_entries_matched_issue_id", "matched_issue_id", "id"), + ) + + imported_story_arc_id: Mapped[int] = mapped_column( + ForeignKey("import_story_arcs.id", ondelete="CASCADE"), nullable=False + ) + import_file_id: Mapped[int | None] = mapped_column( + ForeignKey("import_files.id", ondelete="SET NULL") + ) + matched_issue_id: Mapped[int | None] = mapped_column( + ForeignKey("issues.id", ondelete="SET NULL") + ) + materialized_membership_id: Mapped[int | None] = mapped_column( + ForeignKey("issue_story_arcs.id", ondelete="SET NULL") + ) + source_ordinal: Mapped[int] = mapped_column(Integer, nullable=False) + reading_order: Mapped[int | None] = mapped_column(Integer) + reading_order_raw: Mapped[str | None] = mapped_column(String(50)) + resolution_state: Mapped[StoryArcResolutionState] = mapped_column( + story_arc_enum(StoryArcResolutionState), + default=StoryArcResolutionState.PENDING, + server_default=StoryArcResolutionState.PENDING.value, + nullable=False, + ) + source_kind: Mapped[StoryArcSourceKind] = mapped_column( + story_arc_enum(StoryArcSourceKind), nullable=False + ) + source_entry_id: Mapped[str | None] = mapped_column(String(255)) + source_arc_id: Mapped[str | None] = mapped_column(String(255)) + source_issue_id: Mapped[str | None] = mapped_column(String(255)) + source_series_id: Mapped[str | None] = mapped_column(String(255)) + source_issue_number_text: Mapped[str | None] = mapped_column(String(320)) + source_series_name: Mapped[str | None] = mapped_column(String(500)) + source_issue_title: Mapped[str | None] = mapped_column(String(500)) + source_publisher: Mapped[str | None] = mapped_column(String(255)) + source_release_date_text: Mapped[str | None] = mapped_column(String(50)) + source_issue_date_text: Mapped[str | None] = mapped_column(String(50)) + resolution_confidence: Mapped[float | None] = mapped_column(Float) + resolution_method: Mapped[str | None] = mapped_column(String(50)) + evidence: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + source_location: Mapped[str | None] = mapped_column(String(1000)) + selected_for_import: Mapped[bool] = mapped_column( + default=False, server_default="0", nullable=False + ) + diagnostics: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, default=dict, server_default="{}", nullable=False + ) + + imported_story_arc: Mapped[ImportedStoryArc] = relationship(back_populates="entries") + import_file: Mapped[ImportedFile | None] = relationship( + back_populates="story_arc_entries", + foreign_keys=[import_file_id], + ) + matched_issue: Mapped[Issue | None] = relationship(foreign_keys=[matched_issue_id]) + materialized_membership: Mapped[IssueStoryArc | None] = relationship( + foreign_keys=[materialized_membership_id] + ) diff --git a/src/pullbox/models/story_arc_sync.py b/src/pullbox/models/story_arc_sync.py new file mode 100644 index 00000000..513189d3 --- /dev/null +++ b/src/pullbox/models/story_arc_sync.py @@ -0,0 +1,189 @@ +"""Durable work records for automatic story-arc placement synchronization.""" + +from __future__ import annotations + +import enum +from datetime import datetime # noqa: TC003 - SQLAlchemy needs this at runtime +from typing import TYPE_CHECKING + +from sqlalchemy import ( + JSON, + BigInteger, + Boolean, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, +) +from sqlalchemy import Enum as SQLAlchemyEnum +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from pullbox.models.base import Base, IdentityMixin, TimestampMixin, UTCDateTime + +if TYPE_CHECKING: + from pullbox.models.import_job import ImportJobAction + from pullbox.models.library import LibraryFile + from pullbox.models.story_arc import IssueStoryArc + + +class StoryArcSyncWorkState(enum.StrEnum): + """Durable lifecycle for one desired story-arc placement generation.""" + + QUEUED = "queued" + RUNNING = "running" + RETRY_WAIT = "retry_wait" + FAILED = "failed" + COMPLETED = "completed" + CANCELLED = "cancelled" + + +class StoryArcSyncReason(enum.StrEnum): + """Why automatic synchronization work was created.""" + + CANONICAL_REGISTERED = "canonical_registered" + DISCREPANCY_RECOVERY = "discrepancy_recovery" + + +def _enum(enum_cls: type[enum.Enum]) -> SQLAlchemyEnum: + return SQLAlchemyEnum( + enum_cls, + values_callable=lambda cls: [str(member.value) for member in cls], + native_enum=False, + create_constraint=True, + validate_strings=True, + ) + + +class StoryArcSyncWork(Base, IdentityMixin, TimestampMixin): + """One idempotent, leased request to synchronize a canonical file into an arc.""" + + __tablename__ = "story_arc_sync_work" + __table_args__ = ( + UniqueConstraint( + "issue_story_arc_id", + "desired_generation", + name="uq_story_arc_sync_work_generation", + ), + UniqueConstraint( + "origin_import_action_id", + name="uq_story_arc_sync_work_origin_import_action", + ), + Index( + "ix_story_arc_sync_work_queued", + "claimable", + "state", + "created_at", + "id", + ), + Index( + "ix_story_arc_sync_work_ready", + "claimable", + "state", + "next_attempt_at", + "id", + ), + Index( + "ix_story_arc_sync_work_stale_claim", + "claimable", + "state", + "claimed_at", + "id", + ), + Index( + "ix_story_arc_sync_work_origin_job_state", + "origin_import_job_id", + "state", + "id", + ), + Index( + "ix_story_arc_sync_work_membership", + "issue_story_arc_id", + "id", + ), + Index( + "ix_story_arc_sync_work_library_file", + "library_file_id", + "id", + ), + ) + + issue_story_arc_id: Mapped[int] = mapped_column( + ForeignKey("issue_story_arcs.id", ondelete="CASCADE"), + nullable=False, + ) + library_file_id: Mapped[int] = mapped_column( + ForeignKey("library_files.id", ondelete="CASCADE"), + nullable=False, + ) + origin_import_action_id: Mapped[int | None] = mapped_column( + ForeignKey("import_job_actions.id", ondelete="SET NULL") + ) + origin_import_job_id: Mapped[int | None] = mapped_column( + ForeignKey("import_jobs.id", ondelete="SET NULL") + ) + origin_imported_story_arc_id: Mapped[int | None] = mapped_column( + ForeignKey("import_story_arcs.id", ondelete="SET NULL") + ) + origin_imported_story_arc_entry_id: Mapped[int | None] = mapped_column( + ForeignKey("import_story_arc_entries.id", ondelete="SET NULL") + ) + desired_generation: Mapped[str] = mapped_column(String(64), nullable=False) + source_signature_hash: Mapped[str] = mapped_column(String(64), nullable=False) + source_file_path: Mapped[str] = mapped_column(String(1000), nullable=False) + source_file_size: Mapped[int] = mapped_column(BigInteger, nullable=False) + source_file_modified_at: Mapped[datetime] = mapped_column(UTCDateTime, nullable=False) + source_file_hash: Mapped[str | None] = mapped_column(String(64)) + source_signature_schema_version: Mapped[int | None] = mapped_column(Integer) + source_signature_resolved_path: Mapped[str | None] = mapped_column(String(1000)) + source_signature_size: Mapped[int | None] = mapped_column(BigInteger) + source_signature_mtime_ns: Mapped[int | None] = mapped_column(BigInteger) + source_signature_device: Mapped[int | None] = mapped_column(BigInteger) + source_signature_inode: Mapped[int | None] = mapped_column(BigInteger) + story_arc_revision: Mapped[int] = mapped_column(Integer, nullable=False) + membership_sequence: Mapped[int] = mapped_column(Integer, nullable=False) + policy_schema_version: Mapped[int] = mapped_column(Integer, nullable=False) + reason: Mapped[StoryArcSyncReason] = mapped_column( + _enum(StoryArcSyncReason), + default=StoryArcSyncReason.CANONICAL_REGISTERED, + server_default=StoryArcSyncReason.CANONICAL_REGISTERED.value, + nullable=False, + ) + state: Mapped[StoryArcSyncWorkState] = mapped_column( + _enum(StoryArcSyncWorkState), + default=StoryArcSyncWorkState.QUEUED, + server_default=StoryArcSyncWorkState.QUEUED.value, + nullable=False, + ) + claimable: Mapped[bool] = mapped_column( + Boolean, + default=True, + server_default="1", + nullable=False, + ) + attempt_count: Mapped[int] = mapped_column( + Integer, + default=0, + server_default="0", + nullable=False, + ) + next_attempt_at: Mapped[datetime | None] = mapped_column(UTCDateTime) + claim_token: Mapped[str | None] = mapped_column(String(64)) + claimed_at: Mapped[datetime | None] = mapped_column(UTCDateTime) + cancel_requested_at: Mapped[datetime | None] = mapped_column(UTCDateTime) + last_error_code: Mapped[str | None] = mapped_column(String(100)) + last_error_category: Mapped[str | None] = mapped_column(String(50)) + last_error_detail: Mapped[str | None] = mapped_column(Text) + last_result: Mapped[dict] = mapped_column( # type: ignore[type-arg] + JSON, + default=dict, + server_default="{}", + nullable=False, + ) + + issue_story_arc: Mapped[IssueStoryArc] = relationship() + library_file: Mapped[LibraryFile] = relationship() + origin_import_action: Mapped[ImportJobAction | None] = relationship( + foreign_keys=[origin_import_action_id] + ) diff --git a/src/pullbox/performance/import_target_harness.py b/src/pullbox/performance/import_target_harness.py new file mode 100644 index 00000000..6e2ca61b --- /dev/null +++ b/src/pullbox/performance/import_target_harness.py @@ -0,0 +1,757 @@ +"""Standard, bounded reporting contract for IU7 import target measurements. + +The harness deliberately distinguishes a runnable measurement lane from a +complete release proof. Existing deterministic benchmark scripts can supply +bounded stage/counter evidence today; target-scale reports fail closed until +the API, WAL/lock, cancellation, restart, and rollback probes are present. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.parse +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum +from typing import TYPE_CHECKING, Protocol + +from pullbox.performance.baseline import ( + EndpointSpec, + Fetcher, + collect_context, + default_fetcher, + measure_http_endpoint, + parse_endpoint_spec, + summarize_numbers, +) +from pullbox.performance.direct_download_baseline import extract_json_report + +if TYPE_CHECKING: + from collections.abc import Sequence + from pathlib import Path + +TARGET_REPORT_SCHEMA_VERSION = "1.0" +_MAX_API_URLS = 20 +_MAX_FAILURES = 50 +_PEAK_RSS_TARGET_BYTES = 768 * 1024 * 1024 +_PEAK_RSS_HARD_STOP_BYTES = 1024 * 1024 * 1024 + + +class ImportTargetBackend(StrEnum): + """Database backend selected for one target measurement.""" + + SQLITE = "sqlite" + POSTGRESQL = "postgresql" + + +class ImportTargetScaleProfile(StrEnum): + """Frozen deterministic scale profiles used by IU7.""" + + CI = "ci" + FILES_10K = "10k" + FILES_50K = "50k" + FILES_100K = "100k" + FILES_200K = "200k" + + +class ImportTargetSourceLane(StrEnum): + """Import lifecycle lane measured by the child benchmark.""" + + METADATA = "metadata" + FOLDER = "folder" + MYLAR3 = "mylar3" + MANAGED_COPY = "managed_copy" + + +class ImportTargetCacheState(StrEnum): + """Declared warm/cold state for a repeatable run.""" + + WARM = "warm" + COLD = "cold" + + +@dataclass(frozen=True, slots=True) +class ImportTargetScaleShape: + """Exact synthetic shape represented by one named profile.""" + + file_count: int + series_count: int + story_arc_count: int + + @property + def files_per_series(self) -> int: + return self.file_count // self.series_count + + +_SCALE_SHAPES: dict[ImportTargetScaleProfile, ImportTargetScaleShape] = { + ImportTargetScaleProfile.CI: ImportTargetScaleShape(100, 25, 10), + ImportTargetScaleProfile.FILES_10K: ImportTargetScaleShape(10_000, 2_500, 500), + ImportTargetScaleProfile.FILES_50K: ImportTargetScaleShape(50_000, 12_500, 2_500), + ImportTargetScaleProfile.FILES_100K: ImportTargetScaleShape(100_000, 25_000, 5_000), + ImportTargetScaleProfile.FILES_200K: ImportTargetScaleShape(200_000, 50_000, 10_000), +} + + +@dataclass(frozen=True, slots=True) +class ImportTargetConfig: + """Complete non-secret input contract for one harness invocation.""" + + repo_root: Path + seed: int + backend: ImportTargetBackend + scale_profile: ImportTargetScaleProfile + source_lane: ImportTargetSourceLane + cache_state: ImportTargetCacheState + injection_point: str + api_urls: tuple[str, ...] + samples: int + timeout_seconds: float + environment_label: str + filesystem_label: str + + def __post_init__(self) -> None: + if isinstance(self.seed, bool) or self.seed < 0: + raise ValueError("seed must be a non-negative integer") + if self.samples < 1 or self.samples > 20: + raise ValueError("samples must be between 1 and 20") + if self.timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + if not self.environment_label.strip() or len(self.environment_label) > 100: + raise ValueError("environment_label must contain 1 to 100 characters") + if not self.filesystem_label.strip() or len(self.filesystem_label) > 100: + raise ValueError("filesystem_label must contain 1 to 100 characters") + if not self.injection_point.strip() or len(self.injection_point) > 100: + raise ValueError("injection_point must contain 1 to 100 characters") + if len(self.api_urls) > _MAX_API_URLS: + raise ValueError(f"api_urls cannot contain more than {_MAX_API_URLS} entries") + for raw in self.api_urls: + _sanitized_endpoint_spec(raw) + + +@dataclass(frozen=True, slots=True) +class ImportTargetWorkload: + """One deterministic child command and any known capability blockers.""" + + command: tuple[str, ...] + capability_failures: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class TargetCommandSample: + """One isolated child result retained only until it is summarized.""" + + report: dict[str, object] | None + wall_elapsed_ms: float + error: str | None = None + + +class TargetSampleRunner(Protocol): + """Injectable command boundary used by unit tests and the CLI.""" + + def __call__( + self, + workload: ImportTargetWorkload, + config: ImportTargetConfig, + ) -> TargetCommandSample: ... + + +def scale_shape(profile: ImportTargetScaleProfile) -> ImportTargetScaleShape: + """Return the immutable shape for one named profile.""" + return _SCALE_SHAPES[profile] + + +def build_target_workload(config: ImportTargetConfig) -> ImportTargetWorkload: + """Map a complete harness configuration to one shell-free child command.""" + failures: list[str] = [] + if ( + config.backend is ImportTargetBackend.POSTGRESQL + and config.source_lane is not ImportTargetSourceLane.METADATA + ): + failures.append("postgresql_source_lane_not_instrumented") + if config.injection_point != "none": + failures.append("cancel_restart_injection_not_instrumented") + if failures: + return ImportTargetWorkload(command=(), capability_failures=tuple(failures)) + + shape = scale_shape(config.scale_profile) + common = ( + "--series-count", + str(shape.series_count), + "--files-per-series", + str(shape.files_per_series), + ) + command: tuple[str, ...] + if config.source_lane is ImportTargetSourceLane.METADATA: + command = ( + "scripts/benchmark_import_metadata_scale.py", + *common, + "--story-arc-count", + str(shape.story_arc_count), + ) + if config.backend is ImportTargetBackend.POSTGRESQL: + command = ( + *command, + "--backend", + "postgresql", + "--reset-dedicated-database", + ) + elif config.source_lane is ImportTargetSourceLane.FOLDER: + command = ("scripts/benchmark_import_scan.py", *common) + elif config.source_lane is ImportTargetSourceLane.MYLAR3: + command = ( + "scripts/benchmark_mylar3_import.py", + *common, + "--annual-count", + str(max(shape.series_count // 10, 1)), + ) + else: + command = ( + "scripts/benchmark_import_execute.py", + *common, + "--file-work-profile", + "mixed-small", + "--report-sample-limit", + "3", + ) + return ImportTargetWorkload(command=command) + + +def run_target_command_sample( + workload: ImportTargetWorkload, + config: ImportTargetConfig, +) -> TargetCommandSample: + """Run one child measurement without shell interpolation or secret capture.""" + if not workload.command: + return TargetCommandSample( + report=None, + wall_elapsed_ms=0.0, + error="workload has unresolved capability failures", + ) + environment = os.environ.copy() + environment["PULLBOX_IMPORT_BENCHMARK_SEED"] = str(config.seed) + environment["PULLBOX_IMPORT_BENCHMARK_CACHE_STATE"] = config.cache_state.value + started_at = time.perf_counter() + try: + result = subprocess.run( + [sys.executable, *workload.command], + cwd=config.repo_root, + env=environment, + capture_output=True, + text=True, + timeout=config.timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired: + return TargetCommandSample( + report=None, + wall_elapsed_ms=(time.perf_counter() - started_at) * 1000, + error=f"timed out after {config.timeout_seconds:g}s", + ) + except OSError as exc: + return TargetCommandSample( + report=None, + wall_elapsed_ms=(time.perf_counter() - started_at) * 1000, + error=f"child_launch_error_{type(exc).__name__}", + ) + + elapsed_ms = (time.perf_counter() - started_at) * 1000 + if result.returncode != 0: + # Child output may contain fixture paths. Retain only a fixed error code. + return TargetCommandSample( + report=None, + wall_elapsed_ms=elapsed_ms, + error=f"child_exit_{result.returncode}", + ) + try: + report = extract_json_report(result.stdout) + except ValueError: + return TargetCommandSample( + report=None, + wall_elapsed_ms=elapsed_ms, + error="child_report_invalid", + ) + return TargetCommandSample(report=report, wall_elapsed_ms=elapsed_ms) + + +_NUMERIC_FIELDS = ( + "represented_file_count", + "series_count", + "story_arc_count", + "confirmed_arc_count", + "final_matched_series_count", + "final_no_match_file_count", + "final_confirmed_arc_count", + "elapsed_ms", + "total_elapsed_ms", + "seed_elapsed_ms", + "confirm_elapsed_ms", + "rollback_elapsed_ms", + "peak_rss_bytes", + "database_bytes", + "wal_bytes", + "confirm_select_count", + "rollback_select_count", + "query_count", + "transaction_p95_ms", + "transaction_p99_ms", + "lock_wait_p95_ms", + "lock_wait_p99_ms", + "filesystem_scan_count", + "archive_payload_count", + "archive_safety_inspection_count", + "archive_member_list_read_count", + "archive_member_payload_read_count", + "provider_call_count", + "provider_search_calls", + "provider_get_series_calls", + "provider_issue_summary_calls", + "provider_issue_number_calls", + "progress_event_count", + "progress_write_count", + "cancel_latency_ms", + "restart_latency_ms", + "rollback_recovery_ms", + "orphan_discrepancy_count", + "source_mutation_count", +) + +_PARITY_SEMANTIC_METRICS = ( + "represented_file_count", + "series_count", + "story_arc_count", + "confirmed_arc_count", + "final_matched_series_count", + "final_no_match_file_count", + "final_confirmed_arc_count", + "archive_payload_count", + "provider_call_count", + "filesystem_scan_count", + "confirm_select_count", + "rollback_select_count", +) + + +def _numeric_value(report: Mapping[str, object], field: str) -> float | None: + value = report.get(field) + if isinstance(value, int | float) and not isinstance(value, bool): + return float(value) + return None + + +def _summarize_samples( + workload: ImportTargetWorkload, + samples: Sequence[TargetCommandSample], +) -> dict[str, object]: + successful = [sample for sample in samples if sample.report is not None] + failures = [ + sample.error or "child_report_missing" for sample in samples if sample.report is None + ][:_MAX_FAILURES] + metrics: dict[str, dict[str, float | int]] = {} + for field in _NUMERIC_FIELDS: + values = [ + numeric + for sample in successful + if sample.report is not None + and (numeric := _numeric_value(sample.report, field)) is not None + ] + if values: + metrics[field] = summarize_numbers(values) + wall_values = [sample.wall_elapsed_ms for sample in successful] + digests = [ + hashlib.sha256( + json.dumps(sample.report, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + for sample in successful + ] + return { + "command": list(workload.command), + "capability_failures": list(workload.capability_failures), + "samples_requested": len(samples), + "samples_completed": len(successful), + "failure_count": len(samples) - len(successful), + "failures": failures, + "wall_timing_ms": summarize_numbers(wall_values) if wall_values else None, + "metrics": metrics, + # Raw child reports can contain path samples and grow with new fields. + # Digests prove repeatability without retaining those payloads. + "sample_report_digests": digests, + } + + +def _sanitized_endpoint_spec(raw: str) -> EndpointSpec: + spec = parse_endpoint_spec(raw) + parsed = urllib.parse.urlparse(spec.target) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("target API URLs must be absolute HTTP(S) URLs") + if parsed.username is not None or parsed.password is not None: + raise ValueError("target API URLs cannot contain credentials") + sanitized = urllib.parse.urlunparse( + (parsed.scheme, parsed.netloc, parsed.path or "/", "", "", "") + ) + return EndpointSpec(label=spec.label[:100], target=sanitized) + + +def _measure_api( + config: ImportTargetConfig, + *, + fetcher: Fetcher, +) -> list[dict[str, object]]: + measurements: list[dict[str, object]] = [] + for raw in config.api_urls: + measurement = measure_http_endpoint( + _sanitized_endpoint_spec(raw), + base_url="http://unused.invalid", + samples=config.samples, + timeout=config.timeout_seconds, + fetcher=fetcher, + ).to_dict() + errors = measurement.get("errors") + if isinstance(errors, list): + measurement["errors"] = [ + str(error).partition(":")[0] for error in errors[:_MAX_FAILURES] + ] + measurements.append(measurement) + return measurements + + +def _observed_shape(report: Mapping[str, object]) -> tuple[int | None, int | None]: + raw_files = report.get("represented_file_count", report.get("expected_file_count")) + raw_series = report.get("series_count", report.get("source_series_count")) + series_count = ( + int(raw_series) + if isinstance(raw_series, int) and not isinstance(raw_series, bool) + else None + ) + file_count = ( + int(raw_files) if isinstance(raw_files, int) and not isinstance(raw_files, bool) else None + ) + raw_files_per_series = report.get("files_per_series") + if ( + file_count is None + and series_count is not None + and isinstance(raw_files_per_series, int) + and not isinstance(raw_files_per_series, bool) + ): + file_count = series_count * raw_files_per_series + return file_count, series_count + + +def _elapsed_value(report: Mapping[str, object]) -> float | None: + total = _numeric_value(report, "total_elapsed_ms") + return total if total is not None else _numeric_value(report, "elapsed_ms") + + +def _evaluate_gates( + config: ImportTargetConfig, + workload: ImportTargetWorkload, + samples: Sequence[TargetCommandSample], + api_measurements: Sequence[Mapping[str, object]], +) -> dict[str, object]: + hard_failures: list[str] = list(workload.capability_failures) + warnings: list[str] = [] + successful = [sample for sample in samples if sample.report is not None] + if len(successful) != config.samples: + hard_failures.append("workload_sample_failed") + + peak_values = [ + value + for sample in successful + if sample.report is not None + and (value := _numeric_value(sample.report, "peak_rss_bytes")) is not None + ] + if peak_values and max(peak_values) > _PEAK_RSS_HARD_STOP_BYTES: + hard_failures.append("peak_rss_hard_stop_exceeded") + + elapsed_values = [ + value + for sample in successful + if sample.report is not None and (value := _elapsed_value(sample.report)) is not None + ] + hard_elapsed_ms = ( + 15 * 60 * 1000 if config.source_lane is ImportTargetSourceLane.METADATA else 30 * 60 * 1000 + ) + if elapsed_values and max(elapsed_values) > hard_elapsed_ms: + hard_failures.append("wall_time_hard_stop_exceeded") + + if config.source_lane is ImportTargetSourceLane.MYLAR3: + for sample in successful: + assert sample.report is not None + if _numeric_value(sample.report, "provider_call_count") not in {0.0}: + hard_failures.append("trusted_mylar_provider_call_detected") + if _numeric_value(sample.report, "archive_member_list_read_count") not in {0.0}: + hard_failures.append("archive_member_index_reopened") + if _numeric_value(sample.report, "archive_member_payload_read_count") not in {0.0}: + hard_failures.append("archive_member_payload_reread") + + for sample in successful: + assert sample.report is not None + source_mutations = _numeric_value(sample.report, "source_mutation_count") + orphan_count = _numeric_value(sample.report, "orphan_discrepancy_count") + if source_mutations is not None and source_mutations > 0: + hard_failures.append("source_mutation_detected") + if orphan_count is not None and orphan_count > 0: + hard_failures.append("orphan_discrepancy_detected") + + for measurement in api_measurements: + samples_completed = measurement.get("samples_completed") + if samples_completed != config.samples: + hard_failures.append("api_measurement_incomplete") + status_codes = measurement.get("status_codes") + if isinstance(status_codes, Mapping) and any( + str(code).startswith("5") and int(count) > 0 + for code, count in status_codes.items() + if isinstance(count, int) + ): + hard_failures.append("api_5xx_response") + timing = measurement.get("timing_ms") + if isinstance(timing, Mapping): + maximum = timing.get("max") + if isinstance(maximum, int | float) and maximum > 5000: + hard_failures.append("api_request_hard_stop_exceeded") + else: + hard_failures.append("api_timing_not_measured") + + if config.scale_profile is ImportTargetScaleProfile.FILES_200K: + shape = scale_shape(config.scale_profile) + if any( + sample.report is None + or _observed_shape(sample.report) != (shape.file_count, shape.series_count) + for sample in samples + ): + hard_failures.append("target_shape_mismatch") + if peak_values and max(peak_values) > _PEAK_RSS_TARGET_BYTES: + hard_failures.append("target_peak_rss_exceeded") + if len(peak_values) != len(successful): + hard_failures.append("target_peak_rss_not_measured") + if len(elapsed_values) != len(successful): + hard_failures.append("target_wall_time_not_measured") + if not api_measurements: + hard_failures.append("target_api_latency_not_measured") + required_database_metrics = { + "wal_bytes", + "transaction_p99_ms", + "lock_wait_p99_ms", + } + if not successful or any( + sample.report is None + or any( + _numeric_value(sample.report, metric) is None + for metric in required_database_metrics + ) + for sample in samples + ): + hard_failures.append("target_wal_and_lock_metrics_not_measured") + required_recovery_metrics = { + "cancel_latency_ms", + "restart_latency_ms", + "rollback_recovery_ms", + "orphan_discrepancy_count", + } + if not successful or any( + sample.report is None + or any( + _numeric_value(sample.report, metric) is None + for metric in required_recovery_metrics + ) + for sample in samples + ): + hard_failures.append("target_cancel_restart_recovery_not_measured") + else: + if not api_measurements: + warnings.append("api_latency_not_sampled") + warnings.append("smoke_profile_is_not_release_proof") + + unique_failures = list(dict.fromkeys(hard_failures))[:_MAX_FAILURES] + return { + "passed": not unique_failures, + "hard_failures": unique_failures, + "warnings": list(dict.fromkeys(warnings))[:_MAX_FAILURES], + } + + +def build_import_target_report( + config: ImportTargetConfig, + *, + sample_runner: TargetSampleRunner = run_target_command_sample, + api_fetcher: Fetcher = default_fetcher, +) -> dict[str, object]: + """Run one target lane and return a bounded, gate-evaluated report.""" + workload = build_target_workload(config) + samples = ( + [sample_runner(workload, config) for _ in range(config.samples)] + if workload.command + else [ + TargetCommandSample( + report=None, + wall_elapsed_ms=0.0, + error="workload_capability_missing", + ) + for _ in range(config.samples) + ] + ) + api_measurements = _measure_api(config, fetcher=api_fetcher) + context = collect_context(config.repo_root) + context.pop("repo_root", None) + context.update( + { + "environment_label": config.environment_label, + "filesystem_label": config.filesystem_label, + "cpu_count": os.cpu_count(), + } + ) + shape = scale_shape(config.scale_profile) + return { + "schema_version": TARGET_REPORT_SCHEMA_VERSION, + "context": context, + "settings": { + "seed": config.seed, + "backend": config.backend.value, + "scale_profile": config.scale_profile.value, + "source_lane": config.source_lane.value, + "cache_state": config.cache_state.value, + "injection_point": config.injection_point, + "samples": config.samples, + "timeout_seconds": config.timeout_seconds, + "api_urls": [_sanitized_endpoint_spec(raw).target for raw in config.api_urls], + }, + "shape": { + "file_count": shape.file_count, + "series_count": shape.series_count, + "story_arc_count": shape.story_arc_count, + "files_per_series": shape.files_per_series, + }, + "workload": _summarize_samples(workload, samples), + "api_measurements": api_measurements, + "gate_evaluation": _evaluate_gates( + config, + workload, + samples, + api_measurements, + ), + } + + +def assert_comparable_target_reports( + left: Mapping[str, object], + right: Mapping[str, object], +) -> None: + """Refuse scale comparisons across unlike backend/hardware/filesystem lanes.""" + left_context = left.get("context") + right_context = right.get("context") + left_settings = left.get("settings") + right_settings = right.get("settings") + if not all( + isinstance(value, Mapping) + for value in (left_context, right_context, left_settings, right_settings) + ): + raise ValueError("target reports are missing comparison context") + assert isinstance(left_context, Mapping) + assert isinstance(right_context, Mapping) + assert isinstance(left_settings, Mapping) + assert isinstance(right_settings, Mapping) + comparisons = ( + ("environment_label", left_context, right_context), + ("filesystem_label", left_context, right_context), + ("platform", left_context, right_context), + ("backend", left_settings, right_settings), + ("source_lane", left_settings, right_settings), + ("cache_state", left_settings, right_settings), + ) + for key, left_values, right_values in comparisons: + if left_values.get(key) != right_values.get(key): + raise ValueError(f"target reports differ in {key}") + + +def _report_mapping(report: Mapping[str, object], key: str) -> Mapping[str, object]: + value = report.get(key) + if not isinstance(value, Mapping): + raise ValueError(f"target report is missing {key}") + return value + + +def _metric_median(report: Mapping[str, object], metric: str) -> float | None: + workload = _report_mapping(report, "workload") + metrics = workload.get("metrics") + if not isinstance(metrics, Mapping): + return None + summary = metrics.get(metric) + if not isinstance(summary, Mapping): + return None + median = summary.get("median") + if isinstance(median, int | float) and not isinstance(median, bool): + return float(median) + return None + + +def build_import_database_parity_report( + left: Mapping[str, object], + right: Mapping[str, object], +) -> dict[str, object]: + """Compare bounded SQLite/PostgreSQL reports for semantic parity.""" + left_settings = _report_mapping(left, "settings") + right_settings = _report_mapping(right, "settings") + backends = sorted( + str(value) for value in (left_settings.get("backend"), right_settings.get("backend")) + ) + failures: list[str] = [] + warnings: list[str] = [] + if backends != ["postgresql", "sqlite"]: + failures.append("sqlite_and_postgresql_reports_required") + for key in ( + "seed", + "scale_profile", + "source_lane", + "cache_state", + "injection_point", + "samples", + ): + if left_settings.get(key) != right_settings.get(key): + failures.append(f"setting_mismatch_{key}") + left_shape = _report_mapping(left, "shape") + right_shape = _report_mapping(right, "shape") + if left_shape != right_shape: + failures.append("target_shape_mismatch") + for report in (left, right): + settings = _report_mapping(report, "settings") + workload = _report_mapping(report, "workload") + if ( + workload.get("samples_completed") != settings.get("samples") + or workload.get("failure_count") != 0 + ): + failures.append("source_workload_incomplete") + gates = _report_mapping(report, "gate_evaluation") + if gates.get("passed") is not True: + warnings.append("source_release_gates_not_complete") + + semantic_metrics: dict[str, dict[str, float | None]] = {} + for metric in _PARITY_SEMANTIC_METRICS: + left_value = _metric_median(left, metric) + right_value = _metric_median(right, metric) + semantic_metrics[metric] = { + str(left_settings.get("backend")): left_value, + str(right_settings.get("backend")): right_value, + } + if left_value is None or right_value is None: + failures.append(f"semantic_metric_missing_{metric}") + elif left_value != right_value: + failures.append(f"semantic_metric_mismatch_{metric}") + hard_failures = list(dict.fromkeys(failures))[:_MAX_FAILURES] + return { + "schema_version": TARGET_REPORT_SCHEMA_VERSION, + "report_kind": "import_database_parity", + "backends": backends, + "scale_profile": left_settings.get("scale_profile"), + "shape": { + key: left_shape.get(key) + for key in ("file_count", "series_count", "story_arc_count", "files_per_series") + }, + "semantic_metrics": semantic_metrics, + "passed": not hard_failures, + "hard_failures": hard_failures, + "warnings": list(dict.fromkeys(warnings))[:_MAX_FAILURES], + } diff --git a/src/pullbox/providers/base.py b/src/pullbox/providers/base.py index 934e47b4..efee45d0 100644 --- a/src/pullbox/providers/base.py +++ b/src/pullbox/providers/base.py @@ -79,6 +79,7 @@ class IssueSummary: release_date: str | None cover_url: str | None issue_type: str + issue_number_text: str | None = None @dataclass(frozen=True) @@ -97,6 +98,7 @@ class IssueMetadata: comicvine_url: str | None creators: list[dict[str, str]] = field(default_factory=list) story_arcs: list[dict[str, str]] = field(default_factory=list) + issue_number_text: str | None = None @dataclass(frozen=True) @@ -358,6 +360,7 @@ def __init__(self) -> None: self._download_clients: dict[int, DownloadClient] = {} self._download_priorities: dict[int, int] = {} self._indexers: dict[int, Indexer] = {} + self._indexer_aliases: dict[int, int] = {} def register_metadata_provider(self, name: str, provider: MetadataProvider) -> None: self._metadata_providers[name] = provider @@ -372,8 +375,13 @@ def register_download_client( self._download_priorities[config_id] = priority def register_indexer(self, config_id: int, indexer: Indexer) -> None: + self._indexer_aliases.pop(config_id, None) self._indexers[config_id] = indexer + def register_indexer_alias(self, config_id: int, aggregate_id: int) -> None: + """Resolve a persisted source ID without adding another search request.""" + self._indexer_aliases[config_id] = aggregate_id + def get_metadata_provider(self, name: str = "comicvine") -> MetadataProvider: """Get a metadata provider by name.""" return self._metadata_providers[name] @@ -392,7 +400,7 @@ def get_indexers(self) -> list[Indexer]: def get_indexer(self, config_id: int) -> Indexer | None: """Get one registered indexer by its persisted config ID.""" - return self._indexers.get(config_id) + return self._indexers.get(self._indexer_aliases.get(config_id, config_id)) def get_indexer_items(self) -> list[tuple[int, Indexer]]: """Get (config_id, indexer) pairs for all registered indexers.""" diff --git a/src/pullbox/providers/download/qbittorrent.py b/src/pullbox/providers/download/qbittorrent.py index ce90688b..8fa0e07f 100644 --- a/src/pullbox/providers/download/qbittorrent.py +++ b/src/pullbox/providers/download/qbittorrent.py @@ -9,7 +9,6 @@ from __future__ import annotations -import hashlib import time from typing import Any @@ -17,6 +16,7 @@ import structlog from pullbox.core.config_resolver import resolve_runtime_service_url +from pullbox.core.torrent_metadata import torrent_info_hashes from pullbox.core.url_validation import normalize_peer_base_url from pullbox.providers.base import ClientOptions, DownloadStatus, ProviderHealthResult @@ -661,90 +661,11 @@ def _extract_magnet_hash(url: str) -> str | None: def _torrent_info_hashes(content: bytes) -> frozenset[str]: """Return exact v1/v2 info hashes from one bounded bencoded descriptor.""" try: - info_start, info_end = _top_level_info_span(content) + return torrent_info_hashes(content) except ValueError as exc: raise QBittorrentError( "Invalid torrent descriptor: missing valid bencoded info data" ) from exc - info = content[info_start:info_end] - return frozenset( - ( - hashlib.sha1(info, usedforsecurity=False).hexdigest(), - hashlib.sha256(info).hexdigest(), - ) - ) - - -def _top_level_info_span(content: bytes) -> tuple[int, int]: - if not content or content[0] != ord("d"): - raise ValueError("torrent descriptor must be a dictionary") - index = 1 - info_span: tuple[int, int] | None = None - while index < len(content) and content[index] != ord("e"): - key, index = _parse_bencoded_bytes(content, index) - value_start = index - index = _skip_bencoded_value(content, index, depth=1) - if key == b"info": - if info_span is not None: - raise ValueError("torrent descriptor has duplicate info dictionaries") - info_span = (value_start, index) - if index >= len(content) or content[index] != ord("e") or index + 1 != len(content): - raise ValueError("torrent descriptor is truncated or has trailing data") - if info_span is None or content[info_span[0]] != ord("d"): - raise ValueError("torrent descriptor has no info dictionary") - return info_span - - -def _skip_bencoded_value(content: bytes, index: int, *, depth: int) -> int: - if depth > 100 or index >= len(content): - raise ValueError("invalid bencode nesting") - marker = content[index] - if 48 <= marker <= 57: - _, end = _parse_bencoded_bytes(content, index) - return end - if marker == ord("i"): - end = content.find(b"e", index + 1) - if end < 0: - raise ValueError("unterminated bencoded integer") - value = content[index + 1 : end] - digits = value[1:] if value.startswith(b"-") else value - if ( - not digits - or not digits.isdigit() - or (len(digits) > 1 and digits.startswith(b"0")) - or value == b"-0" - ): - raise ValueError("invalid bencoded integer") - return end + 1 - if marker not in {ord("l"), ord("d")}: - raise ValueError("invalid bencoded value") - cursor = index + 1 - while cursor < len(content) and content[cursor] != ord("e"): - if marker == ord("d"): - _, cursor = _parse_bencoded_bytes(content, cursor) - cursor = _skip_bencoded_value(content, cursor, depth=depth + 1) - if cursor >= len(content): - raise ValueError("unterminated bencoded collection") - return cursor + 1 - - -def _parse_bencoded_bytes(content: bytes, index: int) -> tuple[bytes, int]: - colon = content.find(b":", index) - if colon < 0: - raise ValueError("invalid bencoded byte string") - raw_length = content[index:colon] - if ( - not raw_length - or not raw_length.isdigit() - or (len(raw_length) > 1 and raw_length.startswith(b"0")) - ): - raise ValueError("invalid bencoded byte-string length") - length = int(raw_length) - start = colon + 1 - end = start + length - if end > len(content): - raise ValueError("truncated bencoded byte string") - return content[start:end], end def _map_torrent_state(state: str) -> str: diff --git a/src/pullbox/providers/download/sabnzbd.py b/src/pullbox/providers/download/sabnzbd.py index 01b9cb95..2fcd0f57 100644 --- a/src/pullbox/providers/download/sabnzbd.py +++ b/src/pullbox/providers/download/sabnzbd.py @@ -9,6 +9,7 @@ from __future__ import annotations +import asyncio import time from typing import Any @@ -21,6 +22,8 @@ logger = structlog.get_logger(__name__) _REQUEST_TIMEOUT = 10.0 +_NZB_FETCH_TIMEOUT = httpx.Timeout(10.0, read=60.0) +_NZB_FETCH_DEADLINE = 90.0 class SABnzbdError(Exception): @@ -111,9 +114,13 @@ def _response_looks_like_nzb(content_type: str, body: bytes) -> bool: async def _download_nzb_bytes(self, url: str) -> bytes: """Fetch NZB bytes locally so SAB does not need direct indexer access.""" try: - response = await self._client.get(url, follow_redirects=True) + # Indexer proxies may retry upstream; keep local SAB control calls short. + async with asyncio.timeout(_NZB_FETCH_DEADLINE): + response = await self._client.get( + url, follow_redirects=True, timeout=_NZB_FETCH_TIMEOUT + ) response.raise_for_status() - except httpx.TimeoutException: + except (TimeoutError, httpx.TimeoutException): raise SABnzbdError("Failed to download NZB from URL: Request timed out") from None except httpx.HTTPStatusError as exc: raise SABnzbdError( diff --git a/src/pullbox/providers/indexer/newznab.py b/src/pullbox/providers/indexer/newznab.py index 048fe599..3e2876fb 100644 --- a/src/pullbox/providers/indexer/newznab.py +++ b/src/pullbox/providers/indexer/newznab.py @@ -21,6 +21,7 @@ from defusedxml.common import DefusedXmlException from pullbox.core.acquisition import AcquisitionProtocol +from pullbox.core.issue_numbers import format_issue_number from pullbox.providers.base import ( IndexerCapabilities, ProviderHealthResult, @@ -146,11 +147,7 @@ async def search(self, query: SearchQuery) -> list[ReleaseResult]: search_term = query.series_title if query.issue_number is not None: - issue_str = ( - str(int(query.issue_number)) - if query.issue_number == int(query.issue_number) - else str(query.issue_number) - ) + issue_str = format_issue_number(query.issue_number) search_term = f"{search_term} {issue_str}" params: dict[str, Any] = {"t": "search", "q": search_term} diff --git a/src/pullbox/providers/indexer/prowlarr.py b/src/pullbox/providers/indexer/prowlarr.py index 0158cf55..767bef89 100644 --- a/src/pullbox/providers/indexer/prowlarr.py +++ b/src/pullbox/providers/indexer/prowlarr.py @@ -16,6 +16,7 @@ import structlog from pullbox.core.acquisition import AcquisitionProtocol +from pullbox.core.issue_numbers import format_issue_number from pullbox.providers.base import ( IndexerCapabilities, ProviderHealthResult, @@ -23,6 +24,12 @@ SearchQuery, ) from pullbox.providers.indexer.newznab import NewznabIndexer +from pullbox.providers.indexer.torznab_transport import ( + ResolverAttemptCallback, + TorznabDescriptor, + TorznabTransport, + TorznabTransportError, +) logger = structlog.get_logger(__name__) @@ -138,6 +145,25 @@ async def _newznab_request(self, params: dict[str, Any]) -> str: # -- Indexer implementation --------------------------------------------- + async def fetch_torrent_descriptor( + self, + url: str, + *, + on_attempt: ResolverAttemptCallback | None = None, + ) -> TorznabDescriptor: + """Fetch manager-hosted torrent metadata without exposing its URL to the client.""" + transport = TorznabTransport( + http_client=self._client, + configured_base_url=self._base_url, + cache_namespace=f"prowlarr-descriptor:{self._base_url}", + ) + try: + return await transport.fetch_descriptor(url, on_attempt=on_attempt) + except TorznabTransportError as exc: + raise ProwlarrError( + f"Could not retrieve torrent metadata from Prowlarr: {exc}" + ) from exc + async def search(self, query: SearchQuery) -> list[ReleaseResult]: """Search across all Prowlarr indexers via the REST API.""" log = logger.bind( @@ -148,11 +174,7 @@ async def search(self, query: SearchQuery) -> list[ReleaseResult]: search_term = query.series_title if query.issue_number is not None: - issue_str = ( - str(int(query.issue_number)) - if query.issue_number == int(query.issue_number) - else str(query.issue_number) - ) + issue_str = format_issue_number(query.issue_number) search_term = f"{search_term} {issue_str}" params: dict[str, Any] = {"query": search_term, "type": "search"} diff --git a/src/pullbox/providers/indexer/torznab.py b/src/pullbox/providers/indexer/torznab.py index 5cb89f74..9eb4a198 100644 --- a/src/pullbox/providers/indexer/torznab.py +++ b/src/pullbox/providers/indexer/torznab.py @@ -80,7 +80,7 @@ def __init__( @property def browser_resolver_enabled(self) -> bool: - """Whether descriptor handoff must stay inside Pullbox.""" + """Whether this manual indexer opted into browser challenge resolution.""" return self._browser_resolver_enabled @property diff --git a/src/pullbox/providers/indexer/torznab_transport.py b/src/pullbox/providers/indexer/torznab_transport.py index 43c375de..ce8d0786 100644 --- a/src/pullbox/providers/indexer/torznab_transport.py +++ b/src/pullbox/providers/indexer/torznab_transport.py @@ -12,6 +12,7 @@ import httpx import structlog +from pullbox.core.torrent_metadata import torrent_info_hashes from pullbox.providers.direct.resolver import ( DirectResolverCookie, DirectResolverError, @@ -254,7 +255,7 @@ async def _get( headers["cookie"] = cookie_header try: request = self._client.build_request("GET", url, params=params, headers=headers) - response = await self._client.send(request, stream=True) + response = await self._client.send(request, stream=True, follow_redirects=False) try: declared_length = response.headers.get("content-length") if declared_length is not None: @@ -346,12 +347,11 @@ def _cookie_header(cookies: Sequence[DirectResolverCookie], url: str) -> str: def _looks_like_torrent(content: bytes) -> bool: - return ( - len(content) >= 12 - and content.startswith(b"d") - and content.endswith(b"e") - and b"4:info" in content - ) + try: + torrent_info_hashes(content) + except ValueError: + return False + return True def _read_cached_material( diff --git a/src/pullbox/providers/metadata/comicvine.py b/src/pullbox/providers/metadata/comicvine.py index 91d15c1b..f6be830c 100644 --- a/src/pullbox/providers/metadata/comicvine.py +++ b/src/pullbox/providers/metadata/comicvine.py @@ -11,14 +11,17 @@ from __future__ import annotations import asyncio +import hashlib import html import re import time -from typing import Any +import weakref +from typing import TYPE_CHECKING, Any import httpx import structlog +from pullbox.core.issue_numbers import format_issue_number, parse_issue_number_text from pullbox.core.naming import detect_issue_type from pullbox.providers.base import ( IssueMetadata, @@ -28,6 +31,11 @@ SeriesSearchResult, ) +if TYPE_CHECKING: + from collections.abc import Sequence + + from pullbox.providers.story_arcs import StoryArcMetadata, StoryArcSearchResult + logger = structlog.get_logger(__name__) # ComicVine API status codes @@ -45,6 +53,8 @@ _GLOBAL_SERIES_SEARCH_BATCH_SIZE = 100 _GLOBAL_SERIES_SEARCH_MAX_RESULTS = 1000 _GLOBAL_SERIES_SEARCH_CACHE_TTL_SECONDS = 300.0 +_BULK_RESOURCE_PAGE_SIZE = 100 +_MAX_BULK_PROVIDER_IDS = 5000 _GLOBAL_SERIES_SEARCH_CACHE: dict[ tuple[str, int, int], tuple[float, tuple[SeriesSearchResult, ...], int], @@ -88,6 +98,78 @@ def _refill(self) -> None: self._last_refill = now +class _RequestVelocityGate: + """Serialize request starts without conflating velocity and hourly quotas.""" + + def __init__(self) -> None: + self._last_started_at: float | None = None + self._lock = asyncio.Lock() + + async def acquire(self, requests_per_second: int) -> None: + minimum_interval = 1.0 / max(int(requests_per_second), 1) + async with self._lock: + now = time.monotonic() + if self._last_started_at is not None: + wait_seconds = minimum_interval - (now - self._last_started_at) + if wait_seconds > 0: + await asyncio.sleep(wait_seconds) + self._last_started_at = time.monotonic() + + +class _ComicVineRateCoordinator: + """Share per-resource hourly budgets and velocity pacing across clients.""" + + def __init__(self, rate_limit: int) -> None: + self._rate_limit = max(int(rate_limit), 1) + self._resource_limiters: dict[str, _TokenBucket] = {} + self._velocity_gate = _RequestVelocityGate() + + async def acquire(self, resource: str, *, requests_per_second: int) -> None: + limiter = self._resource_limiters.get(resource) + if limiter is None: + limiter = _TokenBucket( + max_tokens=self._rate_limit, + refill_rate=self._rate_limit / 3600.0, + ) + self._resource_limiters[resource] = limiter + await limiter.acquire() + await self._velocity_gate.acquire(requests_per_second) + + +_RATE_COORDINATORS: weakref.WeakKeyDictionary[ + asyncio.AbstractEventLoop, + dict[tuple[str, int], _ComicVineRateCoordinator], +] = weakref.WeakKeyDictionary() + + +def _resource_rate_key(endpoint: str) -> str: + """Return ComicVine's first path segment, which defines its API resource.""" + return endpoint.strip("/").partition("/")[0] or "unknown" + + +def _rate_coordinator_for( + *, + api_key: str, + rate_limit: int, + requests_per_second: int, +) -> _ComicVineRateCoordinator: + """Return one process-local rate coordinator without retaining an API key.""" + _ = requests_per_second + loop = asyncio.get_running_loop() + coordinators = _RATE_COORDINATORS.setdefault(loop, {}) + # This non-secret fingerprint only groups an in-memory rate limiter; it is + # not stored or used to authenticate the API key as a password. + # codeql[py/weak-sensitive-data-hashing] + key_fingerprint = hashlib.sha256(api_key.encode("utf-8")).hexdigest() + normalized_rate_limit = max(int(rate_limit), 1) + coordinator_key = (key_fingerprint, normalized_rate_limit) + coordinator = coordinators.get(coordinator_key) + if coordinator is None: + coordinator = _ComicVineRateCoordinator(normalized_rate_limit) + coordinators[coordinator_key] = coordinator + return coordinator + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -108,18 +190,22 @@ def _parse_issue_number(value: str | None) -> float: Handles fraction characters (½ → 0.5) and numeric strings. Falls back to 0.0 for unparseable values like "Annual 1". """ + return _parse_issue_number_fields(value)[0] + + +def _parse_issue_number_fields(value: str | None) -> tuple[float, str | None]: + """Parse ComicVine numeric compatibility and preserve exact raw semantics.""" if value is None: - return 0.0 - value = value.replace("½", ".5").replace("¼", ".25").replace("¾", ".75") + return 0.0, None try: - return float(value) + return parse_issue_number_text(value) except ValueError: - return 0.0 + return 0.0, None def _format_issue_number_filter(value: float) -> str: """Format an issue number for ComicVine's exact issue_number filter.""" - return str(int(value)) if value == int(value) else f"{value:g}" + return format_issue_number(value) def _safe_int(value: Any) -> int | None: @@ -160,20 +246,89 @@ def _series_search_result_from_item(item: dict[str, Any]) -> SeriesSearchResult: ) +def _series_metadata_from_item( + item: dict[str, Any], + *, + fallback_provider_id: str, +) -> SeriesMetadata: + publisher_data = item.get("publisher") + publisher_name = publisher_data.get("name") if isinstance(publisher_data, dict) else None + image_data = item.get("image") + cover_url = image_data.get("medium_url") if isinstance(image_data, dict) else None + title = item.get("name", "Unknown") + return SeriesMetadata( + provider_id=str(item.get("id", fallback_provider_id)), + title=title, + sort_title=_make_sort_title(title), + year_start=_safe_int(item.get("start_year")), + year_end=None, + status=None, + publisher=publisher_name, + description=_strip_html(item.get("description") or item.get("deck")), + cover_url=cover_url, + issue_count=_safe_int(item.get("count_of_issues")), + comicvine_url=item.get("site_detail_url"), + ) + + def _issue_summary_from_item(item: dict[str, Any]) -> IssueSummary: image_data = item.get("image") cover_url = image_data.get("medium_url") if isinstance(image_data, dict) else None title = item.get("name") + issue_number, issue_number_text = _parse_issue_number_fields(item.get("issue_number")) return IssueSummary( provider_id=str(item["id"]), - issue_number=_parse_issue_number(item.get("issue_number")), + issue_number=issue_number, title=title, release_date=item.get("cover_date"), cover_url=cover_url, issue_type=detect_issue_type(str(title or "")), + issue_number_text=issue_number_text, + ) + + +def _issue_metadata_from_item( + item: dict[str, Any], + *, + fallback_provider_id: str, +) -> IssueMetadata: + volume_data = item.get("volume") + series_provider_id = str(volume_data["id"]) if isinstance(volume_data, dict) else "" + image_data = item.get("image") + cover_url = image_data.get("medium_url") if isinstance(image_data, dict) else None + issue_number, issue_number_text = _parse_issue_number_fields(item.get("issue_number")) + return IssueMetadata( + provider_id=str(item.get("id", fallback_provider_id)), + series_provider_id=series_provider_id, + issue_number=issue_number, + title=item.get("name"), + description=_strip_html(item.get("description") or item.get("deck")), + release_date=item.get("cover_date"), + store_date=item.get("store_date"), + cover_url=cover_url, + page_count=None, + comicvine_url=item.get("site_detail_url"), + creators=_extract_creators(item.get("person_credits")), + story_arcs=_extract_story_arcs(item.get("story_arc_credits")), + issue_number_text=issue_number_text, ) +def _normalize_bulk_provider_ids(provider_ids: Sequence[str]) -> list[str]: + if isinstance(provider_ids, (str, bytes)) or len(provider_ids) > _MAX_BULK_PROVIDER_IDS: + raise ValueError("ComicVine batch requests require at most 5000 provider IDs") + normalized: list[str] = [] + seen: set[str] = set() + for value in provider_ids: + provider_id = str(value).strip() + if re.fullmatch(r"[1-9][0-9]{0,18}", provider_id) is None: + raise ValueError("ComicVine provider IDs must be positive integers") + if provider_id not in seen: + seen.add(provider_id) + normalized.append(provider_id) + return normalized + + def _discard_global_series_search_inflight( key: _GlobalSeriesSearchInflightKey, task: asyncio.Future[tuple[tuple[SeriesSearchResult, ...], int]], @@ -262,15 +417,10 @@ def __init__( headers={"User-Agent": "Pullbox/1.0"}, ) normalized_rate_limit = max(int(rate_limit), 1) - normalized_burst_limit = ( - normalized_rate_limit - if burst_limit is None - else max(1, min(int(burst_limit), normalized_rate_limit)) - ) - self._rate_limiter = _TokenBucket( - max_tokens=normalized_burst_limit, - refill_rate=normalized_rate_limit / 3600.0, - ) + normalized_burst_limit = 1 if burst_limit is None else max(1, min(int(burst_limit), 10)) + self._rate_limit = normalized_rate_limit + self._requests_per_second = normalized_burst_limit + self._rate_coordinator: _ComicVineRateCoordinator | None = None @property def name(self) -> str: @@ -284,7 +434,16 @@ async def _request( params: dict[str, Any] | None = None, ) -> dict[str, Any]: """Make a rate-limited GET request to the ComicVine API.""" - await self._rate_limiter.acquire() + if self._rate_coordinator is None: + self._rate_coordinator = _rate_coordinator_for( + api_key=self._api_key, + rate_limit=self._rate_limit, + requests_per_second=self._requests_per_second, + ) + await self._rate_coordinator.acquire( + _resource_rate_key(endpoint), + requests_per_second=self._requests_per_second, + ) request_params: dict[str, Any] = { "api_key": self._api_key, @@ -344,6 +503,35 @@ async def _request( log.debug("comicvine_response_ok", total_results=data.get("number_of_total_results")) return data + # -- Optional StoryArcMetadataProvider capability ----------------------- + + async def search_story_arcs( + self, query: str, *, limit: int = 20, offset: int = 0 + ) -> list[StoryArcSearchResult]: + """Search a bounded page of Comic Vine story arcs by name.""" + results, _ = await self.search_story_arcs_page(query, limit=limit, offset=offset) + return results + + async def search_story_arcs_page( + self, query: str, *, limit: int = 20, offset: int = 0 + ) -> tuple[list[StoryArcSearchResult], int]: + """Return arc-name search results and the provider's arc result count.""" + from pullbox.providers.metadata.comicvine_story_arcs import search_story_arcs_page + + return await search_story_arcs_page(self._request, query, limit=limit, offset=offset) + + async def get_story_arc(self, provider_id: str) -> StoryArcMetadata: + """Read explicit membership without claiming a curated reading order.""" + from pullbox.providers.metadata.comicvine_story_arcs import get_story_arc + + return await get_story_arc(self._request, provider_id) + + async def get_story_arc_issues(self, issue_provider_ids: Sequence[str]) -> list[IssueMetadata]: + """Hydrate the exact unique issue set in requested order.""" + from pullbox.providers.metadata.comicvine_story_arcs import get_story_arc_issues + + return await get_story_arc_issues(self._request, issue_provider_ids) + # -- MetadataProvider implementation ------------------------------------ async def search_series( @@ -586,27 +774,34 @@ async def get_series(self, provider_id: str) -> SeriesMetadata: data = await self._request(f"/volume/{_VOLUME_PREFIX}-{provider_id}/", params) item: dict[str, Any] = data.get("results", {}) - publisher_data = item.get("publisher") - publisher_name = publisher_data.get("name") if isinstance(publisher_data, dict) else None - - image_data = item.get("image") - cover_url = image_data.get("medium_url") if isinstance(image_data, dict) else None - - title = item.get("name", "Unknown") - - return SeriesMetadata( - provider_id=str(item.get("id", provider_id)), - title=title, - sort_title=_make_sort_title(title), - year_start=_safe_int(item.get("start_year")), - year_end=None, # ComicVine doesn't provide end year directly - status=None, # ComicVine volumes lack an explicit status field - publisher=publisher_name, - description=_strip_html(item.get("description") or item.get("deck")), - cover_url=cover_url, - issue_count=_safe_int(item.get("count_of_issues")), - comicvine_url=item.get("site_detail_url"), + return _series_metadata_from_item(item, fallback_provider_id=provider_id) + + async def get_series_batch(self, provider_ids: Sequence[str]) -> dict[str, SeriesMetadata]: + """Fetch volume profiles in bounded ID-filter batches.""" + normalized = _normalize_bulk_provider_ids(provider_ids) + found: dict[str, SeriesMetadata] = {} + field_list = ( + "id,name,start_year,count_of_issues,publisher,image,description,deck,site_detail_url" ) + for start in range(0, len(normalized), _BULK_RESOURCE_PAGE_SIZE): + batch = normalized[start : start + _BULK_RESOURCE_PAGE_SIZE] + data = await self._request( + "/volumes/", + { + "filter": f"id:{'|'.join(batch)}", + "field_list": field_list, + "limit": _BULK_RESOURCE_PAGE_SIZE, + "offset": 0, + }, + ) + for raw_item in data.get("results", []): + item = raw_item if isinstance(raw_item, dict) else {} + metadata = _series_metadata_from_item(item, fallback_provider_id="") + if metadata.provider_id in batch: + found[metadata.provider_id] = metadata + return { + provider_id: found[provider_id] for provider_id in normalized if provider_id in found + } async def get_issue(self, provider_id: str) -> IssueMetadata: """Get full issue metadata by ComicVine issue ID.""" @@ -624,26 +819,35 @@ async def get_issue(self, provider_id: str) -> IssueMetadata: data = await self._request(f"/issue/{_ISSUE_PREFIX}-{provider_id}/", params) item: dict[str, Any] = data.get("results", {}) - volume_data = item.get("volume") - series_provider_id = str(volume_data["id"]) if isinstance(volume_data, dict) else "" + return _issue_metadata_from_item(item, fallback_provider_id=provider_id) - image_data = item.get("image") - cover_url = image_data.get("medium_url") if isinstance(image_data, dict) else None - - return IssueMetadata( - provider_id=str(item.get("id", provider_id)), - series_provider_id=series_provider_id, - issue_number=_parse_issue_number(item.get("issue_number")), - title=item.get("name"), - description=_strip_html(item.get("description") or item.get("deck")), - release_date=item.get("cover_date"), - store_date=item.get("store_date"), - cover_url=cover_url, - page_count=None, # Not available from ComicVine - comicvine_url=item.get("site_detail_url"), - creators=_extract_creators(item.get("person_credits")), - story_arcs=_extract_story_arcs(item.get("story_arc_credits")), + async def get_issue_batch(self, provider_ids: Sequence[str]) -> dict[str, IssueMetadata]: + """Fetch full issue metadata in bounded ID-filter batches.""" + normalized = _normalize_bulk_provider_ids(provider_ids) + found: dict[str, IssueMetadata] = {} + field_list = ( + "id,volume,issue_number,name,description,deck,cover_date,store_date," + "image,site_detail_url,person_credits,story_arc_credits" ) + for start in range(0, len(normalized), _BULK_RESOURCE_PAGE_SIZE): + batch = normalized[start : start + _BULK_RESOURCE_PAGE_SIZE] + data = await self._request( + "/issues/", + { + "filter": f"id:{'|'.join(batch)}", + "field_list": field_list, + "limit": _BULK_RESOURCE_PAGE_SIZE, + "offset": 0, + }, + ) + for raw_item in data.get("results", []): + item = raw_item if isinstance(raw_item, dict) else {} + metadata = _issue_metadata_from_item(item, fallback_provider_id="") + if metadata.provider_id in batch: + found[metadata.provider_id] = metadata + return { + provider_id: found[provider_id] for provider_id in normalized if provider_id in found + } async def get_issues_for_series(self, series_provider_id: str) -> list[IssueSummary]: """Get all issues for a series (volume), paginating automatically.""" @@ -680,6 +884,40 @@ async def get_issues_for_series(self, series_provider_id: str) -> list[IssueSumm log.debug("comicvine_issues_fetched", count=len(all_issues)) return all_issues + async def get_issue_catalog_batch( + self, + series_provider_ids: Sequence[str], + ) -> dict[str, list[IssueSummary]]: + """Fetch and group issue summaries for multiple volumes.""" + normalized = _normalize_bulk_provider_ids(series_provider_ids) + grouped: dict[str, list[IssueSummary]] = {provider_id: [] for provider_id in normalized} + for start in range(0, len(normalized), _BULK_RESOURCE_PAGE_SIZE): + batch = normalized[start : start + _BULK_RESOURCE_PAGE_SIZE] + offset = 0 + while True: + data = await self._request( + "/issues/", + { + "filter": f"volume:{'|'.join(batch)}", + "field_list": "id,volume,issue_number,name,cover_date,image", + "sort": "issue_number:asc", + "limit": _BULK_RESOURCE_PAGE_SIZE, + "offset": offset, + }, + ) + items = data.get("results", []) + for raw_item in items: + item = raw_item if isinstance(raw_item, dict) else {} + volume = item.get("volume") + volume_id = str(volume.get("id")) if isinstance(volume, dict) else "" + if volume_id in grouped: + grouped[volume_id].append(_issue_summary_from_item(item)) + total = _safe_int(data.get("number_of_total_results")) or 0 + offset += len(items) + if not items or offset >= total: + break + return grouped + async def get_recent_issues_for_series( self, series_provider_id: str, diff --git a/src/pullbox/providers/metadata/comicvine_story_arcs.py b/src/pullbox/providers/metadata/comicvine_story_arcs.py new file mode 100644 index 00000000..802cc478 --- /dev/null +++ b/src/pullbox/providers/metadata/comicvine_story_arcs.py @@ -0,0 +1,275 @@ +"""Strict, bounded Comic Vine story-arc requests using shared request plumbing. + +The live detail endpoint returns its entire nested issues array, including more +than 100 members; offset/limit paginate neither that array nor its reading order. +The envelope total counts arcs. The misspelled issue counter is often a stale +zero, so neither value may replace the explicit membership array. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any + +from pullbox.core.issue_numbers import parse_issue_number_text +from pullbox.providers.base import IssueMetadata +from pullbox.providers.metadata.comicvine import ( + ComicVineError, + _comicvine_name_filter, + _strip_html, +) +from pullbox.providers.story_arcs import ( + MAX_STORY_ARC_MEMBERS, + StoryArcMetadata, + StoryArcSearchResult, +) + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Sequence + + _Request = Callable[[str, dict[str, Any] | None], Awaitable[dict[str, Any]]] + +_PAGE_SIZE = 100 +_MAX_SEARCH_OFFSET = 10_000 +_MAX_HYDRATION_PAGES_PER_BATCH = 10 +_ARC_FIELDS = "id,name,description,deck,publisher,image,site_detail_url,count_of_isssue_appearances" +_ISSUE_FIELDS = ( + "id,volume,issue_number,name,description,deck,cover_date,store_date,image,site_detail_url" +) + + +def _incompatible() -> ComicVineError: + """Never include provider response contents, credentials, or URLs in errors.""" + return ComicVineError(0, "Comic Vine returned an incompatible story-arc response") + + +def _canonical_id(value: object) -> str: + if type(value) is int: + value = str(value) + if ( + not isinstance(value, str) + or re.fullmatch(r"[1-9][0-9]{0,18}", value) is None + or int(value) > 2**63 - 1 + ): + raise ValueError("A positive canonical provider ID is required") + return value + + +def _provider_id(value: object) -> str: + try: + return _canonical_id(value) + except ValueError: + raise _incompatible() from None + + +def _object(value: object) -> dict[str, Any]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise _incompatible() + return value + + +def _text(value: object, *, required: bool = False) -> str | None: + if value is None and not required: + return None + if not isinstance(value, str) or (required and not value.strip()): + raise _incompatible() + return value + + +def _nested_text(item: dict[str, Any], key: str, child_key: str) -> str | None: + value = item.get(key) + return None if value is None else _text(_object(value).get(child_key)) + + +def _count(value: object) -> int: + if type(value) is not int or value < 0: + raise _incompatible() + return value + + +async def _request_json(request: _Request, endpoint: str, params: dict[str, Any]) -> dict[str, Any]: + try: + data = await request(endpoint, params) + except ComicVineError as exc: + # Keep retry/status semantics without reflecting an upstream error body. + raise ComicVineError( + exc.status_code, + "Comic Vine story-arc request failed", + retryable=exc.retryable, + ) from None + except (ValueError, TypeError, AttributeError): + # _request parses JSON and accesses the envelope before returning it. + raise _incompatible() from None + data = _object(data) + if "status_code" in data and (type(data["status_code"]) is not int or data["status_code"] != 1): + raise _incompatible() + return data + + +def _list_page(data: dict[str, Any], *, limit: int, offset: int) -> tuple[list[Any], int]: + items = data.get("results") + if not isinstance(items, list) or len(items) > limit: + raise _incompatible() + total = _count(data.get("number_of_total_results")) + if "number_of_page_results" in data and _count(data["number_of_page_results"]) != len(items): + raise _incompatible() + if items and offset + len(items) > total: + raise _incompatible() + if not items and offset < total: + raise _incompatible() + return items, total + + +def _arc_result(item: dict[str, Any]) -> StoryArcSearchResult: + title = _text(item.get("name"), required=True) + assert title is not None + raw_count = item.get("count_of_isssue_appearances") + count = None if raw_count is None else _count(raw_count) + return StoryArcSearchResult( + provider_id=_provider_id(item.get("id")), + title=title, + description=_strip_html(_text(item.get("description")) or _text(item.get("deck"))), + publisher=_nested_text(item, "publisher", "name"), + cover_url=_nested_text(item, "image", "medium_url"), + comicvine_url=_text(item.get("site_detail_url")), + declared_issue_count=count or None, + ) + + +async def search_story_arcs_page( + request: _Request, query: str, *, limit: int = 20, offset: int = 0 +) -> tuple[list[StoryArcSearchResult], int]: + """Search one explicit page; never use the unreliable generic search resource.""" + if type(limit) is not int or not 1 <= limit <= _PAGE_SIZE: + raise ValueError("Story-arc search limit must be from 1 to 100") + if type(offset) is not int or not 0 <= offset <= _MAX_SEARCH_OFFSET: + raise ValueError("Story-arc search offset must be from 0 to 10000") + if not isinstance(query, str) or len(query) > 500: + raise ValueError("Story-arc query must be text of at most 500 characters") + name_filter = _comicvine_name_filter(query) + if not name_filter: + return [], 0 + data = await _request_json( + request, + "/story_arcs/", + { + "filter": name_filter, + "field_list": _ARC_FIELDS, + "limit": limit, + "offset": offset, + }, + ) + items, total = _list_page(data, limit=limit, offset=offset) + results = [_arc_result(_object(item)) for item in items] + if len({item.provider_id for item in results}) != len(results): + raise _incompatible() + return results, total + + +async def get_story_arc(request: _Request, provider_id: str) -> StoryArcMetadata: + """Preserve the full observed member list and explicitly qualified order.""" + provider_id = _canonical_id(provider_id) + data = await _request_json( + request, f"/story_arc/4045-{provider_id}/", {"field_list": f"{_ARC_FIELDS},issues"} + ) + item = _object(data.get("results")) + result = _arc_result(item) + members = item.get("issues") + if ( + result.provider_id != provider_id + or not isinstance(members, list) + or len(members) > MAX_STORY_ARC_MEMBERS + ): + raise _incompatible() + ids = tuple(_provider_id(_object(member).get("id")) for member in members) + if len(set(ids)) != len(ids): + raise _incompatible() + complete = result.declared_issue_count is None or result.declared_issue_count == len(ids) + warnings: list[str] = [] + if item.get("count_of_isssue_appearances") == 0 and ids: + warnings.append("unreliable_zero_issue_count") + if not complete: + warnings.append("issue_count_mismatch") + return StoryArcMetadata( + provider_id=result.provider_id, + title=result.title, + issue_provider_ids=ids, + description=result.description, + publisher=result.publisher, + cover_url=result.cover_url, + comicvine_url=result.comicvine_url, + declared_issue_count=result.declared_issue_count, + membership_complete=complete, + warnings=tuple(warnings), + ) + + +def _issue_metadata(item: dict[str, Any]) -> IssueMetadata: + number = _text(item.get("issue_number"), required=True) + assert number is not None + try: + numeric_number, exact_number = parse_issue_number_text(number) + except ValueError: + raise _incompatible() from None + return IssueMetadata( + provider_id=_provider_id(item.get("id")), + series_provider_id=_provider_id(_object(item.get("volume")).get("id")), + issue_number=numeric_number, + issue_number_text=exact_number, + title=_text(item.get("name")), + description=_strip_html(_text(item.get("description")) or _text(item.get("deck"))), + release_date=_text(item.get("cover_date")), + store_date=_text(item.get("store_date")), + cover_url=_nested_text(item, "image", "medium_url"), + page_count=None, + comicvine_url=_text(item.get("site_detail_url")), + ) + + +async def _hydrate_batch(request: _Request, ids: tuple[str, ...]) -> list[IssueMetadata]: + expected_ids = set(ids) + found: dict[str, IssueMetadata] = {} + offset = 0 + for _ in range(_MAX_HYDRATION_PAGES_PER_BATCH): + data = await _request_json( + request, + "/issues/", + { + "filter": f"id:{'|'.join(ids)}", + "field_list": _ISSUE_FIELDS, + "limit": _PAGE_SIZE, + "offset": offset, + }, + ) + items, total = _list_page(data, limit=_PAGE_SIZE, offset=offset) + if total != len(ids): + raise _incompatible() + for raw_item in items: + issue = _issue_metadata(_object(raw_item)) + if issue.provider_id not in expected_ids or issue.provider_id in found: + raise _incompatible() + found[issue.provider_id] = issue + offset += len(items) + if offset == total: + if set(found) != expected_ids: + raise _incompatible() + return [found[provider_id] for provider_id in ids] + raise _incompatible() + + +async def get_story_arc_issues( + request: _Request, issue_provider_ids: Sequence[str] +) -> list[IssueMetadata]: + """Hydrate bounded ID batches atomically to the caller, not partial results.""" + if ( + isinstance(issue_provider_ids, (str, bytes)) + or len(issue_provider_ids) > MAX_STORY_ARC_MEMBERS + ): + raise ValueError("Story-arc hydration requires at most 5000 distinct issue IDs") + ids = tuple(_canonical_id(value) for value in issue_provider_ids) + if len(set(ids)) != len(ids): + raise ValueError("Story-arc hydration requires distinct issue IDs") + results: list[IssueMetadata] = [] + for start in range(0, len(ids), _PAGE_SIZE): + results.extend(await _hydrate_batch(request, ids[start : start + _PAGE_SIZE])) + return results diff --git a/src/pullbox/providers/story_arcs.py b/src/pullbox/providers/story_arcs.py new file mode 100644 index 00000000..a24ad971 --- /dev/null +++ b/src/pullbox/providers/story_arcs.py @@ -0,0 +1,73 @@ +"""Optional metadata capability for explicit provider story-arc membership. + +Membership completeness describes the received provider list, not bibliographic +completeness or a curated reading order. Consumers must hydrate every member ID +successfully before publishing a resolved arc. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, Protocol, runtime_checkable + +if TYPE_CHECKING: + from collections.abc import Sequence + + from pullbox.providers.base import IssueMetadata + +MAX_STORY_ARC_MEMBERS = 5_000 + + +@dataclass(frozen=True) +class StoryArcSearchResult: + """An arc search result; an unavailable or unreliable count remains unknown.""" + + provider_id: str + title: str + description: str | None = None + publisher: str | None = None + cover_url: str | None = None + comicvine_url: str | None = None + declared_issue_count: int | None = None + + +@dataclass(frozen=True) +class StoryArcMetadata: + """Provider membership in response order, never claimed to be curated order.""" + + provider_id: str + title: str + issue_provider_ids: tuple[str, ...] + description: str | None = None + publisher: str | None = None + cover_url: str | None = None + comicvine_url: str | None = None + declared_issue_count: int | None = None + order_basis: Literal["response_order"] = "response_order" + membership_complete: bool = False + warnings: tuple[str, ...] = () + + +@runtime_checkable +class StoryArcMetadataProvider(Protocol): + """Optional capability; ordinary MetadataProvider implementations need not implement it.""" + + async def search_story_arcs( + self, query: str, *, limit: int = 20, offset: int = 0 + ) -> list[StoryArcSearchResult]: + """Search one bounded page of provider arc names.""" + ... + + async def search_story_arcs_page( + self, query: str, *, limit: int = 20, offset: int = 0 + ) -> tuple[list[StoryArcSearchResult], int]: + """Return a page and the provider's total number of matching arcs.""" + ... + + async def get_story_arc(self, provider_id: str) -> StoryArcMetadata: + """Read an explicit member list without inventing a reading order.""" + ... + + async def get_story_arc_issues(self, issue_provider_ids: Sequence[str]) -> list[IssueMetadata]: + """Hydrate the exact unique member set, returning it in requested order.""" + ... diff --git a/src/pullbox/schemas/config.py b/src/pullbox/schemas/config.py index c17c7275..e56b2331 100644 --- a/src/pullbox/schemas/config.py +++ b/src/pullbox/schemas/config.py @@ -1,7 +1,11 @@ """Configuration request/response schemas.""" +from typing import Literal + from pydantic import BaseModel, ConfigDict, Field +from pullbox.schemas.import_job import FutureRootPolicyPayload + class ConfigResponse(BaseModel): """System configuration key-value pair.""" @@ -20,6 +24,174 @@ class ConfigUpdate(BaseModel): values: dict[str, str] = Field(..., min_length=1, description="Key-value pairs to update") +class NamingSettingsState(BaseModel): + """Effective naming values for global defaults or one library.""" + + library_root_id: int | None + fingerprint: str + policy: FutureRootPolicyPayload + use_global: bool + revision: int = 0 + source: str = "global_default" + + +class NamingSettingsUpdate(BaseModel): + """Save one complete scope without changing other media settings.""" + + model_config = ConfigDict(extra="forbid") + + library_root_id: int | None = Field(None, gt=0) + expected_fingerprint: str = Field(..., min_length=64, max_length=64) + policy: FutureRootPolicyPayload + use_global: bool = False + + +class NamingSettingsPreviewRequest(BaseModel): + """Preview the same complete policy that the scoped editor saves.""" + + model_config = ConfigDict(extra="forbid") + + policy: FutureRootPolicyPayload + + +class NamingSettingsPreview(BaseModel): + """Rich examples grouped by naming field.""" + + examples: dict[str, list[dict[str, str]]] + + +class LibraryRootCreate(BaseModel): + """Create an explicit persistent container-visible library root.""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(..., min_length=1, max_length=255) + path: str = Field(..., min_length=1, max_length=1000) + allow_referenced_registrations: bool = True + allow_managed_writes: bool = True + is_default_managed_destination: bool = False + + +class LibraryRootUpdate(BaseModel): + """Update mutable root metadata without rebinding its path.""" + + model_config = ConfigDict(extra="forbid") + + name: str | None = Field(None, min_length=1, max_length=255) + enabled: bool | None = None + allow_referenced_registrations: bool | None = None + allow_managed_writes: bool | None = None + is_default_managed_destination: bool | None = None + + +class LibraryRootState(BaseModel): + """Persisted root configuration plus a live capability snapshot.""" + + id: int + name: str + path: str + enabled: bool + allow_referenced_registrations: bool + allow_managed_writes: bool + is_default_managed_destination: bool + available: bool + readable: bool + writable: bool + free_bytes: int | None + status: Literal["ready", "read_only", "low_capacity", "unavailable"] + warnings: list[str] + can_disable: bool + + +class LibraryRootPreviewResponse(BaseModel): + """Non-persisting validation result for a proposed root.""" + + name: str + path: str + allow_referenced_registrations: bool + allow_managed_writes: bool + is_default_managed_destination: bool + available: bool + readable: bool + writable: bool + free_bytes: int | None + status: Literal["ready", "read_only", "low_capacity", "unavailable"] + warnings: list[str] + blocking_reasons: list[str] + can_create: bool + + +class LibraryRootRemovalPreview(BaseModel): + id: int + name: str + path: str + can_remove: bool + blocking_reasons: list[str] + dependencies: dict[str, int] + history_count: int + has_naming_policy: bool + preview_token: str | None + + +class LibraryRootRemovalConfirm(BaseModel): + model_config = ConfigDict(extra="forbid") + preview_token: str = Field(min_length=1, max_length=8192) + confirmation: Literal["REMOVE"] + + +class LibraryRootRebindPreviewRequest(BaseModel): + """Request a write-free preview for an established root path replacement.""" + + model_config = ConfigDict(extra="forbid") + + replacement_path: str = Field(..., min_length=1, max_length=1000) + + +class LibraryRootRebindImpact(BaseModel): + """Aggregate persisted associations affected by changing one root identity path.""" + + library_file_count: int = Field(..., ge=0) + series_count: int = Field(..., ge=0) + preferred_series_count: int = Field(..., ge=0) + story_arc_placement_count: int = Field(..., ge=0) + library_file_blocking_count: int = Field(..., ge=0) + series_blocking_count: int = Field(..., ge=0) + story_arc_placement_blocking_count: int = Field(..., ge=0) + affects_default_destination: bool + affects_preferred_series: bool + + +class LibraryRootRebindPreviewResponse(BaseModel): + """Signed, non-persisting root path rebind preview.""" + + library_root_id: int + root_name: str + current_path: str + replacement_path: str + available: bool + readable: bool + writable: bool + free_bytes: int | None + status: Literal["ready", "read_only", "low_capacity", "unavailable"] + warnings: list[str] + blocking_reasons: list[str] + same_physical_directory: bool + overlaps_current_path: bool + impact: LibraryRootRebindImpact + can_rebind: bool + preview_token: str | None + + +class LibraryRootRebindConfirmRequest(BaseModel): + """Explicit confirmation bound to one signed rebind preview.""" + + model_config = ConfigDict(extra="forbid") + + replacement_path: str = Field(..., min_length=1, max_length=1000) + preview_token: str = Field(..., min_length=1, max_length=4096) + confirmation: Literal["REBIND"] + + class NamingPreview(BaseModel): """Preview of file naming convention applied to sample data.""" @@ -40,3 +212,70 @@ class NamingPreviewGrouped(BaseModel): template: str = Field(description="The naming template string") template_type: str = Field(description="Template type: folder, standard, annual, non_standard") examples: list[NamingPreviewEntry] = Field(description="Preview examples") + + +class LibraryRootPolicyUpdate(BaseModel): + """Optimistic update for one library root's explicit naming policy.""" + + expected_revision: int = Field(..., ge=0) + policy: FutureRootPolicyPayload + + +class LibraryRootPolicyClear(BaseModel): + """Optimistic removal of one library root's explicit naming policy.""" + + expected_revision: int = Field(..., ge=0) + + +class LibraryRootPolicyPreviewExample(BaseModel): + """One bounded real-source example used for old/new policy comparison.""" + + publisher: str | None = Field(None, max_length=255) + series: str = Field(..., min_length=1, max_length=500) + year: int | None = Field(None, ge=1, le=9999) + issue_number: float + issue_title: str | None = Field(None, max_length=500) + + +class LibraryRootPolicyPreviewRequest(BaseModel): + """Unsaved root-policy proposal to render against representative metadata.""" + + policy: FutureRootPolicyPayload + examples: list[LibraryRootPolicyPreviewExample] = Field(default_factory=list, max_length=5) + + +class EffectiveLibraryRootPolicy(BaseModel): + """Complete effective naming policy returned for one root.""" + + schema_version: Literal[1] = 1 + series_path_template: str + series_folder_template: str + comic_file_template: str + annual_file_template: str + non_standard_file_template: str + single_non_standard_file_template: str + replace_illegal_characters: bool + colon_replacement: Literal["dash", "space", "empty", "smart"] + source: Literal["global_default", "import_adoption", "manual"] + source_import_job_id: int | None + + +class LibraryRootPolicyState(BaseModel): + """Effective scope and optimistic revision for a library root policy.""" + + library_root_id: int + library_root_name: str + scope: Literal["global_default", "root_override"] + policy_id: int | None + revision: int + effective_policy: EffectiveLibraryRootPolicy + + +class LibraryRootPolicyPreviewResponse(BaseModel): + """Current and proposed output examples without persisting a policy.""" + + current_scope: Literal["global_default", "root_override"] + current_series_paths: list[str] + proposed_series_paths: list[str] + current_file_names: list[str] + proposed_file_names: list[str] diff --git a/src/pullbox/schemas/import_completed_cleanup.py b/src/pullbox/schemas/import_completed_cleanup.py new file mode 100644 index 00000000..fa05a200 --- /dev/null +++ b/src/pullbox/schemas/import_completed_cleanup.py @@ -0,0 +1,71 @@ +"""Schemas for completed-import recovery cleanup.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field + +from pullbox.services.import_completed_cleanup import ( # noqa: TC001 - Pydantic enum + CompletedImportCleanupAction, +) + + +class CompletedImportCleanupPreviewRead(BaseModel): + """Bounded preview for one recovery action.""" + + job_id: int + action: CompletedImportCleanupAction + affected_count: int + affected_file_count: int + item_unit: str + examples: list[str] = Field(default_factory=list) + preview_token: str + confirmation_text: Literal["APPLY CLEANUP"] = "APPLY CLEANUP" + + +class CompletedImportCleanupApplyRequest(BaseModel): + """Signed cleanup confirmation.""" + + preview_token: str = Field(..., min_length=1) + confirmation: Literal["APPLY CLEANUP"] + + +class CompletedImportCleanupResultRead(BaseModel): + """Completed recovery mutation summary.""" + + job_id: int + action: CompletedImportCleanupAction + affected_count: int + affected_file_count: int + requires_import_retry: bool + + +class CleanLibraryImportPreviewRead(BaseModel): + """Exact referenced scope that will become a managed library.""" + + source_job_id: int + target_root_id: int + eligible_file_count: int + eligible_series_count: int + total_bytes: int + source_preserved: bool + preview_token: str + confirmation_text: Literal["BUILD CLEAN LIBRARY"] = "BUILD CLEAN LIBRARY" + + +class CleanLibraryImportCreateRequest(BaseModel): + """Signed confirmation for a clean managed-library build.""" + + target_root_id: int = Field(..., gt=0) + preview_token: str = Field(..., min_length=1) + confirmation: Literal["BUILD CLEAN LIBRARY"] + + +class CleanLibraryImportResultRead(BaseModel): + """Background managed-copy import created from a reference import.""" + + source_job_id: int + job_id: int + eligible_file_count: int + eligible_series_count: int diff --git a/src/pullbox/schemas/import_job.py b/src/pullbox/schemas/import_job.py index 1b371fe2..ec901709 100644 --- a/src/pullbox/schemas/import_job.py +++ b/src/pullbox/schemas/import_job.py @@ -1,21 +1,42 @@ """Import job request/response schemas.""" -from datetime import datetime +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - Pydantic needs this at runtime from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pullbox.core.mylar3_path_mapping import normalize_mylar3_path_map from pullbox.models.import_job import ( ImportControlRequest, ImportedFileStatus, + ImportFileHandlingMode, ImportJobStatus, ImportSeriesStatus, ImportSourceType, ) +from pullbox.schemas.import_layout import SourceLayoutSpecPayload +from pullbox.schemas.story_arc_placement import ( # noqa: TC001 - Pydantic resolves at runtime + StoryArcPlacementPolicyPayload, +) # ── Request Schemas ────────────────────────────────────────────────────── +class FutureRootPolicyPayload(BaseModel): + """Complete proposed naming policy for a later per-root implementation.""" + + schema_version: Literal[1] = 1 + series_path_template: str = Field(..., min_length=1, max_length=1024) + comic_file_template: str = Field(..., min_length=1, max_length=1024) + annual_file_template: str = Field(..., min_length=1, max_length=1024) + non_standard_file_template: str = Field(..., min_length=1, max_length=1024) + single_non_standard_file_template: str = Field(..., min_length=1, max_length=1024) + replace_illegal_characters: bool + colon_replacement: Literal["dash", "space", "empty", "smart"] + + class ImportJobCreate(BaseModel): """Start a new import job.""" @@ -38,7 +59,20 @@ class ImportJobCreate(BaseModel): ) mylar3_path_map: dict[str, str] = Field( default_factory=dict, - description="Docker volume path mapping {container_prefix: host_prefix}", + description="Mylar stored prefix to Pullbox-visible container prefix mapping", + ) + mylar3_path_map_confirmed: bool = Field( + False, + description="Whether Step 1 preview confirmation froze this exact mapping snapshot", + ) + mylar3_allow_unresolved_paths: bool = Field( + False, + description="Explicitly acknowledge unavailable sources retained as review exceptions", + ) + mylar3_unresolved_fingerprint: str | None = Field( + None, + pattern=r"^[a-f0-9]{64}$", + description="The reviewed unavailable-path set, revalidated when the scan starts", ) cv_match_threshold: float = Field( 0.70, ge=0.50, le=1.00, description="Minimum CV match score to accept" @@ -49,6 +83,62 @@ class ImportJobCreate(BaseModel): file_formats: str | None = Field( None, description="Comma-separated file extensions to scan (e.g. 'cbz, cbr, pdf')" ) + source_layout: SourceLayoutSpecPayload = Field( + default_factory=SourceLayoutSpecPayload, + description="Versioned interpretation of source folders and filenames", + ) + file_handling_mode: ImportFileHandlingMode = Field( + ImportFileHandlingMode.MANAGED_COPY, + description="Whether Pullbox creates a managed artifact or references the source", + ) + future_layout_requested: bool = Field( + False, + description="Whether the proposed layout should become the target root's future policy", + ) + future_root_policy: FutureRootPolicyPayload | None = Field( + None, + description="Complete proposed future policy when future_layout_requested is true", + ) + story_arc_import_requested: bool = Field( + False, + description=( + "Step 1 intent to review and import detected logical story arcs and memberships" + ), + ) + story_arc_materialization_requested: bool = Field( + False, + description=( + "Step 1 intent to review separate story-arc folder placements in addition to " + "logical memberships" + ), + ) + + @model_validator(mode="after") + def validate_future_policy_pair(self) -> ImportJobCreate: + """Require paired Step 1 intent fields to agree.""" + if self.future_layout_requested and self.future_root_policy is None: + raise ValueError("future_root_policy is required when future_layout_requested is true") + if not self.future_layout_requested and self.future_root_policy is not None: + raise ValueError("future_root_policy requires future_layout_requested to be true") + if self.story_arc_materialization_requested and not self.story_arc_import_requested: + raise ValueError( + "Story arc materialization requires story_arc_import_requested to be true" + ) + if self.mylar3_path_map and self.source_type != ImportSourceType.MYLAR3: + raise ValueError("Mylar path mapping is only supported for Mylar imports.") + if self.mylar3_path_map_confirmed and self.source_type != ImportSourceType.MYLAR3: + raise ValueError("Mylar path mapping confirmation is only supported for Mylar imports.") + if self.mylar3_allow_unresolved_paths and self.source_type != ImportSourceType.MYLAR3: + raise ValueError("Unresolved Mylar paths can only be acknowledged for Mylar imports.") + if self.source_type == ImportSourceType.MYLAR3 and not self.mylar3_path_map_confirmed: + raise ValueError("Review and confirm the Mylar path mapping before starting the scan.") + return self + + @field_validator("mylar3_path_map") + @classmethod + def validate_mylar3_path_map(cls, value: dict[str, str]) -> dict[str, str]: + """Normalize the frozen mapping with segment-aware overlap checks.""" + return normalize_mylar3_path_map(value) @field_validator("file_formats") @classmethod @@ -109,6 +199,90 @@ class ConflictResolution(BaseModel): chosen_file_id: int = Field(..., gt=0, description="ImportedFile ID to keep") +class StoryArcReviewDecision(BaseModel): + """Explicit Step 3 decision for one staged story arc.""" + + imported_story_arc_id: int = Field(..., gt=0, description="ImportedStoryArc ID") + action: Literal["select", "skip"] = Field(..., description="Import or skip this arc") + proposed_story_arc_id: int | None = Field( + None, + gt=0, + description="Existing StoryArc target when the staged arc should be merged", + ) + + @model_validator(mode="after") + def validate_skip_has_no_merge_target(self) -> StoryArcReviewDecision: + """A skipped staged arc cannot retain an active merge decision.""" + if self.action == "skip" and self.proposed_story_arc_id is not None: + raise ValueError("proposed_story_arc_id is only valid when action is select") + return self + + +class StoryArcReviewDecisionRequest(BaseModel): + """Update one staged story-arc decision from the Step 3 review UI.""" + + action: Literal["select", "skip"] = Field(..., description="Import or skip this arc") + proposed_story_arc_id: int | None = Field( + None, + gt=0, + description="Existing StoryArc target when the staged arc should be merged", + ) + + @model_validator(mode="after") + def validate_skip_has_no_merge_target(self) -> StoryArcReviewDecisionRequest: + """A skipped staged arc cannot retain an active merge decision.""" + if self.action == "skip" and self.proposed_story_arc_id is not None: + raise ValueError("proposed_story_arc_id is only valid when action is select") + return self + + +class StoryArcReviewDecisionResponse(BaseModel): + """Persisted review state for one staged story arc.""" + + imported_story_arc_id: int + status: str + selected_for_import: bool + proposed_story_arc_id: int | None + + +class StoryArcPolicyConfirmationRequest(BaseModel): + """Explicitly freeze one staged arc policy during Step 3 review.""" + + confirm_policy: Literal[True] + expected_policy_digest: str = Field(..., pattern=r"^[0-9a-f]{64}$") + materialize_filesystem: bool + monitored: bool + search_missing: bool + include_upcoming: bool + placement_policy: StoryArcPlacementPolicyPayload + + @model_validator(mode="after") + def validate_materialization_choice(self) -> StoryArcPolicyConfirmationRequest: + """Keep logical membership and filesystem materialization independent.""" + logical = self.placement_policy.mode.value == "logical" + if self.materialize_filesystem == logical: + raise ValueError( + "materialize_filesystem must be false for logical policy and true otherwise" + ) + if not self.monitored and (self.search_missing or self.include_upcoming): + raise ValueError("search_missing and include_upcoming require monitored to be true") + return self + + +class StoryArcPolicyConfirmationResponse(BaseModel): + """Sanitized persisted state for one confirmed staged policy.""" + + imported_story_arc_id: int + activation: Literal["confirmed"] + materialize_filesystem: bool + mode: str + monitored: bool + search_missing: bool + include_upcoming: bool + sync_enabled: bool + policy_digest: str + + class ConfirmImportRequest(BaseModel): """Confirm which series to import from a REVIEW-state job.""" @@ -150,6 +324,17 @@ class ConfirmImportRequest(BaseModel): conflict_resolutions: list[ConflictResolution] = Field( default_factory=list, description="Conflict group resolutions" ) + story_arc_ids: list[int] = Field( + default_factory=list, + description=( + "Compatibility list of staged story arcs to select. Durable per-arc " + "review decisions remain authoritative when already present." + ), + ) + story_arc_decisions: list[StoryArcReviewDecision] = Field( + default_factory=list, + description="Explicit select/skip and optional merge decisions for staged story arcs", + ) @field_validator("series_ids") @classmethod @@ -160,6 +345,26 @@ def validate_series_ids(cls, v: list[int]) -> list[int]: raise ValueError(msg) return list(dict.fromkeys(v)) + @field_validator("story_arc_ids") + @classmethod + def validate_story_arc_ids(cls, v: list[int]) -> list[int]: + """Keep story-arc identity separate from series selection.""" + if any(arc_id <= 0 for arc_id in v): + raise ValueError("All story_arc_ids must be positive integers") + return list(dict.fromkeys(v)) + + @field_validator("story_arc_decisions") + @classmethod + def validate_unique_story_arc_decisions( + cls, + v: list[StoryArcReviewDecision], + ) -> list[StoryArcReviewDecision]: + """Reject ambiguous duplicate decisions in one confirmation request.""" + ids = [decision.imported_story_arc_id for decision in v] + if len(ids) != len(set(ids)): + raise ValueError("story_arc_decisions must contain at most one decision per arc") + return v + class SeriesSearchOverride(BaseModel): """User manually sets a ComicVine ID for a series candidate.""" @@ -312,10 +517,13 @@ class ImportedFileRead(BaseModel): class FileConflictGroup(BaseModel): """A group of files that conflict on the same issue match.""" - conflict_group_id: int - matched_issue_id: int + kind: Literal["file_conflict", "series_conflict"] = "file_conflict" + conflict_group_id: int | str + matched_issue_id: int | None + series_id: int | None = None issue_title: str | None = None - files: list[ImportedFileRead] = Field(..., min_length=1) + diagnostics: dict[str, object] = Field(default_factory=dict) + files: list[ImportedFileRead] = Field(min_length=1) class ImportJobRead(BaseModel): @@ -348,15 +556,26 @@ class ImportJobRead(BaseModel): match_completed_at: datetime | None import_started_at: datetime | None import_completed_at: datetime | None + archived_at: datetime | None = None # Settings target_library_root_id: int | None + removed_library_root_snapshot: dict[str, str | int] | None = None monitored: bool search_on_add: bool move_to_library: bool = True transfer_method: str = "move" convert_to_preferred_format: bool = False update_embedded_comicinfo_from_match: bool = False + file_handling_mode: ImportFileHandlingMode = ImportFileHandlingMode.MANAGED_COPY + source_layout_snapshot: SourceLayoutSpecPayload = Field(default_factory=SourceLayoutSpecPayload) + future_layout_requested: bool = False + future_root_policy_snapshot: FutureRootPolicyPayload | None = None + future_root_policy_applied_at: datetime | None = None + story_arc_import_requested: bool = False + story_arc_materialization_requested: bool = False + mylar3_path_map: dict[str, str] = Field(default_factory=dict) + mylar3_path_map_confirmed: bool = False cv_match_threshold: float min_files_per_series: int file_formats: str | None @@ -411,6 +630,7 @@ class ImportJobListItem(BaseModel): series_failed: int created_at: datetime import_completed_at: datetime | None + archived_at: datetime | None = None class ImportPreviewResponse(BaseModel): @@ -476,6 +696,13 @@ class ImportProgressEvent(BaseModel): total_files_no_match: int | None = None total_files_imported: int | None = None total_files_failed: int | None = None + story_arc_placements_total: int | None = None + story_arc_placements_queued: int | None = None + story_arc_placements_running: int | None = None + story_arc_placements_retry_wait: int | None = None + story_arc_placements_failed: int | None = None + story_arc_placements_completed: int | None = None + story_arc_placements_cancelled: int | None = None review_summary: dict[str, int] | None = None control_state: dict[str, object] | None = None @@ -519,11 +746,13 @@ class ImportedFilesResponse(BaseModel): class ConflictGroupsResponse(BaseModel): - """All conflict groups for an import job.""" + """One bounded page of conflict groups for an import job.""" job_id: int groups: list[FileConflictGroup] total: int + page: int + page_size: int class FileSelectionUpdateRequest(BaseModel): @@ -743,6 +972,20 @@ class RetryFailedResponse(BaseModel): retrying_count: int +class ImportJobDeleteResponse(BaseModel): + """Accepted deletion whose cooperative rollback has not finished yet.""" + + status: Literal["rollback_pending"] = "rollback_pending" + message: str + + +class RetryStoryArcPlacementsResponse(BaseModel): + """Result of reopening failed/cancelled placement work for a stalled import.""" + + job_id: int + retrying_count: int + + class RetryImportResponse(BaseModel): """Result of creating a brand-new retry job from history.""" diff --git a/src/pullbox/schemas/import_job_archive.py b/src/pullbox/schemas/import_job_archive.py new file mode 100644 index 00000000..375f3a27 --- /dev/null +++ b/src/pullbox/schemas/import_job_archive.py @@ -0,0 +1,15 @@ +"""Schemas for non-destructive import history archival.""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - Pydantic resolves this at runtime + +from pydantic import BaseModel + + +class ImportJobArchiveResponse(BaseModel): + """Current archive state for one import job.""" + + job_id: int + archived: bool + archived_at: datetime | None diff --git a/src/pullbox/schemas/import_layout.py b/src/pullbox/schemas/import_layout.py new file mode 100644 index 00000000..4f02dd63 --- /dev/null +++ b/src/pullbox/schemas/import_layout.py @@ -0,0 +1,130 @@ +"""Request and response schemas for read-only import layout analysis.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from pullbox.core.filesystem_policy import resolve_preview_source +from pullbox.core.library_layout import ( + ImportLayoutMode, + LayoutClassification, + LayoutTemplateError, + SourceLayoutSpec, + resolve_source_layout_spec, +) +from pullbox.models.import_job import ImportSourceType + + +class SourceLayoutSpecPayload(BaseModel): + """Versioned source layout selection supplied by an API client.""" + + model_config = ConfigDict(from_attributes=True) + + schema_version: Literal[1] = 1 + mode: ImportLayoutMode = ImportLayoutMode.AUTO + preset: str | None = None + series_path_template: str | None = None + issue_filename_template: str | None = None + selected_cluster_id: str | None = None + fallback_to_auto: bool = True + + def to_core(self) -> SourceLayoutSpec: + """Return the validated internal DTO without API-layer dependencies.""" + return SourceLayoutSpec( + schema_version=self.schema_version, + mode=self.mode, + preset=self.preset, + series_path_template=self.series_path_template, + issue_filename_template=self.issue_filename_template, + selected_cluster_id=self.selected_cluster_id, + fallback_to_auto=self.fallback_to_auto, + ) + + @model_validator(mode="after") + def validate_layout_contract(self) -> SourceLayoutSpecPayload: + """Apply the same grammar validation used by the analyzer.""" + try: + effective = resolve_source_layout_spec(self.to_core()) + if effective.mode != ImportLayoutMode.AUTO: + from pullbox.core.library_layout import compile_source_layout + + compile_source_layout(effective) + except LayoutTemplateError as exc: + raise ValueError(str(exc)) from exc + return self + + +class LayoutPreviewRequest(BaseModel): + """Read-only source layout preflight request.""" + + source_path: str = Field(..., description="Existing filesystem directory to inspect") + source_type: ImportSourceType = Field(..., description="Import source type") + layout: SourceLayoutSpecPayload = Field(default_factory=SourceLayoutSpecPayload) + + @field_validator("source_path") + @classmethod + def validate_source_directory(cls, value: str) -> str: + """Resolve an existing directory without accepting a file as a scan root.""" + path = resolve_preview_source(value) + if not path.is_dir(): + raise ValueError("Layout preview source must be a directory") + return str(path) + + @model_validator(mode="after") + def validate_supported_source_type(self) -> LayoutPreviewRequest: + """Keep the first endpoint read-only and filesystem-specific.""" + if self.source_type != ImportSourceType.FILESYSTEM: + raise ValueError("Layout preview currently supports filesystem sources only") + return self + + +class LayoutExampleResponse(BaseModel): + """One bounded root-relative layout example.""" + + model_config = ConfigDict(from_attributes=True) + + relative_path: str + publisher: str | None + series: str | None + year: int | None + issue_number: str | None + issue_title: str | None + evidence: list[str] + warnings: list[str] + + +class LayoutClusterResponse(BaseModel): + """Bounded summary of one detected source layout cluster.""" + + model_config = ConfigDict(from_attributes=True) + + cluster_id: str + classification: LayoutClassification + file_count: int + directory_count: int + confidence: Literal["high", "medium", "low"] + proposed_series_path_template: str | None + proposed_issue_filename_template: str | None + examples: list[LayoutExampleResponse] + + +class LayoutAnalysisResponse(BaseModel): + """Serializable read-only layout analysis result.""" + + model_config = ConfigDict(from_attributes=True) + + effective_spec: SourceLayoutSpecPayload + classification: LayoutClassification + clusters: list[LayoutClusterResponse] + directories_considered: int + files_considered: int + files_fitting: int + files_ambiguous: int + files_outside_root: int + archive_probes: int + can_keep_in_place: bool + can_apply_future_policy: bool + partial: bool + warnings: list[str] diff --git a/src/pullbox/schemas/import_mylar3_path_preflight.py b/src/pullbox/schemas/import_mylar3_path_preflight.py new file mode 100644 index 00000000..6e761c4d --- /dev/null +++ b/src/pullbox/schemas/import_mylar3_path_preflight.py @@ -0,0 +1,205 @@ +"""Typed contracts for Mylar path-mapping analysis in Import Step 1.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from pullbox.core.filesystem_policy import resolve_preview_source +from pullbox.core.mylar3_path_mapping import ( + MAX_MYLAR3_PATH_MAPPINGS, + normalize_mylar3_path_mapping_items, +) +from pullbox.models.import_job import ImportFileHandlingMode, ImportSourceType + +MylarPathOutcome = Literal[ + "identity", + "mapped", + "mapped_missing", + "missing", + "unmapped", + "outside_root", + "unreadable", + "ambiguous", + "invalid", +] +MylarPathAttentionActionKind = Literal[ + "acknowledge_unavailable", + "register_reference_root", + "remove_ineffective_mapping", +] + + +class MylarPathMappingDraft(BaseModel): + """One editable Mylar-stored to Pullbox-visible mapping row.""" + + stored_prefix: str = Field(..., min_length=1, max_length=4096) + pullbox_prefix: str = Field(..., min_length=1, max_length=4096) + + +class MylarPathPreviewRequest(BaseModel): + """Read-only path-mapping preview request.""" + + source_path: str = Field( + ..., + description="Existing Mylar database or directory containing mylar.db", + ) + source_type: ImportSourceType = ImportSourceType.MYLAR3 + file_handling_mode: ImportFileHandlingMode = ImportFileHandlingMode.MANAGED_COPY + auto_detect: bool = True + mappings: list[MylarPathMappingDraft] = Field( + default_factory=list, + max_length=MAX_MYLAR3_PATH_MAPPINGS, + ) + + @field_validator("source_path") + @classmethod + def validate_source(cls, value: str) -> str: + """Resolve the selected source without allowing a database path race.""" + selected = resolve_preview_source(value) + database = selected / "mylar.db" if selected.is_dir() else selected + database = resolve_preview_source(database) + if not database.is_file(): + raise ValueError("Mylar path preview requires a mylar.db file") + return str(selected) + + @model_validator(mode="after") + def validate_mapping_drafts(self) -> MylarPathPreviewRequest: + """Reject malformed or conflicting editor rows before filesystem probing.""" + if self.source_type != ImportSourceType.MYLAR3: + raise ValueError("Mylar path preview only supports Mylar imports") + normalize_mylar3_path_mapping_items( + (mapping.stored_prefix, mapping.pullbox_prefix) for mapping in self.mappings + ) + return self + + +class MylarPathResolutionCounts(BaseModel): + """Complete aggregate resolution counts for inspected ComicLocation rows.""" + + model_config = ConfigDict(from_attributes=True) + + locations: int = Field(0, ge=0) + identity_resolved: int = Field(0, ge=0) + mapped_existing: int = Field(0, ge=0) + mapped_missing: int = Field(0, ge=0) + missing: int = Field(0, ge=0) + unmapped: int = Field(0, ge=0) + outside_root: int = Field(0, ge=0) + unreadable: int = Field(0, ge=0) + ambiguous: int = Field(0, ge=0) + invalid: int = Field(0, ge=0) + + +class MylarPathExample(BaseModel): + """One bounded path relative to its displayed mapping prefix.""" + + relative_path: str + outcome: MylarPathOutcome + + +class MylarPathException(BaseModel): + """An unresolved location, with enough context for an operator to repair it.""" + + series_id: str + series_name: str + stored_path: str + attempted_path: str + outcome: MylarPathOutcome + reason: str + suggested_action: str + + +class MylarPathProblemGroup(BaseModel): + """One root-level explanation for repeated path failures.""" + + root_path: str + outcome: MylarPathOutcome + series_count: int = Field(ge=0) + location_count: int = Field(ge=0) + reason: str + suggested_action: str + can_register_reference_root: bool = False + + +class MylarPathAttentionAction(BaseModel): + """One server-authorized, deterministic Step 1 resolution.""" + + kind: MylarPathAttentionActionKind + fingerprint: str = Field(min_length=64, max_length=64) + root_path: str | None = None + stored_prefix: str | None = None + pullbox_prefix: str | None = None + + +class MylarPathAttentionDetails(BaseModel): + """Plain-language evidence and manual next steps for one grouped finding.""" + + title: str + series_count: int = Field(ge=0) + location_count: int = Field(ge=0) + known_paths: list[str] = Field(default_factory=list, max_length=6) + steps: list[str] = Field(default_factory=list, min_length=1, max_length=6) + + +class MylarPathAttentionItem(BaseModel): + """One grouped issue that Step 1 can resolve or explain.""" + + key: str = Field(min_length=64, max_length=64) + code: str + blocks_import: bool + reason: str + suggested_action: str + root_path: str | None = None + action: MylarPathAttentionAction | None = None + details: MylarPathAttentionDetails + + +class MylarIdentityGroupPreview(BaseModel): + """Identity-resolved paths grouped under one enabled root.""" + + stored_prefix: str + library_root_id: int + library_root_name: str + resolution: MylarPathResolutionCounts + examples: list[MylarPathExample] = Field(default_factory=list) + + +class MylarPathMappingPreview(BaseModel): + """One normalized mapping row with complete resolution evidence.""" + + stored_prefix: str + pullbox_prefix: str + library_root_id: int | None = None + library_root_name: str | None = None + provenance: Literal["automatic", "manual"] + status: Literal["ready", "review", "blocked"] + resolution: MylarPathResolutionCounts + examples: list[MylarPathExample] = Field(default_factory=list) + warnings: list[str] = Field(default_factory=list) + blocking_reasons: list[str] = Field(default_factory=list) + + +class MylarPathPreviewResponse(BaseModel): + """Safe, bounded Step 1 mapping evidence and frozen compatibility map.""" + + source_type: ImportSourceType = ImportSourceType.MYLAR3 + resolution: MylarPathResolutionCounts + identity_groups: list[MylarIdentityGroupPreview] = Field(default_factory=list) + mappings: list[MylarPathMappingPreview] = Field(default_factory=list) + path_map: dict[str, str] = Field(default_factory=dict) + requires_confirmation: bool + can_confirm: bool + can_continue_with_unresolved: bool = False + requires_unresolved_acknowledgement: bool = False + unresolved_fingerprint: str | None = None + exception_count: int = 0 + exceptions: list[MylarPathException] = Field(default_factory=list) + problem_groups: list[MylarPathProblemGroup] = Field(default_factory=list) + attention_items: list[MylarPathAttentionItem] = Field(default_factory=list) + attention_fingerprint: str | None = None + report_id: str | None = None + blocking_reasons: list[str] = Field(default_factory=list) + partial: bool = False + warnings: list[str] = Field(default_factory=list) diff --git a/src/pullbox/schemas/import_safety_bulk.py b/src/pullbox/schemas/import_safety_bulk.py new file mode 100644 index 00000000..6d6410d6 --- /dev/null +++ b/src/pullbox/schemas/import_safety_bulk.py @@ -0,0 +1,49 @@ +"""API schemas for category-scoped import safety review.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from pullbox.models.import_job import ImportSourceType # noqa: TC001 +from pullbox.services.import_safety_diagnostics import ImportSafetyCategory # noqa: TC001 + + +class ImportSafetyBulkAllowRequest(BaseModel): + """Explicit confirmation of a signed category preview.""" + + preview_token: str = Field(min_length=1, max_length=4096) + confirmation: Literal["ALLOW ONCE"] + + +class ImportSafetyBulkPreviewRead(BaseModel): + """Sanitized bounded preview of one job/category scope.""" + + model_config = ConfigDict(from_attributes=True) + + job_id: int + source_type: ImportSourceType + category: ImportSafetyCategory + action: Literal["allow_once"] = "allow_once" + matching_count: int + affected_count: int + skipped_count: int + examples: list[str] + overrideable: bool + requires_confirmation: bool = False + confirmation_text: Literal["ALLOW ONCE"] | None = None + preview_token: str | None + + +class ImportSafetyBulkResultRead(BaseModel): + """Sanitized outcome counts for one category-scoped action.""" + + model_config = ConfigDict(from_attributes=True) + + job_id: int + source_type: ImportSourceType + category: ImportSafetyCategory + action: Literal["allow_once"] = "allow_once" + affected_count: int + skipped_count: int diff --git a/src/pullbox/schemas/import_story_arc_preflight.py b/src/pullbox/schemas/import_story_arc_preflight.py new file mode 100644 index 00000000..eeee9282 --- /dev/null +++ b/src/pullbox/schemas/import_story_arc_preflight.py @@ -0,0 +1,106 @@ +"""Typed request and response contracts for Story Arc Step 1 analysis.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from pullbox.core.filesystem_policy import resolve_preview_source +from pullbox.models.import_job import ImportSourceType + + +class StoryArcPreflightRequest(BaseModel): + """Read-only source request made before an import job exists.""" + + source_path: str = Field(..., description="Existing folder or Mylar database to inspect") + source_type: ImportSourceType + + @model_validator(mode="after") + def validate_source(self) -> StoryArcPreflightRequest: + """Resolve a source without broadening the import source contract.""" + path = resolve_preview_source(self.source_path) + if self.source_type is ImportSourceType.FILESYSTEM and not path.is_dir(): + raise ValueError("Filesystem Story Arc analysis requires a directory") + if self.source_type is ImportSourceType.MYLAR3: + database = path / "mylar.db" if path.is_dir() else path + resolve_preview_source(database) + if not database.is_file(): + raise ValueError("Mylar Story Arc analysis requires a mylar.db file") + self.source_path = str(path) + return self + + +class StoryArcResolutionPreview(BaseModel): + """Pre-match counts; unresolved states remain explicit rather than inferred.""" + + model_config = ConfigDict(from_attributes=True) + + resolved: int = Field(0, ge=0) + pending: int = Field(0, ge=0) + missing: int = Field(0, ge=0) + ambiguous: int = Field(0, ge=0) + conflicts: int = Field(0, ge=0) + duplicates: int = Field(0, ge=0) + + +class StoryArcSettingPreview(BaseModel): + """One allowlisted, path-safe Mylar setting shown in Step 1.""" + + model_config = ConfigDict(from_attributes=True) + + key: str + value: bool | str | None + used_default: bool + + +class StoryArcEvidenceExample(BaseModel): + """One bounded, sanitized source-relative Story Arc example.""" + + model_config = ConfigDict(from_attributes=True) + + story_arc: str | None + series: str | None + issue_number: str | None + issue_title: str | None + reading_order: str | None + status: str | None + relative_path: str | None = None + + +class StoryArcPolicyPreview(BaseModel): + """Path-safe summary of a policy that still requires Step 3 confirmation.""" + + model_config = ConfigDict(from_attributes=True) + + mode: str + destination_root_configured: bool + folder_template: str + file_template: str + reading_order_prefix: bool + synchronize: bool + requires_confirmation: bool = True + + +class StoryArcPreflightResponse(BaseModel): + """Bounded source evidence for conditional Story Arc controls in Step 1.""" + + model_config = ConfigDict(from_attributes=True) + + source_type: ImportSourceType + evidence_detected: bool + arcs_detected: int = Field(0, ge=0) + entries_detected: int = Field(0, ge=0) + resolution: StoryArcResolutionPreview = Field(default_factory=StoryArcResolutionPreview) + existing_arc_files_detected: bool = False + existing_arc_folders_detected: bool = False + pattern_summary: str | None = None + settings: list[StoryArcSettingPreview] = Field(default_factory=list) + examples: list[StoryArcEvidenceExample] = Field(default_factory=list) + provider_calls_required: bool + provider_call_summary: str + proposed_policy: StoryArcPolicyPreview + readlist_present: bool = False + readlist_count: int = Field(0, ge=0) + readlist_import_state: str | None = None + archive_probes: int = Field(0, ge=0) + partial: bool = False + warnings: list[str] = Field(default_factory=list) diff --git a/src/pullbox/schemas/issue.py b/src/pullbox/schemas/issue.py index e68455be..1cf8a864 100644 --- a/src/pullbox/schemas/issue.py +++ b/src/pullbox/schemas/issue.py @@ -23,6 +23,7 @@ class IssueResponse(BaseModel): comicvine_id: int | None = None series_id: int issue_number: float + issue_number_text: str title: str | None = None description: str | None = None release_date: date | None = None @@ -109,6 +110,7 @@ class IssueListResponse(BaseModel): id: int series_id: int issue_number: float + issue_number_text: str title: str | None = None release_date: date | None = None status: IssueStatus diff --git a/src/pullbox/schemas/library.py b/src/pullbox/schemas/library.py index b2de6b98..36ab78d9 100644 --- a/src/pullbox/schemas/library.py +++ b/src/pullbox/schemas/library.py @@ -80,6 +80,8 @@ class LibraryBrowserDeleteContext(BaseModel): linked_file_count: int = 0 tracked_file_count: int = 0 tracked_series_count: int = 0 + managed_file_count: int = 0 + referenced_file_count: int = 0 has_linked_issue: bool = False issue_status_after_delete: str | None = None issue_status_reason: str | None = None diff --git a/src/pullbox/schemas/series.py b/src/pullbox/schemas/series.py index 2cb28174..c2196aa8 100644 --- a/src/pullbox/schemas/series.py +++ b/src/pullbox/schemas/series.py @@ -67,6 +67,10 @@ class SeriesResponse(BaseModel): publisher_id: int | None = None path: str | None = Field(None, description="Absolute path to series folder") library_root_id: int | None = Field(None, description="Library root this series belongs to") + preferred_library_root_id: int | None = Field( + None, + description="Managed library root selected for future acquisitions", + ) alternate_names: list[str] = Field(default_factory=list, description="Alternate series names") publisher_name: str | None = Field( None, description="Publisher name, resolved from relationship" @@ -111,6 +115,8 @@ class SeriesDeleteContextResponse(BaseModel): series_count: int = Field(..., description="Number of resolved series in the request") linked_file_count: int = Field(..., ge=0, description="Linked files that still exist on disk") + managed_file_count: int = Field(..., ge=0, description="Pullbox-owned files in scope") + referenced_file_count: int = Field(..., ge=0, description="Referenced files detached only") class SeriesListResponse(BaseModel): @@ -134,6 +140,7 @@ class SeriesListResponse(BaseModel): publisher_name: str | None = None path: str | None = None library_root_id: int | None = None + preferred_library_root_id: int | None = None owned_count: int = Field(0, description="Issues in library") wanted_count: int = Field(0, description="Issues actively wanted") cover_path: str | None = None diff --git a/src/pullbox/schemas/story_arc.py b/src/pullbox/schemas/story_arc.py new file mode 100644 index 00000000..4cb7412c --- /dev/null +++ b/src/pullbox/schemas/story_arc.py @@ -0,0 +1,158 @@ +"""Request and response schemas for first-class logical story arcs.""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - Pydantic resolves this at runtime +from typing import Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from pullbox.models.story_arc import ( # noqa: TC001 - Pydantic resolves these at runtime + StoryArcLifecycle, + StoryArcResolutionState, + StoryArcSourceKind, +) + + +class StoryArcCreate(BaseModel): + """Create a Pullbox-owned logical story arc.""" + + name: str = Field(..., min_length=1, max_length=500) + description: str | None = None + monitored: bool = False + search_missing: bool = False + include_upcoming: bool = False + sync_enabled: bool = False + + +class StoryArcUpdate(BaseModel): + """Patch mutable arc metadata with optimistic revision protection.""" + + expected_revision: int = Field(..., ge=1) + name: str | None = Field(None, min_length=1, max_length=500) + description: str | None = None + monitored: bool | None = None + search_missing: bool | None = None + include_upcoming: bool | None = None + sync_enabled: bool | None = None + + @model_validator(mode="after") + def require_mutation(self) -> Self: + """Reject revision-only patches that would advance state without a change.""" + if self.model_fields_set <= {"expected_revision"}: + raise ValueError("At least one story-arc field must be provided") + if "name" in self.model_fields_set and self.name is None: + raise ValueError("Story-arc name cannot be null") + if "monitored" in self.model_fields_set and self.monitored is None: + raise ValueError("Story-arc monitored cannot be null") + if "search_missing" in self.model_fields_set and self.search_missing is None: + raise ValueError("Story-arc search_missing cannot be null") + if "include_upcoming" in self.model_fields_set and self.include_upcoming is None: + raise ValueError("Story-arc include_upcoming cannot be null") + if "sync_enabled" in self.model_fields_set and self.sync_enabled is None: + raise ValueError("Story-arc sync_enabled cannot be null") + return self + + +class StoryArcResponse(BaseModel): + """Logical story-arc metadata plus aggregate membership counts.""" + + model_config = ConfigDict(from_attributes=True) + + id: int + name: str + normalized_name: str + description: str | None = None + comicvine_id: int | None = None + comicvine_url: str | None = None + cover_path: str | None = None + cover_url: str | None = None + source_kind: StoryArcSourceKind + lifecycle: StoryArcLifecycle + monitored: bool + search_missing: bool + include_upcoming: bool + sync_enabled: bool + target_library_root_id: int | None = None + revision: int + membership_count: int = 0 + resolved_count: int = 0 + missing_count: int = 0 + conflict_count: int = 0 + created_at: datetime + updated_at: datetime + + +class StoryArcMembershipCreate(BaseModel): + """Add a resolved or unresolved ordered entry to an arc.""" + + issue_id: int | None = Field(None, gt=0) + sequence_number: int = Field(..., ge=0) + source_ordinal: int = Field(0, ge=0) + source_issue_number_text: str | None = Field(None, min_length=1, max_length=320) + + +class StoryArcMembershipUpdate(BaseModel): + """Patch one membership's order, exact number, or skip state.""" + + sequence_number: int | None = Field(None, ge=0) + source_ordinal: int | None = Field(None, ge=0) + source_issue_number_text: str | None = Field(None, min_length=1, max_length=320) + intentionally_skipped: bool | None = None + + @model_validator(mode="after") + def require_mutation(self) -> Self: + """Reject empty membership patches.""" + if not self.model_fields_set: + raise ValueError("At least one membership field must be provided") + if "sequence_number" in self.model_fields_set and self.sequence_number is None: + raise ValueError("Membership sequence_number cannot be null") + if "source_ordinal" in self.model_fields_set and self.source_ordinal is None: + raise ValueError("Membership source_ordinal cannot be null") + if ( + "source_issue_number_text" in self.model_fields_set + and self.source_issue_number_text is None + ): + raise ValueError("Membership source_issue_number_text cannot be null") + if "intentionally_skipped" in self.model_fields_set and self.intentionally_skipped is None: + raise ValueError("Membership intentionally_skipped cannot be null") + return self + + +class StoryArcMembershipResolve(BaseModel): + """Resolve an entry to an existing canonical issue.""" + + issue_id: int = Field(..., gt=0) + + +class StoryArcMembershipReorder(BaseModel): + """Provide the complete ordered membership identity set for an arc.""" + + expected_revision: int = Field(..., ge=1) + membership_ids: list[int] = Field(..., max_length=50_000) + + +class StoryArcMembershipResponse(BaseModel): + """One durable ordered story-arc membership.""" + + model_config = ConfigDict(from_attributes=True) + + id: int + story_arc_id: int + issue_id: int | None = None + sequence_number: int + source_ordinal: int + resolution_state: StoryArcResolutionState + source_kind: StoryArcSourceKind + source_issue_number_text: str | None = None + source_series_name: str | None = None + source_issue_title: str | None = None + source_publisher: str | None = None + sync_eligible: bool + + +class StoryArcMembershipOrderResponse(BaseModel): + """A completed membership reorder and the arc's new revision.""" + + items: list[StoryArcMembershipResponse] + revision: int diff --git a/src/pullbox/schemas/story_arc_placement.py b/src/pullbox/schemas/story_arc_placement.py new file mode 100644 index 00000000..f9fd4b87 --- /dev/null +++ b/src/pullbox/schemas/story_arc_placement.py @@ -0,0 +1,238 @@ +"""REST schemas for story-arc placement policy, preview, and synchronization.""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 - Pydantic resolves this at runtime +from typing import Self + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from pullbox.models.story_arc import ( # noqa: TC001 - used by Pydantic at runtime + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcPlacementState, + StoryArcSymlinkStyle, +) +from pullbox.services.story_arc_placement_integration import ( + StoryArcPlacementPolicyMode, +) +from pullbox.services.story_arc_policy_migration import ( + STORY_ARC_POLICY_MIGRATION_CONFIRMATION, +) + + +class StoryArcPlacementPolicyPayload(BaseModel): + """Complete candidate policy; omission never inherits hidden global state.""" + + mode: StoryArcPlacementPolicyMode + target_library_root_id: int | None = Field(None, gt=0) + destination_root: str | None = Field(None, max_length=1000) + folder_template: str = Field(..., min_length=1, max_length=1024) + file_template: str = Field(..., min_length=1, max_length=1024) + symlink_style: StoryArcSymlinkStyle | None = None + synchronize: bool + + @model_validator(mode="after") + def validate_mode_shape(self) -> Self: + """Reject contradictory roots/styles before reaching filesystem code.""" + if self.mode is StoryArcPlacementPolicyMode.LOGICAL: + if self.target_library_root_id is not None or self.destination_root is not None: + raise ValueError("Logical-only policy must not configure a placement root") + elif self.target_library_root_id is None or self.destination_root is None: + raise ValueError("Non-logical policy requires a library and destination root") + if self.mode is StoryArcPlacementPolicyMode.SYMLINK: + if self.symlink_style is None: + raise ValueError("Symlink policy requires a symlink style") + elif self.symlink_style is not None: + raise ValueError("Only symlink policy may specify a symlink style") + return self + + +class StoryArcPlacementPolicyUpdate(StoryArcPlacementPolicyPayload): + """Optimistically replace one complete per-arc policy.""" + + expected_revision: int = Field(..., ge=1) + + +class StoryArcPlacementPolicyResponse(BaseModel): + """Effective complete policy, including whether it has been frozen.""" + + configured: bool + revision: int + mode: StoryArcPlacementPolicyMode + target_library_root_id: int | None + destination_root: str | None + folder_template: str + file_template: str + symlink_style: StoryArcSymlinkStyle | None + synchronize: bool + snapshot: dict[str, object] + + +class StoryArcPlacementPreviewItemResponse(BaseModel): + """Read-only placement plan for one ordered membership.""" + + model_config = ConfigDict(from_attributes=True) + + membership_id: int + sequence_number: int + issue_id: int | None + issue_number_text: str + mode: str + state: str + target_path: str | None + collision: str + reason: str | None + required_bytes: int + proposed_ownership: str + overwrite_allowed: bool + classification: str + placement_id: int | None + current_ownership: StoryArcPlacementOwnership | None + inspection_code: str | None + + +class StoryArcPlacementPreviewPageResponse(BaseModel): + """Policy plus one bounded deterministic preview page.""" + + policy: StoryArcPlacementPolicyResponse + items: list[StoryArcPlacementPreviewItemResponse] + total: int + limit: int + offset: int + has_more: bool + + +class StoryArcPlacementResponse(BaseModel): + """Durable placement ownership, fingerprint, and synchronization state.""" + + model_config = ConfigDict(from_attributes=True) + + id: int + issue_story_arc_id: int + library_file_id: int | None + library_root_id: int | None + placement_path: str + mode: StoryArcPlacementMode + ownership: StoryArcPlacementOwnership + symlink_style: StoryArcSymlinkStyle | None + rendered_reading_order: int | None + policy_schema_version: int | None + source_fingerprint: dict[str, object] + target_fingerprint: dict[str, object] + state: StoryArcPlacementState + last_result: dict[str, object] + last_checked_at: datetime | None + + +class StoryArcPlacementSyncRequest(BaseModel): + """Explicitly approve adoption of an identical user artifact when desired.""" + + adopt_identical_existing: bool = False + + +class StoryArcPlacementSyncResponse(BaseModel): + """One completed logical, referenced, created, repaired, or idempotent sync.""" + + membership_id: int + outcome: str + placement: StoryArcPlacementResponse | None + + +class StoryArcPlacementRemovalResponse(BaseModel): + """Ownership-aware removal result that makes preservation guarantees explicit.""" + + placement_id: int + ownership: StoryArcPlacementOwnership + artifact_removed: bool + canonical_preserved: bool + referenced_artifact_preserved: bool + automatic_sync_disabled: bool + + +class StoryArcPolicyMigrationPreviewRequest(StoryArcPlacementPolicyUpdate): + """Complete proposed policy and the exact arc revision being previewed.""" + + +class StoryArcPolicyMigrationConfirmationRequest(StoryArcPlacementPolicyUpdate): + """Actor-bound signed preview plus an exact destructive-change phrase.""" + + preview_token: str = Field(..., min_length=1, max_length=16_384) + confirmation: str = Field( + ..., + min_length=len(STORY_ARC_POLICY_MIGRATION_CONFIRMATION), + max_length=len(STORY_ARC_POLICY_MIGRATION_CONFIRMATION), + json_schema_extra={"const": STORY_ARC_POLICY_MIGRATION_CONFIRMATION}, + ) + + @field_validator("confirmation") + @classmethod + def validate_exact_confirmation(cls, value: str) -> str: + """Expose and enforce one fixed, non-ambiguous confirmation phrase.""" + if value != STORY_ARC_POLICY_MIGRATION_CONFIRMATION: + raise ValueError( + f'Type exactly "{STORY_ARC_POLICY_MIGRATION_CONFIRMATION}" to continue' + ) + return value + + +class StoryArcPolicyMigrationPreviewItemResponse(BaseModel): + """One intended old/new path disclosure in an authenticated preview page.""" + + model_config = ConfigDict(from_attributes=True) + + placement_id: int + membership_id: int + ownership: str + action: str + old_mode: str + new_mode: str + old_path: str + new_path: str | None + collision: str + blocked: bool + reason: str | None + required_bytes: int + + +class StoryArcPolicyMigrationPreviewResponse(BaseModel): + """Complete migration counts/digest plus one bounded keyset item page.""" + + story_arc_id: int + expected_revision: int + current_policy: StoryArcPlacementPolicyResponse + proposed_policy: StoryArcPlacementPolicyResponse + scope_digest: str + preview_token: str + required_confirmation: str + total_placement_count: int + managed_migrate_count: int + managed_remove_count: int + managed_unchanged_count: int + referenced_preserved_count: int + collision_count: int + blocked_count: int + required_bytes: int + available_bytes: int | None + global_block_codes: list[str] + items: list[StoryArcPolicyMigrationPreviewItemResponse] + limit: int + after_placement_id: int + next_cursor: int | None + has_more: bool + requires_confirmation: bool + execution_supported: bool + filesystem_mutated: bool + + +class StoryArcPolicyMigrationConfirmationResponse(BaseModel): + """Validated preparation truth without claiming filesystem execution.""" + + story_arc_id: int + expected_revision: int + scope_digest: str + confirmed: bool + ready_for_execution: bool + execution_supported: bool + mutation_performed: bool + policy_update_block_code: str diff --git a/src/pullbox/services/airdcpp_automatic_search.py b/src/pullbox/services/airdcpp_automatic_search.py index d2acd849..dd824d7c 100644 --- a/src/pullbox/services/airdcpp_automatic_search.py +++ b/src/pullbox/services/airdcpp_automatic_search.py @@ -1,4 +1,4 @@ -"""Attach bounded automatic AirDC++ evaluation without queue mutation.""" +"""Attach bounded, opted-in AirDC++ results before shared acquisition routing.""" from __future__ import annotations diff --git a/src/pullbox/services/airdcpp_search_acquisition.py b/src/pullbox/services/airdcpp_search_acquisition.py new file mode 100644 index 00000000..60ae08e8 --- /dev/null +++ b/src/pullbox/services/airdcpp_search_acquisition.py @@ -0,0 +1,187 @@ +"""Server-owned AirDC++ search handoff and durable review routes.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +from pydantic import TypeAdapter +from sqlalchemy import select, update +from sqlalchemy.orm import selectinload + +from pullbox.composition import airdcpp +from pullbox.core.acquisition import AcquisitionProtocol +from pullbox.core.encryption import decrypt_secret, encrypt_secret, is_encrypted +from pullbox.core.exceptions import ProviderError +from pullbox.core.release_parser import parse_release_title +from pullbox.models.client import DownloadClientConfig +from pullbox.models.download import DownloadClientType, DownloadHistory, DownloadState +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile, MatchConfidence +from pullbox.providers.airdcpp.supervisor import AirDcppSupervisorState +from pullbox.providers.base import ReleaseResult +from pullbox.services.airdcpp_acquisition import AirDcppQueueAcquisitionService +from pullbox.services.airdcpp_search_types import DcMetrics, DcRoute, DcValidatedCandidate +from pullbox.services.blocklist_service import BlocklistService +from pullbox.services.release_validator import ValidationResult + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.models.pending_match import PendingMatch + from pullbox.providers.airdcpp.supervisor import RegistrySupervisor + +_ROUTE_ADAPTER = TypeAdapter(DcRoute) +_ACTIVE_STATES = ( + DownloadState.QUEUED, + DownloadState.SENT, + DownloadState.DOWNLOADING, + DownloadState.FINALIZING, + DownloadState.PAUSED, + DownloadState.RETRY_PENDING, + DownloadState.POST_PROCESSING, + DownloadState.COMPLETED, +) + + +class DcIssueAlreadyOwnedError(ProviderError): + """Stop automatic fallback rather than replacing an already-owned issue.""" + + +async def ready_dc_client( + session: AsyncSession, + candidate: DcValidatedCandidate, + *, + automatic: bool, +) -> tuple[DownloadClientConfig, RegistrySupervisor]: + """Recheck exact client ownership and opt-in after potentially slow search.""" + client = await session.scalar( + select(DownloadClientConfig) + .options(selectinload(DownloadClientConfig.airdcpp_settings)) + .where( + DownloadClientConfig.id == candidate.route.client_config_id, + DownloadClientConfig.client_type == DownloadClientType.AIRDCPP, + DownloadClientConfig.enabled.is_(True), + ) + .execution_options(populate_existing=True) + ) + if ( + client is None + or client.airdcpp_settings is None + or not client.airdcpp_settings.search_enabled + or (automatic and not client.airdcpp_settings.automatic_search_enabled) + ): + raise ProviderError( + "airdcpp", "The selected AirDC++ client is not enabled for this search." + ) + if automatic and not await BlocklistService.filter_results(session, [candidate.release]): + raise ProviderError("airdcpp", "The selected AirDC++ result is blocklisted.") + registry = airdcpp.get_airdcpp_supervisor_registry() + supervisor = registry.get(client.id) if registry else None + if supervisor is None or supervisor.state is not AirDcppSupervisorState.READY: + raise ProviderError("airdcpp", "The selected AirDC++ client is not ready.") + return client, supervisor + + +async def acquire_dc_candidate( + session: AsyncSession, + *, + candidate: DcValidatedCandidate, + issue_id: int, + search_log_id: int | None, + request_key: str, + automatic: bool, +) -> tuple[DownloadHistory, bool]: + """Reuse durable queue intent; never create a second active issue download.""" + await session.commit() + # Serialize the eligibility check and durable intent across workers. The + # acquisition service commits this short transaction before remote mutation. + await session.execute(update(Issue).where(Issue.id == issue_id).values(status=Issue.status)) + client, supervisor = await ready_dc_client(session, candidate, automatic=automatic) + if await session.scalar(select(LibraryFile.id).where(LibraryFile.issue_id == issue_id)): + await session.commit() + raise DcIssueAlreadyOwnedError( + "airdcpp", "This issue already has a library file; use Find Alternative." + ) + existing = await session.scalar( + select(DownloadHistory) + .where( + DownloadHistory.issue_id == issue_id, + DownloadHistory.state.in_(_ACTIVE_STATES), + DownloadHistory.imported_at.is_(None), + ) + .order_by(DownloadHistory.id.desc()) + .limit(1) + ) + if existing is not None: + await session.commit() + return existing, False + assert client.airdcpp_settings is not None + result = await AirDcppQueueAcquisitionService().acquire( + session, + candidate=candidate, + issue_id=issue_id, + request_key=request_key, + search_log_id=search_log_id, + api_client=supervisor.api_client, + queue_priority=client.airdcpp_settings.queue_priority, + replace_existing_file=False, + ) + history = await session.get(DownloadHistory, result.download_history_id) + assert history is not None + return history, True + + +def dc_review_snapshot( + candidate: DcValidatedCandidate, *, issue_id: int, search_log_id: int +) -> str: + """Keep hub/search route details opaque outside the server, including after restart.""" + return encrypt_secret( + json.dumps( + { + "version": 1, + "issue_id": issue_id, + "search_log_id": search_log_id, + "route": _ROUTE_ADAPTER.dump_python(candidate.route, mode="json"), + } + ) + ) + + +def dc_review_candidate(pending: PendingMatch) -> tuple[DcValidatedCandidate, int | None]: + """Rehydrate only a server-issued route bound to this pending issue.""" + snapshot = pending.match_details.get("dc_route_snapshot") + if not isinstance(snapshot, str) or not is_encrypted(snapshot): + raise ValueError("The Direct Connect review route is unavailable; search again.") + data = json.loads(decrypt_secret(snapshot)) + if data.get("version") != 1 or data.get("issue_id") != pending.issue_id: + raise ValueError("The Direct Connect review route belongs to another issue.") + route = _ROUTE_ADAPTER.validate_python(data["route"]) + if route.size_bytes != pending.file_size: + raise ValueError("The Direct Connect review file size does not match its route.") + release = ReleaseResult( + title=pending.release_title, + indexer_name=str(pending.match_details.get("indexer_name", "AirDC++")), + download_url=pending.download_url, + size_bytes=route.size_bytes, + age_days=None, + seeders=None, + leechers=None, + grabs=None, + is_torrent=False, + category=None, + published_at=None, + protocol=AcquisitionProtocol.DC, + ) + parsed = parse_release_title(release.title) + if parsed is None: + raise ValueError("The Direct Connect review title is invalid; search again.") + validation = ValidationResult( + is_match=True, + confidence=MatchConfidence(pending.confidence), + parsed=parsed, + release=release, + ) + return DcValidatedCandidate(release, validation, route, DcMetrics(0, 0, 0, None)), data.get( + "search_log_id" + ) diff --git a/src/pullbox/services/airdcpp_search_coordinator.py b/src/pullbox/services/airdcpp_search_coordinator.py index 4350cefb..fbcba21c 100644 --- a/src/pullbox/services/airdcpp_search_coordinator.py +++ b/src/pullbox/services/airdcpp_search_coordinator.py @@ -194,6 +194,7 @@ async def run(client: AirDcppSearchClient) -> _ClientResult: wanted_series=target.series_title, wanted_issue=target.issue_number, wanted_year=target.search_year, + year_context=target.year_context, wanted_issue_type=target.issue_type, alternate_names=target.alternate_names, wanted_issue_title=target.issue_title, @@ -537,11 +538,7 @@ async def _final_snapshot( def _query_pattern(target: IssueSearchTarget) -> str: - number = ( - str(int(target.issue_number)) - if target.issue_number.is_integer() - else str(target.issue_number) - ) + number = target.effective_issue_number_text return f"{target.series_title} {number}" diff --git a/src/pullbox/services/airdcpp_search_types.py b/src/pullbox/services/airdcpp_search_types.py index 9120d147..76396001 100644 --- a/src/pullbox/services/airdcpp_search_types.py +++ b/src/pullbox/services/airdcpp_search_types.py @@ -4,14 +4,13 @@ import re from dataclasses import dataclass +from datetime import datetime # noqa: TC003 - durable route validation needs this at runtime from enum import StrEnum from typing import TYPE_CHECKING from pullbox.core.acquisition import AcquisitionProtocol if TYPE_CHECKING: - from datetime import datetime - from pullbox.providers.base import ReleaseResult from pullbox.services.release_validator import ValidationResult diff --git a/src/pullbox/services/catalog/__init__.py b/src/pullbox/services/catalog/__init__.py new file mode 100644 index 00000000..2742c6ce --- /dev/null +++ b/src/pullbox/services/catalog/__init__.py @@ -0,0 +1 @@ +"""Verified local Comic Vine catalog downloads and queries.""" diff --git a/src/pullbox/services/catalog/contract.py b/src/pullbox/services/catalog/contract.py new file mode 100644 index 00000000..de358742 --- /dev/null +++ b/src/pullbox/services/catalog/contract.py @@ -0,0 +1,165 @@ +"""Verify the publisher before accepting any download coordinates.""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import json +import re +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING, Any + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +if TYPE_CHECKING: + from collections.abc import Mapping + +MAX_MANIFEST_BYTES = 1024 * 1024 +MAX_ARTIFACT_BYTES = 2 * 1024**3 +VERSION = re.compile(r"\d{8}T\d{6}Z") +SHA256 = re.compile(r"[0-9a-f]{64}") + +# Public verification keys only. Ship a new key before the publisher rotates to it. +TRUSTED_KEYS = { + "catalog-2026-09": Ed25519PublicKey.from_public_bytes( + base64.b64decode("SwsNUSLDGU8PvkvImuLSRopRgKMdQM2Odr2JA5bA+tg=") + ), +} + + +class CatalogError(ValueError): + """A safe, actionable catalog failure.""" + + +@dataclass(frozen=True) +class Artifact: + version: str + download_path: str + sha256: str + size_bytes: int + base_version: str | None = None + + +@dataclass(frozen=True) +class Publication: + latest_version: str + full_snapshot: Artifact + snapshots: tuple[Artifact, ...] + patches: tuple[Artifact, ...] + + def latest_patch(self) -> Artifact | None: + return next( + ( + patch + for patch in self.patches + if patch.version == self.latest_version + and patch.base_version == self.full_snapshot.version + ), + None, + ) + + +def valid_version(value: object) -> str: + if not isinstance(value, str) or VERSION.fullmatch(value) is None: + raise CatalogError("Catalog version is not supported. Update Pullbox and try again.") + try: + datetime.strptime(value, "%Y%m%dT%H%M%SZ") + except ValueError as exc: + raise CatalogError("Catalog version is invalid.") from exc + return value + + +def _object(value: object) -> dict[str, Any]: + if not isinstance(value, dict): + raise CatalogError("Catalog manifest is invalid.") + return value + + +def _artifact(value: object, *, patch: bool = False) -> Artifact: + item = _object(value) + version = valid_version(item.get("target_version" if patch else "version")) + base = valid_version(item.get("base_version")) if patch else None + path = ( + f"/api/v2/catalog/patches/{base}/{version}" + if patch + else f"/api/v2/catalog/snapshots/{version}" + ) + checksum, size = item.get("sha256"), item.get("size_bytes") + if ( + item.get("download_path") != path + or not isinstance(checksum, str) + or SHA256.fullmatch(checksum) is None + or type(size) is not int + or not 0 < size <= MAX_ARTIFACT_BYTES + or (base is not None and base >= version) + ): + raise CatalogError("Catalog artifact information is invalid.") + return Artifact(version, path, checksum, size, base) + + +def verify_manifest( + raw: bytes, + keys: Mapping[str, Ed25519PublicKey] = TRUSTED_KEYS, +) -> Publication: + """Parse only bounded JSON and verify its canonical signed payload first.""" + if len(raw) > MAX_MANIFEST_BYTES: + raise CatalogError("Catalog manifest is too large.") + try: + document = _object(json.loads(raw)) + payload, signature = _object(document.get("payload")), _object(document.get("signature")) + key_id = signature.get("key_id") + if not isinstance(key_id, str) or key_id not in keys: + raise CatalogError("Unknown catalog signing key. Update Pullbox and try again.") + canonical = json.dumps( + payload, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + if ( + signature.get("algorithm") != "Ed25519" + or signature.get("payload_sha256") != hashlib.sha256(canonical).hexdigest() + ): + raise CatalogError("Catalog signature verification failed.") + encoded = signature.get("value") + if not isinstance(encoded, str): + raise CatalogError("Catalog signature verification failed.") + keys[key_id].verify( + base64.b64decode(encoded + "=" * (-len(encoded) % 4), altchars=b"-_", validate=True), + canonical, + ) + if ( + payload.get("format_id") != "pullbox-catalog-v2-publication" + or payload.get("schema_version") != "1" + ): + raise CatalogError("Catalog format is not supported. Update Pullbox and try again.") + full = _artifact(payload.get("full_snapshot")) + snapshots_raw, patches_raw = payload.get("snapshots", []), payload.get("patches", []) + if not isinstance(snapshots_raw, list) or not isinstance(patches_raw, list): + raise CatalogError("Catalog manifest is invalid.") + snapshots = tuple(_artifact(item) for item in snapshots_raw) + patches = tuple(_artifact(item, patch=True) for item in patches_raw) + bases = {full.version, *(entry.version for entry in snapshots)} + coordinates = [item.download_path for item in (full, *snapshots, *patches)] + latest = valid_version(payload.get("latest_version")) + if ( + len(coordinates) != len(set(coordinates)) + or any(item.version >= full.version for item in snapshots) + or any(item.base_version not in bases for item in patches) + or any( + item.base_version != full.version and item.version >= full.version + for item in patches + ) + or latest != max([full.version, *(item.version for item in patches)]) + ): + raise CatalogError("Catalog update lineage is invalid.") + result = Publication(latest, full, snapshots, patches) + if latest != full.version and result.latest_patch() is None: + raise CatalogError("Catalog latest update is unavailable.") + return result + except (InvalidSignature, binascii.Error) as exc: + raise CatalogError("Catalog signature verification failed.") from exc + except CatalogError: + raise + except (ValueError, TypeError, KeyError, RecursionError) as exc: + raise CatalogError("Catalog manifest is invalid.") from exc diff --git a/src/pullbox/services/catalog/database.py b/src/pullbox/services/catalog/database.py new file mode 100644 index 00000000..e15250f3 --- /dev/null +++ b/src/pullbox/services/catalog/database.py @@ -0,0 +1,235 @@ +"""Validate signed SQLite artifacts using the producer's fixed schema and hash. + +This database is an external read-only artifact, separate from the ORM application +database. SQL identifiers below are contract constants, never user input. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import sqlite3 +from contextlib import closing +from datetime import datetime +from typing import TYPE_CHECKING, Any + +from pullbox.services.catalog.contract import CatalogError, valid_version + +if TYPE_CHECKING: + from pathlib import Path + +TABLES = { + "publishers": ("id,name", "id", "publisher"), + "series": ("id,name,start_year,publisher_id,issue_count,cover_url", "id", "series"), + "series_aliases": ( + "series_id,position,alias,normalized_alias", + "series_id,position", + "series_alias", + ), + "issues": ( + "id,series_id,issue_number,normalized_issue_number,sort_number,title,cover_date,store_date,cover_url", + "id", + "issue", + ), +} +FTS_TABLES = { + "series_fts", + *(f"series_fts_{suffix}" for suffix in ("config", "content", "data", "docsize", "idx")), +} +REQUIRED_TABLES = {*TABLES, "series_fts", "dataset_manifest", "schema_migrations"} +# Closed SQL allowlist, generated only from contract constants. Request input +# never becomes an identifier, including in the external artifact database. +CONTENT_QUERIES = { + table: f"SELECT {columns} FROM {table} ORDER BY {order}" + for table, (columns, order, _) in TABLES.items() +} +COUNT_QUERIES = {table: f"SELECT COUNT(*) FROM {table}" for table in TABLES} +PATCH_DELETE_QUERIES = { + table: f"SELECT {keys} FROM {prefix}_deletes" for table, (_, keys, prefix) in TABLES.items() +} +PATCH_UPSERT_QUERIES = { + table: f"SELECT {columns} FROM {prefix}_upserts" + for table, (columns, _, prefix) in TABLES.items() +} +MANIFEST_QUERIES = { + "dataset_manifest": "SELECT key,value FROM dataset_manifest", + "patch_manifest": "SELECT key,value FROM patch_manifest", + "target_dataset_manifest": "SELECT key,value FROM target_dataset_manifest", +} +REBUILD_FTS = """ +INSERT INTO series_fts(rowid,series_id,name,aliases) +SELECT s.id,s.id,s.name,COALESCE(GROUP_CONCAT(a.alias,' '),'') +FROM series s LEFT JOIN series_aliases a ON a.series_id=s.id +GROUP BY s.id,s.name ORDER BY s.id +""" + + +def safe_path(path: Path) -> Path: + """Reject symlink components in app-owned catalog paths.""" + if any(part.is_symlink() for part in (path, *path.parents)): + raise CatalogError("Catalog storage contains a symbolic link. Check the data volume.") + return path + + +def open_readonly(path: Path) -> sqlite3.Connection: + safe_path(path) + db = sqlite3.connect(path.absolute().as_uri() + "?mode=ro&immutable=1", uri=True) + db.execute("PRAGMA query_only=ON") + db.execute("PRAGMA trusted_schema=OFF") + db.execute("PRAGMA cache_size=-8192") + return db + + +def file_sha256(path: Path) -> str: + safe_path(path) + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def read_manifest(db: sqlite3.Connection, table: str = "dataset_manifest") -> dict[str, Any]: + if table not in MANIFEST_QUERIES: + raise CatalogError("Catalog manifest table is invalid.") + return {str(k): json.loads(v) for k, v in db.execute(MANIFEST_QUERIES[table])} + + +def logical_hash(db: sqlite3.Connection) -> str: + digest = hashlib.sha256() + for table in TABLES: + for row in db.execute(CONTENT_QUERIES[table]): + digest.update(table.encode() + b"\0") + digest.update( + json.dumps(row, ensure_ascii=False, separators=(",", ":")).encode() + b"\n" + ) + return digest.hexdigest() + + +def _check_database(db: sqlite3.Connection, application_id: int, allowed: set[str]) -> None: + if ( + db.execute("PRAGMA application_id").fetchone()[0] != application_id + or db.execute("PRAGMA user_version").fetchone()[0] != 1 + ): + raise CatalogError("Catalog database format is not supported. Update Pullbox.") + if db.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise CatalogError("Catalog database integrity check failed. Retry the download.") + objects = db.execute( + "SELECT name,type FROM sqlite_master WHERE type IN ('table','view','trigger')" + ).fetchall() + if any( + kind != "table" or name not in allowed + for name, kind in objects + if not name.startswith("sqlite_") + ): + raise CatalogError("Catalog contains unsupported database objects.") + + +def validate_snapshot(path: Path, version: str) -> dict[str, Any]: + """Recompute logical content before allowing a new catalog to become active.""" + valid_version(version) + try: + with closing(open_readonly(path)) as db: + _check_database(db, 0x50424332, REQUIRED_TABLES | FTS_TABLES) + if db.execute("PRAGMA foreign_key_check").fetchone() is not None: + raise CatalogError("Catalog has invalid record relationships.") + manifest = read_manifest(db) + for key, value in { + "format_id": "pullbox-catalog-v2", + "schema_version": "1", + "compatibility_version": "pullbox-only", + "application_id": 0x50424332, + "dataset_version": version, + }.items(): + if manifest.get(key) != value: + raise CatalogError("Catalog identity does not match its signed publication.") + cutoff = datetime.fromisoformat(str(manifest.get("source_cutoff_at", ""))) + if cutoff.tzinfo is None or cutoff.strftime("%Y%m%dT%H%M%SZ") != version: + raise CatalogError("Catalog source timestamp is invalid.") + counts = {table: db.execute(COUNT_QUERIES[table]).fetchone()[0] for table in TABLES} + if counts != manifest.get("counts") or logical_hash(db) != manifest.get( + "content_sha256" + ): + raise CatalogError("Catalog content checksum failed. Retry the download.") + if db.execute("SELECT COUNT(*) FROM series_fts").fetchone()[0] != counts["series"]: + raise CatalogError("Catalog search index is incomplete.") + if db.execute( + "SELECT 1 FROM series s LEFT JOIN series_fts f ON f.rowid=s.id " + "WHERE f.series_id IS NULL OR f.series_id != s.id OR f.name != s.name LIMIT 1" + ).fetchone(): + raise CatalogError("Catalog search index is inconsistent.") + db.execute("SELECT version,applied_at FROM schema_migrations LIMIT 1").fetchall() + return manifest + except (sqlite3.Error, OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + if isinstance(exc, CatalogError): + raise + raise CatalogError("Catalog database could not be validated. Retry the download.") from exc + + +def apply_catalog_patch( + base: Path, patch: Path, output: Path, base_version: str, target_version: str +) -> dict[str, Any]: + """Always reconstruct from the immutable weekly base, including reverted rows.""" + validate_snapshot(base, base_version) + safe_path(output) + try: + with closing(open_readonly(patch)) as changes: + allowed = {"patch_manifest", "target_dataset_manifest"} + allowed.update( + f"{prefix}_{kind}" + for _, _, prefix in TABLES.values() + for kind in ("upserts", "deletes") + ) + _check_database(changes, 0x50425044, allowed) + meta = read_manifest(changes, "patch_manifest") + expected = { + "format_id": "pullbox-catalog-v2-patch", + "schema_version": "1", + "base_version": base_version, + "target_version": target_version, + "base_snapshot_sha256": file_sha256(base), + } + if any(meta.get(k) != v for k, v in expected.items()): + raise CatalogError("Catalog patch does not match the retained weekly base.") + with base.open("rb") as source, output.open("xb") as destination: + shutil.copyfileobj(source, destination, 1024 * 1024) + with closing(sqlite3.connect(output)) as db: + db.execute("PRAGMA trusted_schema=OFF") + db.execute("PRAGMA foreign_keys=ON") + db.execute("PRAGMA journal_mode=DELETE") + db.execute("PRAGMA synchronous=FULL") + with db: + db.execute("PRAGMA defer_foreign_keys=ON") + for table, (_, keys, _prefix) in reversed(TABLES.items()): + where = " AND ".join(f"{key}=?" for key in keys.split(",")) + db.executemany( + f"DELETE FROM {table} WHERE {where}", + changes.execute(PATCH_DELETE_QUERIES[table]), + ) + for table, (columns, keys, _prefix) in TABLES.items(): + cols = columns.split(",") + assignments = ",".join( + f"{column}=excluded.{column}" + for column in cols + if column not in keys.split(",") + ) + db.executemany( + f"INSERT INTO {table} ({columns}) " + f"VALUES ({','.join('?' for _ in cols)}) " + f"ON CONFLICT ({keys}) DO UPDATE SET {assignments}", + changes.execute(PATCH_UPSERT_QUERIES[table]), + ) + db.execute("DELETE FROM dataset_manifest") + db.executemany( + "INSERT INTO dataset_manifest VALUES (?,?)", + changes.execute("SELECT key,value FROM target_dataset_manifest"), + ) + db.execute("DELETE FROM series_fts") + db.execute(REBUILD_FTS) + result = validate_snapshot(output, target_version) + if result["content_sha256"] != meta.get("target_content_sha256"): + raise CatalogError("Catalog patch target checksum failed.") + with output.open("rb") as stream: + os.fsync(stream.fileno()) + return result + except (sqlite3.Error, OSError, json.JSONDecodeError) as exc: + raise CatalogError("Catalog patch could not be applied. Retry the update.") from exc diff --git a/src/pullbox/services/catalog/lookup.py b/src/pullbox/services/catalog/lookup.py new file mode 100644 index 00000000..0607ac59 --- /dev/null +++ b/src/pullbox/services/catalog/lookup.py @@ -0,0 +1,103 @@ +"""Catalog discovery adapter for existing search and import matching contracts.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from pullbox.services.catalog.contract import CatalogError + +if TYPE_CHECKING: + from pullbox.providers.base import ( + IssueMetadata, + IssueSummary, + SeriesMetadata, + SeriesSearchResult, + ) + from pullbox.services.catalog.reader import CatalogReader + + +class CatalogLookupService: + """Basic catalog discovery only; full metadata refresh still owns its provider.""" + + is_local_catalog = True + + def __init__(self, reader: CatalogReader) -> None: + self.reader = reader + + async def search_series( + self, + query: str, + year: int | None = None, + *, + limit: int = 1000, + offset: int = 0, + suppress_errors: bool = False, + ) -> list[SeriesSearchResult]: + return await self.reader.search(query, year, limit, offset) + + async def search_series_page( + self, + query: str, + year: int | None = None, + *, + limit: int = 100, + offset: int = 0, + suppress_errors: bool = False, + ) -> tuple[list[SeriesSearchResult], int]: + rows = await self.reader.search(query, year, limit, offset) + return rows, len(rows) + + async def search_series_globally( + self, + query: str, + *, + max_results: int = 1000, + page_size: int = 100, + suppress_errors: bool = False, + ) -> tuple[list[SeriesSearchResult], int]: + rows = await self.reader.search(query, limit=max_results) + return rows, len(rows) + + async def get_series(self, series_provider_id: str) -> SeriesMetadata: + result = await self.reader.series(int(series_provider_id)) + if result is None: + raise CatalogError( + "This series is not in the local catalog. Check for a catalog update." + ) + return result + + async def get_series_cached(self, series_provider_id: str) -> SeriesMetadata | None: + return await self.reader.series(int(series_provider_id)) + + async def get_issues_for_series(self, series_provider_id: str) -> list[IssueSummary]: + await self.get_series(series_provider_id) + return await self.reader.issues(int(series_provider_id)) + + async def get_issues_for_series_by_numbers( + self, series_provider_id: str, issue_numbers: list[float] + ) -> list[IssueSummary]: + numbers = set(issue_numbers) + return [ + issue + for issue in await self.get_issues_for_series(series_provider_id) + if issue.issue_number in numbers + ] + + async def get_issue(self, issue_provider_id: str) -> IssueMetadata: + result = await self.reader.issue(int(issue_provider_id)) + if result is None: + raise CatalogError( + "This issue is not in the local catalog. Check for a catalog update." + ) + return result + + async def close(self) -> None: + """Queries own and close their connections individually.""" + + +def catalog_or_provider(provider: Any) -> Any: + """Select the local discovery source without altering full refresh providers.""" + from pullbox.services.catalog.reader import get_catalog_reader + + reader = get_catalog_reader() + return CatalogLookupService(reader) if reader.available else provider diff --git a/src/pullbox/services/catalog/reader.py b/src/pullbox/services/catalog/reader.py new file mode 100644 index 00000000..304a9c70 --- /dev/null +++ b/src/pullbox/services/catalog/reader.py @@ -0,0 +1,193 @@ +"""Bounded SQLite reads from an immutable catalog generation.""" + +from __future__ import annotations + +import re +import sqlite3 +import threading +from contextlib import closing +from dataclasses import dataclass +from datetime import datetime +from functools import lru_cache +from typing import TYPE_CHECKING, Any + +import structlog + +from pullbox.config import get_settings +from pullbox.core.issue_numbers import parse_issue_number_text +from pullbox.core.naming import detect_issue_type_from_metadata_title +from pullbox.providers.base import IssueMetadata, IssueSummary, SeriesMetadata, SeriesSearchResult +from pullbox.services.catalog.contract import CatalogError, valid_version +from pullbox.services.catalog.database import open_readonly, safe_path, validate_snapshot +from pullbox.services.catalog.storage import disk_work, load_json + +if TYPE_CHECKING: + from pathlib import Path + +logger = structlog.get_logger(__name__) +SERIES_COLUMNS = "s.id,s.name,s.start_year,p.name,s.issue_count,s.cover_url" +SERIES_FROM = "series s LEFT JOIN publishers p ON p.id=s.publisher_id" +ISSUE_COLUMNS = ( + "id,series_id,issue_number,normalized_issue_number,sort_number," + "title,cover_date,store_date,cover_url" +) + + +@dataclass(frozen=True) +class CatalogSeriesMetadata(SeriesMetadata): + source_cutoff_at: datetime | None = None + + +@dataclass(frozen=True) +class CatalogIssueSummary(IssueSummary): + source_cutoff_at: datetime | None = None + + +class CatalogReader: + """Pin a file per query; verify a generation once before serving its rows.""" + + def __init__(self, root: Path) -> None: + self.root = root + self._validated: set[tuple[str, int, int, int]] = set() + self._validation_lock = threading.Lock() + + @property + def available(self) -> bool: + return (self.root / "active.json").exists() + + def _generation(self) -> tuple[Path, datetime]: + reference = load_json(self.root / "active.json") + version = valid_version(reference.get("version")) + relative = reference.get("path") + if relative not in {f"bases/{version}.db", f"versions/{version}.db"}: + raise CatalogError("Catalog active file is invalid. Retry the catalog update.") + path = safe_path(self.root / str(relative)) + stat = path.stat() + identity = (str(path), stat.st_ino, stat.st_mtime_ns, stat.st_size) + with self._validation_lock: + if identity not in self._validated: + validate_snapshot(path, version) + if len(self._validated) >= 8: + self._validated.clear() + self._validated.add(identity) + return path, datetime.fromisoformat(str(reference["source_cutoff_at"])) + + def _query(self, sql: str, params: tuple[object, ...]) -> tuple[list[Any], datetime]: + try: + path, cutoff = self._generation() + with closing(open_readonly(path)) as db: + return db.execute(sql, params).fetchall(), cutoff + except (sqlite3.Error, OSError, KeyError) as exc: + raise CatalogError( + "The local catalog could not be read. Retry its update in Metadata settings." + ) from exc + + async def search( + self, query: str, year: int | None = None, limit: int = 1000, offset: int = 0 + ) -> list[SeriesSearchResult]: + terms = re.findall(r"\w+", query[:256], flags=re.UNICODE)[:16] + if not terms: + return [] + expression = " AND ".join(f'"{term}"*' for term in terms) + rows, _ = await disk_work( + self._query, + f"SELECT {SERIES_COLUMNS} FROM series_fts f JOIN series s ON s.id=f.rowid " + "LEFT JOIN publishers p ON p.id=s.publisher_id " + "WHERE series_fts MATCH ? AND (? IS NULL OR s.start_year=?) " + "ORDER BY rank,s.id LIMIT ? OFFSET ?", + (expression, year, year, max(1, min(limit, 1000)), max(0, min(offset, 10000))), + ) + return [ + SeriesSearchResult( + str(r[0]), + r[1], + r[2], + r[3], + r[4], + None, + r[5], + None, + f"https://comicvine.gamespot.com/volume/4050-{r[0]}/", + ) + for r in rows + ] + + async def series(self, series_id: int) -> CatalogSeriesMetadata | None: + rows, cutoff = await disk_work( + self._query, f"SELECT {SERIES_COLUMNS} FROM {SERIES_FROM} WHERE s.id=?", (series_id,) + ) + if not rows: + return None + r = rows[0] + return CatalogSeriesMetadata( + str(r[0]), + r[1], + r[1], + r[2], + None, + None, + r[3], + None, + r[5], + r[4], + f"https://comicvine.gamespot.com/volume/4050-{r[0]}/", + cutoff, + ) + + async def issues(self, series_id: int) -> list[IssueSummary]: + rows, cutoff = await disk_work( + self._query, + f"SELECT {ISSUE_COLUMNS} FROM issues WHERE series_id=? " + "ORDER BY CAST(sort_number AS REAL),issue_number,id", + (series_id,), + ) + return [self._summary(row, cutoff) for row in rows] + + @staticmethod + def _summary(row: Any, cutoff: datetime) -> CatalogIssueSummary: + try: + number, exact = parse_issue_number_text(str(row[2] or row[3] or "0")) + except ValueError: + number, exact = 0.0, None + return CatalogIssueSummary( + str(row[0]), + number, + row[5], + row[6], + row[8], + detect_issue_type_from_metadata_title(row[5] or ""), + exact, + cutoff, + ) + + async def issue(self, issue_id: int) -> IssueMetadata | None: + """Return only the basic identity fields used during import file matching.""" + rows, cutoff = await disk_work( + self._query, f"SELECT {ISSUE_COLUMNS} FROM issues WHERE id=?", (issue_id,) + ) + if not rows: + return None + row = rows[0] + summary = self._summary(row, cutoff) + return IssueMetadata( + summary.provider_id, + str(row[1]), + summary.issue_number, + summary.title, + None, + summary.release_date, + row[7], + summary.cover_url, + None, + f"https://comicvine.gamespot.com/issue/4000-{row[0]}/", + issue_number_text=summary.issue_number_text, + ) + + +@lru_cache(maxsize=4) +def _reader(root: Path) -> CatalogReader: + return CatalogReader(root) + + +def get_catalog_reader() -> CatalogReader: + return _reader(get_settings().data_dir / "catalog") diff --git a/src/pullbox/services/catalog/retention.py b/src/pullbox/services/catalog/retention.py new file mode 100644 index 00000000..1a70c046 --- /dev/null +++ b/src/pullbox/services/catalog/retention.py @@ -0,0 +1,47 @@ +"""Bounded cleanup of catalog-owned artifacts, never of library data.""" + +from __future__ import annotations + +import re +import time +from typing import TYPE_CHECKING + +from pullbox.services.catalog.database import safe_path +from pullbox.services.catalog.storage import load_json + +if TYPE_CHECKING: + from pathlib import Path + + +def cleanup(root: Path, *, completed: bool = False) -> None: + """Caller holds the update lock; keep active, previous and their weekly bases. + + Old generations get a two-day grace period so in-flight readers can finish. + Only fixed-format owned files are removed; unknown files are left alone. + """ + keep = set() + for pointer in ("active.json", "previous.json"): + ref = load_json(root / pointer) + keep.add(str(ref.get("path", ""))) + keep.add(f"bases/{ref.get('base_version', '')}.db") + for directory, pattern in ( + ("staging", r"catalog-[A-Za-z0-9_-]+(?:\.patch)?\.db(?:-journal)?"), + ("downloads", r"[a-f0-9]{64}\.part"), + ("bases", r"\d{8}T\d{6}Z\.db"), + ("versions", r"\d{8}T\d{6}Z\.db"), + ): + folder = safe_path(root / directory) + if not folder.exists(): + continue + for path in folder.iterdir(): + safe_path(path) + if not path.is_file() or not re.fullmatch(pattern, path.name): + continue + relative = str(path.relative_to(root)) + expired = path.stat().st_mtime < time.time() - 2 * 86400 + if ( + directory == "staging" + or (directory == "downloads" and completed) + or (expired and relative not in keep) + ): + path.unlink() diff --git a/src/pullbox/services/catalog/service.py b/src/pullbox/services/catalog/service.py new file mode 100644 index 00000000..99defaf9 --- /dev/null +++ b/src/pullbox/services/catalog/service.py @@ -0,0 +1,353 @@ +"""One resumable, verified catalog update at a time, independent of library writes.""" + +from __future__ import annotations + +import asyncio +import fcntl +import os +import shutil +from datetime import UTC, datetime, timedelta +from functools import lru_cache +from typing import TYPE_CHECKING, Any + +import httpx +import structlog +from pydantic import BaseModel, ValidationError + +from pullbox.config import get_settings +from pullbox.services.catalog.contract import ( + MAX_MANIFEST_BYTES, + TRUSTED_KEYS, + Artifact, + CatalogError, + Publication, + valid_version, + verify_manifest, +) +from pullbox.services.catalog.database import ( + apply_catalog_patch, + file_sha256, + safe_path, + validate_snapshot, +) +from pullbox.services.catalog.retention import cleanup +from pullbox.services.catalog.storage import ( + activate_file, + atomic_json, + decompress, + disk_work, + load_json, + stage_path, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + from pathlib import Path + + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +logger = structlog.get_logger(__name__) +BUSY_PHASES = {"checking", "downloading", "verifying", "decompressing", "installing"} + + +class CatalogStatus(BaseModel): + phase: str = "not_downloaded" + requested: bool = False + automatic_updates: bool = True + installed_version: str | None = None + source_cutoff_at: str | None = None + target_version: str | None = None + bytes_downloaded: int = 0 + bytes_total: int = 0 + catalog_size_bytes: int = 0 + last_checked_at: datetime | None = None + last_updated_at: datetime | None = None + attempt_started_at: datetime | None = None + error: str | None = None + + +class CatalogService: + """Use only signed API coordinates and activate after all validation succeeds.""" + + def __init__( + self, + root: Path, + base_url: str, + *, + keys: Mapping[str, Ed25519PublicKey] | None = None, + transport: httpx.AsyncBaseTransport | None = None, + ) -> None: + self.root = root + url = httpx.URL(base_url) + if ( + url.scheme not in {"http", "https"} + or not url.host + or url.userinfo + or url.query + or url.fragment + ): + raise CatalogError("The Pullbox API address is invalid.") + self.base_url = str(url).rstrip("/") + self.keys = TRUSTED_KEYS if keys is None else keys + self.transport = transport + self._lock = asyncio.Lock() + self._state_lock = asyncio.Lock() + try: + self._state = CatalogStatus.model_validate(load_json(root / "state.json")) + if self._state.phase in BUSY_PHASES: + self._state.phase = "interrupted" + self._state.error = "The previous download was interrupted. It can be resumed." + except (CatalogError, ValidationError): + self._state = CatalogStatus( + phase="failed", error="Catalog state could not be read. Check the data volume." + ) + # The active generation survives a missing or damaged optional status file. + try: + active = load_json(root / "active.json") + if active: + version, cutoff = str(active["version"]), str(active["source_cutoff_at"]) + self._state.installed_version = version + self._state.source_cutoff_at = cutoff + self._state.requested = True + if self._state.phase == "not_downloaded": + self._state.phase = "current" + except (CatalogError, KeyError): + self._state.phase = "failed" + self._state.error = "Catalog state could not be read. Check the data volume." + + def status(self) -> CatalogStatus: + return self._state.model_copy(deep=True) + + async def set_automatic_updates(self, enabled: bool) -> None: + self._state.automatic_updates = enabled + await self._save() + + async def _save(self) -> None: + async with self._state_lock: + await disk_work( + atomic_json, self.root / "state.json", self._state.model_dump(mode="json") + ) + + async def sync(self, *, manual: bool = False) -> bool: + if self._lock.locked(): + return False + if not manual: + if not self._state.requested or not self._state.automatic_updates: + return False + checked = self._state.last_checked_at + # Allow the daily scheduler's jitter to move earlier than yesterday. + if checked and checked > datetime.now(UTC) - timedelta(hours=23): + return False + async with self._lock: + safe_path(self.root).mkdir(parents=True, exist_ok=True) + lock_path = safe_path(self.root / "update.lock") + with lock_path.open("a+b") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return False + try: + self._state.attempt_started_at = datetime.now(UTC) + await disk_work(cleanup, self.root) + self._state.requested = True + self._state.phase, self._state.error = "checking", None + await self._save() + async with httpx.AsyncClient( + timeout=httpx.Timeout(60, connect=15), + follow_redirects=False, + transport=self.transport, + ) as client: + changed = await self._update(client) + await disk_work(lambda: cleanup(self.root, completed=True)) + self._state.phase = "current" + self._state.last_checked_at = datetime.now(UTC) + await self._save() + logger.info( + "catalog_update_complete", + version=self._state.installed_version, + changed=changed, + ) + return changed + except asyncio.CancelledError: + self._state.phase = "interrupted" + await self._save() + raise + except (CatalogError, httpx.HTTPError, OSError) as exc: + message = ( + str(exc) + if isinstance(exc, CatalogError) + else "Catalog download failed. Check the API connection and " + "available disk space, then retry." + ) + self._state.phase, self._state.error = "failed", message + await self._save() + logger.warning("catalog_update_failed", reason=message) + raise CatalogError(message) from exc + finally: + fcntl.flock(lock, fcntl.LOCK_UN) + + async def _manifest(self, client: httpx.AsyncClient) -> Publication: + cache = await disk_work(load_json, self.root / "manifest.json") + headers = {"Accept-Encoding": "identity"} + cached_raw = cache.get("raw") + if isinstance(cached_raw, str) and isinstance(cache.get("etag"), str): + verify_manifest(cached_raw.encode(), self.keys) + headers["If-None-Match"] = cache["etag"] + async with client.stream( + "GET", self.base_url + "/api/v2/catalog/latest", headers=headers + ) as response: + if response.status_code == 304 and isinstance(cached_raw, str): + return verify_manifest(cached_raw.encode(), self.keys) + response.raise_for_status() + raw = bytearray() + async for chunk in response.aiter_bytes(): + raw.extend(chunk) + if len(raw) > MAX_MANIFEST_BYTES: + raise CatalogError("Catalog manifest is too large.") + publication = verify_manifest(bytes(raw), self.keys) + await disk_work( + atomic_json, + self.root / "manifest.json", + {"raw": raw.decode(), "etag": response.headers.get("etag")}, + ) + return publication + + async def _download(self, client: httpx.AsyncClient, artifact: Artifact) -> Path: + directory = safe_path(self.root / "downloads") + directory.mkdir(parents=True, exist_ok=True) + path = safe_path(directory / f"{artifact.sha256}.part") + offset = path.stat().st_size if path.exists() else 0 + if offset > artifact.size_bytes: + path.unlink() + offset = 0 + if shutil.disk_usage(directory).free < artifact.size_bytes - offset + 64 * 1024 * 1024: + raise CatalogError("Not enough disk space to download the catalog.") + self._state.phase = "downloading" + self._state.bytes_downloaded, self._state.bytes_total = offset, artifact.size_bytes + await self._save() + if offset < artifact.size_bytes: + headers = {"Accept-Encoding": "identity"} + if offset: + headers.update({"Range": f"bytes={offset}-", "If-Range": f'"{artifact.sha256}"'}) + async with client.stream( + "GET", self.base_url + artifact.download_path, headers=headers + ) as response: + response.raise_for_status() + if response.status_code == 206: + expected = f"bytes {offset}-{artifact.size_bytes - 1}/{artifact.size_bytes}" + if response.headers.get("content-range") != expected: + raise CatalogError( + "Catalog resume response is invalid. Retry the download." + ) + elif response.status_code == 200: + offset = 0 + else: + raise CatalogError("Catalog download response is invalid.") + with path.open("ab" if offset else "wb") as stream: + async for chunk in response.aiter_bytes(256 * 1024): + offset += len(chunk) + if offset > artifact.size_bytes: + raise CatalogError("Catalog download exceeded its signed size.") + await disk_work(stream.write, chunk) + self._state.bytes_downloaded = offset + await disk_work(stream.flush) + await disk_work(os.fsync, stream.fileno()) + self._state.phase = "verifying" + if ( + path.stat().st_size != artifact.size_bytes + or await disk_work(file_sha256, path) != artifact.sha256 + ): + path.unlink(missing_ok=True) + raise CatalogError("Catalog download checksum failed. Retry the download.") + return path + + async def _snapshot(self, client: httpx.AsyncClient, artifact: Artifact) -> Path: + target = safe_path(self.root / "bases" / f"{artifact.version}.db") + if target.exists(): + try: + await disk_work(validate_snapshot, target, artifact.version) + return target + except CatalogError: + logger.warning("catalog_weekly_base_invalid", version=artifact.version) + archive = await self._download(client, artifact) + stage = stage_path(self.root, ".db") + try: + self._state.phase = "decompressing" + await disk_work(decompress, archive, stage) + self._state.phase = "verifying" + await disk_work(validate_snapshot, stage, artifact.version) + await disk_work(activate_file, stage, target) + finally: + stage.unlink(missing_ok=True) + return target + + async def _update(self, client: httpx.AsyncClient) -> bool: + publication = await self._manifest(client) + active = await disk_work(load_json, self.root / "active.json") + self._state.target_version = publication.latest_version + if active and str(active.get("version", "")) >= publication.latest_version: + version = valid_version(active.get("version")) + relative = active.get("path") + if relative not in {f"bases/{version}.db", f"versions/{version}.db"}: + raise CatalogError("Catalog active file is invalid. Check the data volume.") + try: + await disk_work(validate_snapshot, self.root / str(relative), version) + return False + except CatalogError: + if version > publication.latest_version: + raise CatalogError( + "The API has an older catalog. The installed version was not replaced." + ) from None + active = {} # Do not replace a good previous reference with a broken one. + base = await self._snapshot(client, publication.full_snapshot) + patch = publication.latest_patch() + target = base + if patch: + if shutil.disk_usage(self.root).free < base.stat().st_size * 2 + 64 * 1024 * 1024: + raise CatalogError("Not enough disk space to apply the catalog update.") + archive = await self._download(client, patch) + unpacked, stage = stage_path(self.root, ".patch.db"), stage_path(self.root, ".db") + try: + self._state.phase = "decompressing" + await disk_work(decompress, archive, unpacked) + self._state.phase = "installing" + await disk_work( + apply_catalog_patch, + base, + unpacked, + stage, + publication.full_snapshot.version, + patch.version, + ) + target = self.root / "versions" / f"{patch.version}.db" + await disk_work(activate_file, stage, target) + finally: + unpacked.unlink(missing_ok=True) + stage.unlink(missing_ok=True) + self._state.phase = "verifying" + manifest = await disk_work(validate_snapshot, target, publication.latest_version) + self._state.phase = "installing" + reference: dict[str, Any] = { + "version": publication.latest_version, + "base_version": publication.full_snapshot.version, + "path": str(target.relative_to(self.root)), + "source_cutoff_at": manifest["source_cutoff_at"], + } + if active: + await disk_work(atomic_json, self.root / "previous.json", active) + await disk_work(atomic_json, self.root / "active.json", reference) + self._state.installed_version = publication.latest_version + self._state.source_cutoff_at = str(manifest["source_cutoff_at"]) + self._state.catalog_size_bytes = target.stat().st_size + self._state.last_updated_at = datetime.now(UTC) + return True + + +@lru_cache(maxsize=4) +def _service(root: Path, base_url: str) -> CatalogService: + return CatalogService(root, base_url) + + +def get_catalog_service() -> CatalogService: + settings = get_settings() + return _service(settings.data_dir / "catalog", settings.pullbox_data_api_base_url) diff --git a/src/pullbox/services/catalog/storage.py b/src/pullbox/services/catalog/storage.py new file mode 100644 index 00000000..3a733789 --- /dev/null +++ b/src/pullbox/services/catalog/storage.py @@ -0,0 +1,116 @@ +"""Bounded artifact decompression and atomic catalog state on the data volume.""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Any, TypeVar + +import zstandard + +from pullbox.services.catalog.contract import CatalogError +from pullbox.services.catalog.database import safe_path + +if TYPE_CHECKING: + from collections.abc import Callable + +T = TypeVar("T") +MAX_DATABASE_BYTES = 4 * 1024**3 + + +async def disk_work[T](func: Callable[..., T], *args: Any) -> T: + """Finish an owned disk operation before cancellation releases the update lock.""" + task = asyncio.create_task(asyncio.to_thread(func, *args)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError: + try: + await task + finally: + raise + + +def atomic_json(path: Path, data: dict[str, Any]) -> None: + safe_path(path) + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, name = tempfile.mkstemp(prefix=".catalog-", dir=path.parent) + stage = Path(name) + try: + with os.fdopen(descriptor, "w") as stream: + json.dump(data, stream, sort_keys=True, separators=(",", ":")) + stream.flush() + os.fsync(stream.fileno()) + os.replace(stage, path) + sync_directory(path.parent) + finally: + stage.unlink(missing_ok=True) + + +def sync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def load_json(path: Path) -> dict[str, Any]: + safe_path(path) + if not path.exists(): + return {} + if path.stat().st_size > 1024 * 1024: + raise CatalogError("Catalog state is invalid. Check the data volume.") + try: + result = json.loads(path.read_bytes()) + except (ValueError, OSError) as exc: + raise CatalogError("Catalog state could not be read. Check the data volume.") from exc + if not isinstance(result, dict): + raise CatalogError("Catalog state is invalid.") + return result + + +def decompress(archive: Path, output: Path) -> None: + safe_path(archive) + safe_path(output) + total = 0 + try: + with archive.open("rb") as source, output.open("xb") as destination: + # The C backend passes this limit to ZSTD_DCtx_setMaxWindowSize in bytes. + with zstandard.ZstdDecompressor(max_window_size=128 * 1024 * 1024).stream_reader( + source + ) as reader: + while chunk := reader.read(1024 * 1024): + total += len(chunk) + if total > MAX_DATABASE_BYTES: + raise CatalogError("Catalog exceeds the supported storage size.") + if ( + total % (32 * 1024 * 1024) == 0 + and shutil.disk_usage(output.parent).free < 64 * 1024 * 1024 + ): + raise CatalogError("Not enough disk space to install the catalog.") + destination.write(chunk) + destination.flush() + os.fsync(destination.fileno()) + except zstandard.ZstdError as exc: + raise CatalogError("Catalog decompression failed. Retry the download.") from exc + + +def stage_path(root: Path, suffix: str) -> Path: + directory = safe_path(root / "staging") + directory.mkdir(parents=True, exist_ok=True) + descriptor, name = tempfile.mkstemp(prefix="catalog-", suffix=suffix, dir=directory) + os.close(descriptor) + path = Path(name) + path.unlink() + return path + + +def activate_file(stage: Path, target: Path) -> None: + safe_path(target) + target.parent.mkdir(parents=True, exist_ok=True) + os.replace(stage, target) + sync_directory(target.parent) diff --git a/src/pullbox/services/comicvine_persistent_cache.py b/src/pullbox/services/comicvine_persistent_cache.py index 5af349e3..6c31fd2a 100644 --- a/src/pullbox/services/comicvine_persistent_cache.py +++ b/src/pullbox/services/comicvine_persistent_cache.py @@ -18,6 +18,7 @@ from pullbox.core.name_matcher import NameMatcher from pullbox.models.provider_cache import MetadataProviderCacheEntry from pullbox.providers.base import IssueMetadata, IssueSummary, SeriesMetadata, SeriesSearchResult +from pullbox.providers.story_arcs import StoryArcSearchResult if TYPE_CHECKING: from collections.abc import Awaitable, Callable @@ -156,6 +157,37 @@ async def fetch() -> tuple[list[SeriesSearchResult], int]: _global_series_search_results_from_payload, ) + async def search_story_arcs_page( + self, + query: str, + *, + limit: int = 20, + offset: int = 0, + ) -> tuple[list[StoryArcSearchResult], int]: + request = { + "query": NameMatcher.normalize(query), + "limit": int(limit), + "offset": int(offset), + } + + async def fetch() -> tuple[list[StoryArcSearchResult], int]: + return cast( + "tuple[list[StoryArcSearchResult], int]", + await self._provider.search_story_arcs_page( + query, + limit=limit, + offset=offset, + ), + ) + + return await self._get_or_fetch( + "search_story_arcs_page", + request, + fetch, + _story_arc_search_results_to_payload, + _story_arc_search_results_from_payload, + ) + async def get_series(self, series_provider_id: str) -> SeriesMetadata: request = {"series_provider_id": str(series_provider_id)} return await self._get_or_fetch( @@ -166,6 +198,52 @@ async def get_series(self, series_provider_id: str) -> SeriesMetadata: _series_metadata_from_payload, ) + async def get_series_batch( + self, + series_provider_ids: list[str], + ) -> dict[str, SeriesMetadata]: + """Resolve series profiles from singular cache rows, batching misses upstream.""" + provider_ids = _ordered_provider_ids(series_provider_ids) + found: dict[str, SeriesMetadata] = {} + missing: list[str] = [] + for provider_id in provider_ids: + request = {"series_provider_id": provider_id} + payload = await self._load_cached_payload("get_series", _cache_key(request)) + if payload is None: + self._stats.misses["get_series"] += 1 + missing.append(provider_id) + else: + self._stats.hits["get_series"] += 1 + found[provider_id] = _series_metadata_from_payload(payload) + + if missing: + started_at = time.monotonic() + batch_fetch = _declared_provider_method(self._provider, "get_series_batch") + if callable(batch_fetch): + fetched = await batch_fetch(missing) + else: + fetched = { + provider_id: await self._provider.get_series(provider_id) + for provider_id in missing + } + self._stats.external_calls["get_series"] += 1 + self._stats.external_duration_ms["get_series"] += (time.monotonic() - started_at) * 1000 + for provider_id in missing: + metadata = fetched.get(provider_id) + if metadata is None: + continue + found[provider_id] = metadata + request = {"series_provider_id": provider_id} + await self._store_payload( + "get_series", + _cache_key(request), + request, + _series_metadata_to_payload(metadata), + ) + return { + provider_id: found[provider_id] for provider_id in provider_ids if provider_id in found + } + async def get_series_cached(self, series_provider_id: str) -> SeriesMetadata | None: """Return cached series metadata without making a provider request.""" request = {"series_provider_id": str(series_provider_id)} @@ -175,6 +253,18 @@ async def get_series_cached(self, series_provider_id: str) -> SeriesMetadata | N self._stats.hits["get_series"] += 1 return _series_metadata_from_payload(payload) + async def refresh_series(self, series_provider_id: str) -> SeriesMetadata: + """Bypass a fresh cache row, then replace it with current provider data.""" + metadata = cast("SeriesMetadata", await self._provider.get_series(series_provider_id)) + request = {"series_provider_id": str(series_provider_id)} + await self._store_payload( + "get_series", + _cache_key(request), + request, + _series_metadata_to_payload(metadata), + ) + return metadata + async def get_issue(self, issue_provider_id: str) -> IssueMetadata: request = {"issue_provider_id": str(issue_provider_id)} return await self._get_or_fetch( @@ -185,6 +275,52 @@ async def get_issue(self, issue_provider_id: str) -> IssueMetadata: _issue_metadata_from_payload, ) + async def get_issue_batch( + self, + issue_provider_ids: list[str], + ) -> dict[str, IssueMetadata]: + """Resolve full issue metadata from singular cache rows and one batch miss path.""" + provider_ids = _ordered_provider_ids(issue_provider_ids) + found: dict[str, IssueMetadata] = {} + missing: list[str] = [] + for provider_id in provider_ids: + request = {"issue_provider_id": provider_id} + payload = await self._load_cached_payload("get_issue", _cache_key(request)) + if payload is None: + self._stats.misses["get_issue"] += 1 + missing.append(provider_id) + else: + self._stats.hits["get_issue"] += 1 + found[provider_id] = _issue_metadata_from_payload(payload) + + if missing: + started_at = time.monotonic() + batch_fetch = _declared_provider_method(self._provider, "get_issue_batch") + if callable(batch_fetch): + fetched = await batch_fetch(missing) + else: + fetched = { + provider_id: await self._provider.get_issue(provider_id) + for provider_id in missing + } + self._stats.external_calls["get_issue"] += 1 + self._stats.external_duration_ms["get_issue"] += (time.monotonic() - started_at) * 1000 + for provider_id in missing: + metadata = fetched.get(provider_id) + if metadata is None: + continue + found[provider_id] = metadata + request = {"issue_provider_id": provider_id} + await self._store_payload( + "get_issue", + _cache_key(request), + request, + _issue_metadata_to_payload(metadata), + ) + return { + provider_id: found[provider_id] for provider_id in provider_ids if provider_id in found + } + async def get_issues_for_series(self, series_provider_id: str) -> list[IssueSummary]: request = {"series_provider_id": str(series_provider_id)} return await self._get_or_fetch( @@ -195,6 +331,82 @@ async def get_issues_for_series(self, series_provider_id: str) -> list[IssueSumm _issue_summaries_from_payload, ) + async def get_issue_catalog_batch( + self, + series_provider_ids: list[str], + ) -> dict[str, list[IssueSummary]]: + """Resolve issue catalogs from singular cache rows, batching missing volumes.""" + provider_ids = _ordered_provider_ids(series_provider_ids) + found: dict[str, list[IssueSummary]] = {} + missing: list[str] = [] + for provider_id in provider_ids: + request = {"series_provider_id": provider_id} + payload = await self._load_cached_payload( + "get_issues_for_series", + _cache_key(request), + ) + if payload is None: + self._stats.misses["get_issues_for_series"] += 1 + missing.append(provider_id) + else: + self._stats.hits["get_issues_for_series"] += 1 + found[provider_id] = _issue_summaries_from_payload(payload) + + if missing: + started_at = time.monotonic() + batch_fetch = _declared_provider_method(self._provider, "get_issue_catalog_batch") + if callable(batch_fetch): + fetched = await batch_fetch(missing) + else: + fetched = { + provider_id: await self._provider.get_issues_for_series(provider_id) + for provider_id in missing + } + self._stats.external_calls["get_issues_for_series"] += 1 + self._stats.external_duration_ms["get_issues_for_series"] += ( + time.monotonic() - started_at + ) * 1000 + for provider_id in missing: + summaries = fetched.get(provider_id) + if summaries is None: + continue + found[provider_id] = summaries + request = {"series_provider_id": provider_id} + await self._store_payload( + "get_issues_for_series", + _cache_key(request), + request, + _issue_summaries_to_payload(summaries), + ) + return { + provider_id: found[provider_id] for provider_id in provider_ids if provider_id in found + } + + async def refresh_issue_catalog(self, series_provider_id: str) -> list[IssueSummary]: + """Bypass a fresh catalog cache row, then replace it atomically.""" + batch_fetch = _declared_provider_method(self._provider, "get_issue_catalog_batch") + if callable(batch_fetch): + summaries = cast( + "list[IssueSummary]", + (await batch_fetch([str(series_provider_id)])).get( + str(series_provider_id), + [], + ), + ) + else: + summaries = cast( + "list[IssueSummary]", + await self._provider.get_issues_for_series(series_provider_id), + ) + request = {"series_provider_id": str(series_provider_id)} + await self._store_payload( + "get_issues_for_series", + _cache_key(request), + request, + _issue_summaries_to_payload(summaries), + ) + return summaries + async def get_issues_for_series_by_numbers( self, series_provider_id: str, @@ -509,6 +721,10 @@ def _cache_key(request: dict[str, Any]) -> str: return hashlib.sha256(canonical.encode("utf-8")).hexdigest() +def _ordered_provider_ids(provider_ids: list[str]) -> list[str]: + return list(dict.fromkeys(str(provider_id) for provider_id in provider_ids)) + + def _clear_inflight( inflight_key: tuple[str, str], completed: asyncio.Task[dict[str, Any]], @@ -549,6 +765,23 @@ def _global_series_search_results_from_payload( return items, int(payload.get("total_results") or len(items)) +def _story_arc_search_results_to_payload( + results: tuple[list[StoryArcSearchResult], int], +) -> dict[str, Any]: + items, total_results = results + return { + "items": [asdict(result) for result in items], + "total_results": int(total_results), + } + + +def _story_arc_search_results_from_payload( + payload: dict[str, Any], +) -> tuple[list[StoryArcSearchResult], int]: + items = [StoryArcSearchResult(**item) for item in payload.get("items", [])] + return items, int(payload.get("total_results") or len(items)) + + def _declared_provider_method(provider: object, name: str) -> Any: """Return provider methods declared on the wrapped type, not dynamic mock attributes.""" if getattr(type(provider), name, None) is None: diff --git a/src/pullbox/services/cover_cache_service.py b/src/pullbox/services/cover_cache_service.py index 343f645d..b188ebdb 100644 --- a/src/pullbox/services/cover_cache_service.py +++ b/src/pullbox/services/cover_cache_service.py @@ -4,6 +4,7 @@ import asyncio import shutil +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING from urllib.parse import urlparse @@ -12,18 +13,34 @@ import structlog from pullbox.config import get_settings +from pullbox.core.exceptions import ConfigurationError +from pullbox.core.library_file_ownership import build_managed_placement_signature from pullbox.services.cover_resolver import resolve_covers_dir if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession from pullbox.models.series import Series + from pullbox.models.story_arc import StoryArc logger = structlog.get_logger(__name__) _SERIES_COVER_LOCKS: dict[int, asyncio.Lock] = {} +_STORY_ARC_COVER_LOCKS: dict[int, asyncio.Lock] = {} _SUPPORTED_COVER_SUFFIXES = frozenset({".jpg", ".jpeg", ".png", ".webp"}) +@dataclass(frozen=True, slots=True) +class ImportedSeriesCoverCacheResult: + """Cached local artwork plus exact directory ownership for import rollback.""" + + path: Path + covers_base: Path + ownership_boundary_path: Path + created_directory_paths: tuple[Path, ...] + artifact_created: bool + artifact_signature: dict[str, int | str] | None + + def find_cover_file(directory: Path, stem: str) -> Path | None: """Look for a cover file with any common image extension.""" for ext in (".jpg", ".jpeg", ".png", ".webp"): @@ -87,7 +104,7 @@ async def cache_imported_series_cover( session: AsyncSession, series: Series, source_path: Path, -) -> Path | None: +) -> ImportedSeriesCoverCacheResult | None: """Copy discovered local artwork into Pullbox's managed cover cache.""" try: source = source_path.expanduser().resolve(strict=True) @@ -98,22 +115,49 @@ async def cache_imported_series_cover( covers_base = await resolve_covers_dir(session) covers_dir = covers_base / str(series.id) + try: + ownership_boundary_path = _nearest_existing_directory(covers_base) + except OSError: + logger.exception( + "imported_series_cover_cache_boundary_invalid", + series_id=series.id, + path=str(covers_base), + ) + return None existing = find_cover_file(covers_dir, "series") if existing is not None: series.cover_path = f"/api/v1/series/{series.id}/cover" - return existing + return ImportedSeriesCoverCacheResult( + path=existing, + covers_base=covers_base, + ownership_boundary_path=ownership_boundary_path, + created_directory_paths=(), + artifact_created=False, + artifact_signature=None, + ) destination = covers_dir / f"series{source.suffix.lower()}" temporary = covers_dir / f".{destination.name}.tmp" - def _copy() -> None: - covers_dir.mkdir(parents=True, exist_ok=True) - shutil.copyfile(source, temporary) - temporary.replace(destination) + def _copy() -> tuple[tuple[Path, ...], dict[str, int | str]]: + created_directories = _create_cover_cache_directories( + covers_dir, + ownership_boundary_path, + ) + try: + shutil.copyfile(source, temporary) + temporary.replace(destination) + artifact_signature = build_managed_placement_signature(destination) + except (ConfigurationError, OSError): + temporary.unlink(missing_ok=True) + destination.unlink(missing_ok=True) + _remove_owned_empty_directories(created_directories) + raise + return created_directories, artifact_signature try: - await asyncio.to_thread(_copy) - except OSError: + created_directory_paths, artifact_signature = await asyncio.to_thread(_copy) + except (ConfigurationError, OSError): logger.exception( "imported_series_cover_cache_failed", series_id=series.id, @@ -122,7 +166,63 @@ def _copy() -> None: return None series.cover_path = f"/api/v1/series/{series.id}/cover" - return destination + return ImportedSeriesCoverCacheResult( + path=destination, + covers_base=covers_base, + ownership_boundary_path=ownership_boundary_path, + created_directory_paths=created_directory_paths, + artifact_created=True, + artifact_signature=artifact_signature, + ) + + +def _nearest_existing_directory(path: Path) -> Path: + """Return the first existing directory above a possibly missing cache base.""" + candidate = path + while not candidate.exists(): + parent = candidate.parent + if parent == candidate: + break + candidate = parent + if not candidate.is_dir(): + raise NotADirectoryError(candidate) + return candidate + + +def _create_cover_cache_directories( + covers_dir: Path, + ownership_boundary_path: Path, +) -> tuple[Path, ...]: + """Create every missing cache segment and report exactly what was created.""" + try: + relative = covers_dir.relative_to(ownership_boundary_path) + except ValueError as exc: + raise OSError("Cover cache directory is outside its ownership boundary") from exc + + created: list[Path] = [] + current = ownership_boundary_path + try: + for segment in relative.parts: + current /= segment + try: + current.mkdir() + except FileExistsError: + if not current.is_dir(): + raise + else: + created.append(current) + except OSError: + _remove_owned_empty_directories(created) + raise + return tuple(created) + + +def _remove_owned_empty_directories(paths: list[Path] | tuple[Path, ...]) -> None: + for directory in reversed(paths): + try: + directory.rmdir() + except (FileNotFoundError, OSError): + continue async def purge_series_cover_cache( @@ -193,3 +293,44 @@ async def cache_series_cover(session: AsyncSession, series: Series) -> Path | No cover_dest.write_bytes(response.content) series.cover_path = f"/api/v1/series/{series.id}/cover" return cover_dest + + +async def resolve_story_arc_cover_file(session: AsyncSession, story_arc: StoryArc) -> Path | None: + """Return a locally cached Story Arc cover if one exists.""" + covers_base = await resolve_covers_dir(session) + return find_cover_file(covers_base / "story_arcs" / str(story_arc.id), "story_arc") + + +async def cache_story_arc_cover(session: AsyncSession, story_arc: StoryArc) -> Path | None: + """Download provider artwork into the managed Story Arc cover cache.""" + from pullbox.services.cover_url_service import story_arc_provider_cover_url + + source_url = story_arc_provider_cover_url(story_arc) + if not source_url: + return None + if not story_arc.cover_url: + story_arc.cover_url = source_url + lock = _STORY_ARC_COVER_LOCKS.setdefault(story_arc.id, asyncio.Lock()) + async with lock: + existing = await resolve_story_arc_cover_file(session, story_arc) + if existing: + story_arc.cover_path = f"/api/v1/story-arcs/{story_arc.id}/cover" + return existing + covers_base = await resolve_covers_dir(session) + covers_dir = covers_base / "story_arcs" / str(story_arc.id) + try: + async with httpx.AsyncClient( + timeout=10.0, + headers={"User-Agent": "Pullbox/1.0"}, + ) as client: + response = await client.get(source_url) + response.raise_for_status() + except httpx.HTTPError: + logger.exception("story_arc_cover_download_failed", story_arc_id=story_arc.id) + return None + suffix = suffix_for_cover(response.headers.get("content-type"), source_url) + cover_dest = covers_dir / f"story_arc{suffix}" + covers_dir.mkdir(parents=True, exist_ok=True) + cover_dest.write_bytes(response.content) + story_arc.cover_path = f"/api/v1/story-arcs/{story_arc.id}/cover" + return cover_dest diff --git a/src/pullbox/services/cover_url_service.py b/src/pullbox/services/cover_url_service.py index 747b8554..24f22ce2 100644 --- a/src/pullbox/services/cover_url_service.py +++ b/src/pullbox/services/cover_url_service.py @@ -41,3 +41,55 @@ def build_series_cover_url(series: object) -> str | None: version = hashlib.sha256(_series_cover_version_key(series).encode("utf-8")).hexdigest()[:12] return f"/api/v1/series/{series_id}/cover?v={version}" + + +def story_arc_provider_cover_url(story_arc: object) -> str | None: + """Return the first-class or legacy provider-snapshot cover URL for an arc.""" + cover_url = getattr(story_arc, "cover_url", None) + if isinstance(cover_url, str) and cover_url: + return cover_url + diagnostics = getattr(story_arc, "diagnostics", None) + if not isinstance(diagnostics, dict): + return None + catalog = diagnostics.get("provider_catalog") + if not isinstance(catalog, dict): + return None + snapshot = catalog.get("snapshot") + if not isinstance(snapshot, dict): + return None + legacy_url = snapshot.get("cover_url") + return legacy_url if isinstance(legacy_url, str) and legacy_url else None + + +def _story_arc_cover_version_key(story_arc: object) -> str: + """Build a stable key that changes when the arc identity or cover changes.""" + updated_at = getattr(story_arc, "updated_at", None) + updated_at_key = "" + if isinstance(updated_at, datetime): + normalized = updated_at if updated_at.tzinfo else updated_at.replace(tzinfo=UTC) + updated_at_key = normalized.astimezone(UTC).isoformat(timespec="microseconds") + return "|".join( + ( + str(getattr(story_arc, "id", "") or ""), + str(getattr(story_arc, "comicvine_id", "") or ""), + str(getattr(story_arc, "name", "") or ""), + story_arc_provider_cover_url(story_arc) or "", + updated_at_key, + ) + ) + + +def build_story_arc_cover_url(story_arc: object) -> str | None: + """Return a versioned Story Arc cover URL suitable for private browser caches.""" + story_arc_id = getattr(story_arc, "id", None) + if not story_arc_id: + return None + cover_path = getattr(story_arc, "cover_path", None) + if not (cover_path or story_arc_provider_cover_url(story_arc)): + return None + if cover_path and not str(cover_path).startswith("/api/v1/story-arcs/"): + return str(cover_path) + version = hashlib.sha256(_story_arc_cover_version_key(story_arc).encode("utf-8")).hexdigest()[ + :12 + ] + return f"/api/v1/story-arcs/{story_arc_id}/cover?v={version}" diff --git a/src/pullbox/services/dashboard_metrics.py b/src/pullbox/services/dashboard_metrics.py index 7e9b131c..21267079 100644 --- a/src/pullbox/services/dashboard_metrics.py +++ b/src/pullbox/services/dashboard_metrics.py @@ -719,7 +719,7 @@ async def load_unmatched_clusters(self) -> tuple[FailureCluster, ...]: detail=f"{count} files still need a clean match.", count=count, cta_label="Review unmatched", - cta_href="/import?tab=unmatched", + cta_href="/import?tab=follow-up", state="watch", ) ) diff --git a/src/pullbox/services/dashboard_priorities.py b/src/pullbox/services/dashboard_priorities.py index 9b851642..985c0607 100644 --- a/src/pullbox/services/dashboard_priorities.py +++ b/src/pullbox/services/dashboard_priorities.py @@ -363,5 +363,5 @@ def build_unmatched_growth_priority( snapshot.computed_at, ), cta_label="Review unmatched", - cta_href="/import?tab=unmatched", + cta_href="/import?tab=follow-up", ) diff --git a/src/pullbox/services/dashboard_storage_path.py b/src/pullbox/services/dashboard_storage_path.py index e8da0361..e1e26b48 100644 --- a/src/pullbox/services/dashboard_storage_path.py +++ b/src/pullbox/services/dashboard_storage_path.py @@ -17,15 +17,20 @@ async def resolve_dashboard_storage_path(session: AsyncSession) -> Path: """Return the filesystem path dashboard storage cards should measure. - The dashboard is library-focused, so prefer the primary enabled library - root. Runtime settings provide a safe fallback for first-run installs before - a database-backed root exists. + The dashboard is library-focused, so prefer the explicit default managed + destination. A managed-capable root is the fallback before a default has + been selected; reference-only roots are measured last. Runtime settings + remain the first-run fallback before a database-backed root exists. """ root_path = ( await session.execute( select(LibraryRoot.path) .where(LibraryRoot.enabled.is_(True)) - .order_by(LibraryRoot.id) + .order_by( + LibraryRoot.is_default_managed_destination.desc(), + LibraryRoot.allow_managed_writes.desc(), + LibraryRoot.id, + ) .limit(1) ) ).scalar_one_or_none() diff --git a/src/pullbox/services/db_check_service.py b/src/pullbox/services/db_check_service.py index 19ca4d65..e1058263 100644 --- a/src/pullbox/services/db_check_service.py +++ b/src/pullbox/services/db_check_service.py @@ -13,9 +13,17 @@ from sqlalchemy import select from pullbox.core.comicinfo_reader import read_comicinfo +from pullbox.core.exceptions import ValidationError +from pullbox.core.library_file_ownership import build_file_identity_signature from pullbox.core.release_parser import normalize_issue_number, parse_release_title from pullbox.models.issue import Issue -from pullbox.models.library import FileFormat, LibraryFile, LibraryRoot, MatchConfidence +from pullbox.models.library import ( + FileFormat, + LibraryFile, + LibraryFileStorageMode, + LibraryRoot, + MatchConfidence, +) from pullbox.models.series import Series if TYPE_CHECKING: @@ -338,6 +346,15 @@ async def register_stale_library_file( "folder": str(file_path.parent), "reason": "File is not inside any configured library root.", } + if not library_root.allow_referenced_registrations: + return { + "file_path": file_path_str, + "folder": str(file_path.parent), + "reason": ( + "The containing library root does not allow referenced registrations. " + "DB Check cannot prove that an untracked file was created by Pullbox." + ), + } parent_folder = normalize_library_path(file_path.parent) if parent_folder is None: @@ -364,13 +381,14 @@ async def register_stale_library_file( confidence = MatchConfidence.UNMATCHED if series is not None and parsed and parsed.issue_number is not None: - issue_result = await session.execute( - select(Issue).where( - Issue.series_id == series.id, - Issue.issue_number == parsed.issue_number, - ) - ) - issue = issue_result.scalar_one_or_none() + issue_filters = [Issue.series_id == series.id] + if parsed.issue_number_text is not None: + issue_filters.append(Issue.issue_number_text == parsed.issue_number_text) + else: + issue_filters.append(Issue.issue_number == parsed.issue_number) + issue_result = await session.execute(select(Issue).where(*issue_filters).limit(2)) + issue_candidates = list(issue_result.scalars().all()) + issue = issue_candidates[0] if len(issue_candidates) == 1 else None if issue is not None: issue_id = issue.id confidence = MatchConfidence.HIGH @@ -399,6 +417,8 @@ async def register_stale_library_file( parsed_year=parsed.year if parsed else None, issue_id=issue_id, library_root_id=library_root.id, + storage_mode=LibraryFileStorageMode.REFERENCED, + source_signature=build_file_identity_signature(file_path), ) session.add(library_file) return None @@ -442,8 +462,9 @@ async def repair_series_path( if next_path is not None: library_file.file_path = next_path library_file.file_name = Path(next_path).name - if target_root is not None: - library_file.library_root_id = target_root.id + repaired_root = resolve_enabled_root_for_path(library_file.file_path, enabled_roots) + if repaired_root is not None: + library_file.library_root_id = repaired_root.id await refresh_library_file_filesystem_fields(library_file) @@ -456,6 +477,11 @@ async def repair_series_root_id( """Repair Series.library_root_id when the path maps to a different root.""" series = await session.get(Series, series_id) if series is not None: + await _require_repair_root_contains_path( + session, + target_root_id=target_root_id, + path_value=series.path, + ) series.library_root_id = target_root_id @@ -469,10 +495,30 @@ async def repair_library_file_root_id( library_file = await session.get(LibraryFile, library_file_id) if library_file is None: return + await _require_repair_root_contains_path( + session, + target_root_id=target_root_id, + path_value=library_file.file_path, + ) library_file.library_root_id = target_root_id await refresh_library_file_filesystem_fields(library_file) +async def _require_repair_root_contains_path( + session: AsyncSession, + *, + target_root_id: int, + path_value: str | Path | None, +) -> LibraryRoot: + """Validate client-carried DB Check repair context against live root identity.""" + root = await session.get(LibraryRoot, target_root_id) + if root is None or not root.enabled: + raise ValidationError("The DB Check repair target library root is unavailable.") + if resolve_enabled_root_for_path(path_value, [root]) is None: + raise ValidationError("The DB Check repair target root does not contain the record path.") + return root + + async def reindex_library_root( session: AsyncSession, *, diff --git a/src/pullbox/services/diagnostic_db_snapshot.py b/src/pullbox/services/diagnostic_db_snapshot.py index 4ab3151d..cf95bdc9 100644 --- a/src/pullbox/services/diagnostic_db_snapshot.py +++ b/src/pullbox/services/diagnostic_db_snapshot.py @@ -29,6 +29,18 @@ def create_sanitized_db_copy(db_path: Path) -> bytes | None: src.backup(dst) src.close() + tables = { + str(row[0]) + for row in dst.execute( + "SELECT name FROM sqlite_schema WHERE type='table'" + ).fetchall() + } + if "audit_logs" in tables: + # Keep operational evidence while removing its private account link. + dst.execute("UPDATE audit_logs SET user_id = NULL") + if "issue_reader_states" in tables: + # Per-user reading state has no meaning once users are removed. + dst.execute("DELETE FROM issue_reader_states") dst.execute("DELETE FROM users") dst.execute("DELETE FROM api_keys") diff --git a/src/pullbox/services/diagnostic_import_collectors.py b/src/pullbox/services/diagnostic_import_collectors.py new file mode 100644 index 00000000..604ac429 --- /dev/null +++ b/src/pullbox/services/diagnostic_import_collectors.py @@ -0,0 +1,537 @@ +"""Bounded aggregate diagnostics for imports and first-class Story Arcs.""" + +from __future__ import annotations + +import enum +import math +import re +import resource +import sys +from collections import defaultdict +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, or_, select + +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobAction, + ImportJobLog, + ImportSeriesStatus, +) +from pullbox.models.story_arc import IssueStoryArc, StoryArc, StoryArcPlacement +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.models.story_arc_sync import StoryArcSyncWork +from pullbox.services.import_safety_diagnostics import summarize_import_safety_failures + +if TYPE_CHECKING: + from collections.abc import Sequence + from datetime import datetime + + from sqlalchemy.ext.asyncio import AsyncSession + +MAX_GROUPS = 32 +MAX_RECENT_JOBS = 100 +MAX_STEP2_TIMING_ROWS = 100 +MAX_CANCEL_EVENT_ROWS = 400 +MAX_SAFETY_ROWS = 500 +MAX_FAILED_ISSUE_NUMBER_ROWS = 50 + +_SAFE_CLASS_RE = re.compile(r"^[a-z0-9][a-z0-9_.:-]{0,79}$") +_FAILED_RESOLUTION_STATES = ("missing", "ambiguous", "conflict") +_CANCEL_TERMINAL_EVENTS = frozenset({"import_scan_cancelled", "import_cancelled_after_rollback"}) +_STEP2_FIELDS = { + "scan_duration_ms": "scan", + "analyze_duration_ms": "analyze", + "series_matching_duration_ms": "series_matching", + "file_matching_duration_ms": "file_matching", + "total_duration_ms": "total", +} + + +def _safe_group_key(value: object) -> str: + if value is None: + return "unset" + if isinstance(value, enum.Enum): + value = value.value + if isinstance(value, bool): + return "true" if value else "false" + normalized = str(value).strip().lower() + return normalized if _SAFE_CLASS_RE.fullmatch(normalized) else "other" + + +async def _group_counts( + session: AsyncSession, + model: type[Any], + column: Any, + *, + predicates: Sequence[Any] = (), + limit: int = MAX_GROUPS, +) -> dict[str, int]: + """Count values with bounded response cardinality and stable safe labels.""" + count_expression = func.count() + statement = ( + select(column, count_expression) + .select_from(model) + .where(*predicates) + .group_by(column) + .order_by(count_expression.desc(), column.asc()) + .limit(limit + 1) + ) + rows = list((await session.execute(statement)).all()) + counts: dict[str, int] = {} + for raw_key, raw_count in rows[:limit]: + key = _safe_group_key(raw_key) + counts[key] = counts.get(key, 0) + int(raw_count or 0) + if len(rows) > limit: + total = int( + await session.scalar(select(func.count()).select_from(model).where(*predicates)) or 0 + ) + counts["other"] = counts.get("other", 0) + max(total - sum(counts.values()), 0) + return dict(sorted(counts.items())) + + +async def _count( + session: AsyncSession, + model: type[Any], + *, + predicates: Sequence[Any] = (), +) -> int: + return int( + await session.scalar(select(func.count()).select_from(model).where(*predicates)) or 0 + ) + + +def _duration_ms(start: datetime | None, end: datetime | None) -> int | None: + if start is None or end is None or end < start: + return None + return round((end - start).total_seconds() * 1000) + + +def _percentile(values: Sequence[int], percentile: float) -> int | None: + if not values: + return None + ordered = sorted(values) + index = max(math.ceil(percentile * len(ordered)) - 1, 0) + return ordered[index] + + +def _timing_summary(values: Sequence[int]) -> dict[str, int | None]: + if not values: + return { + "sample_count": 0, + "minimum": None, + "p50": None, + "p95": None, + "average": None, + "maximum": None, + } + return { + "sample_count": len(values), + "minimum": min(values), + "p50": _percentile(values, 0.50), + "p95": _percentile(values, 0.95), + "average": round(sum(values) / len(values)), + "maximum": max(values), + } + + +async def _collect_performance(session: AsyncSession) -> dict[str, object]: + job_rows = list( + ( + await session.execute( + select( + ImportJob.id, + ImportJob.scan_started_at, + ImportJob.scan_completed_at, + ImportJob.match_started_at, + ImportJob.match_completed_at, + ImportJob.import_started_at, + ImportJob.import_completed_at, + ) + .order_by(ImportJob.id.desc()) + .limit(MAX_RECENT_JOBS) + ) + ).all() + ) + durations: dict[str, list[int]] = defaultdict(list) + job_ids: list[int] = [] + for job_row in job_rows: + job_ids.append(int(job_row.id)) + for name, value in ( + ("scan", _duration_ms(job_row.scan_started_at, job_row.scan_completed_at)), + ("matching", _duration_ms(job_row.match_started_at, job_row.match_completed_at)), + ("import", _duration_ms(job_row.import_started_at, job_row.import_completed_at)), + ("total", _duration_ms(job_row.scan_started_at, job_row.import_completed_at)), + ): + if value is not None: + durations[name].append(value) + + step2_values: dict[str, list[int]] = defaultdict(list) + step2_rows = list( + ( + await session.scalars( + select(ImportJobLog.data) + .where(ImportJobLog.event == "import_step2_timing") + .order_by(ImportJobLog.id.desc()) + .limit(MAX_STEP2_TIMING_ROWS) + ) + ).all() + ) + for data in step2_rows: + if not isinstance(data, dict): + continue + for source_key, output_key in _STEP2_FIELDS.items(): + value = data.get(source_key) + if isinstance(value, int | float) and not isinstance(value, bool) and value >= 0: + step2_values[output_key].append(round(value)) + + cancellation_values: list[int] = [] + cancellation_rows_truncated = False + if job_ids: + cancel_rows = list( + ( + await session.execute( + select( + ImportJobLog.id, + ImportJobLog.import_job_id, + ImportJobLog.event, + ImportJobLog.logged_at, + ) + .where( + ImportJobLog.import_job_id.in_(job_ids), + ImportJobLog.event.in_( + ("import_cancel_requested", *_CANCEL_TERMINAL_EVENTS) + ), + ) + .order_by(ImportJobLog.id.desc()) + .limit(MAX_CANCEL_EVENT_ROWS + 1) + ) + ).all() + ) + cancellation_rows_truncated = len(cancel_rows) > MAX_CANCEL_EVENT_ROWS + events_by_job: dict[int, list[Any]] = defaultdict(list) + for cancel_row in cancel_rows[:MAX_CANCEL_EVENT_ROWS]: + events_by_job[int(cancel_row.import_job_id)].append(cancel_row) + for events in events_by_job.values(): + requested_at: datetime | None = None + for event in sorted(events, key=lambda item: (item.logged_at, item.id)): + if event.event == "import_cancel_requested": + requested_at = event.logged_at + elif requested_at is not None and event.event in _CANCEL_TERMINAL_EVENTS: + latency = _duration_ms(requested_at, event.logged_at) + if latency is not None: + cancellation_values.append(latency) + requested_at = None + + return { + "recent_job_window": { + "limit": MAX_RECENT_JOBS, + "jobs_sampled": len(job_rows), + }, + "stage_duration_ms": { + name: _timing_summary(durations.get(name, [])) + for name in ("scan", "matching", "import", "total") + }, + "recorded_step2_duration_ms": { + name: _timing_summary(step2_values.get(name, [])) + for name in ("scan", "analyze", "series_matching", "file_matching", "total") + }, + "cancellation_latency_ms": { + **_timing_summary(cancellation_values), + "event_rows_truncated": cancellation_rows_truncated, + }, + "resource_snapshot": _resource_snapshot(), + } + + +def _resource_snapshot() -> dict[str, int | None]: + process_rss: int | None = None + database_size: int | None = None + wal_size: int | None = None + try: + rss = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + process_rss = rss if sys.platform == "darwin" else rss * 1024 + except Exception: + pass + try: + from pullbox.config import get_settings + + db_url = get_settings().db_url + if ":///" in db_url: + db_path = Path(db_url.split(":///", 1)[1]) + database_size = db_path.stat().st_size if db_path.is_file() else None + wal_path = db_path.with_name(f"{db_path.name}-wal") + wal_size = wal_path.stat().st_size if wal_path.is_file() else None + except Exception: + pass + return { + "process_peak_rss_bytes": process_rss, + "database_file_size_bytes": database_size, + "sqlite_wal_size_bytes": wal_size, + } + + +async def _collect_safety(session: AsyncSession) -> dict[str, object]: + rows = list( + ( + await session.scalars( + select(ImportedFile.diagnostics) + .where(ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED) + .order_by(ImportedFile.id.desc()) + .limit(MAX_SAFETY_ROWS + 1) + ) + ).all() + ) + sampled = rows[:MAX_SAFETY_ROWS] + failures: list[tuple[str, dict[str, object]]] = [] + for diagnostics in sampled: + if not isinstance(diagnostics, dict): + continue + safety_block = diagnostics.get("safety_block") + if isinstance(safety_block, dict): + failures.append(("", safety_block)) + return { + "rows_sampled": len(sampled), + "sample_limit": MAX_SAFETY_ROWS, + "sample_truncated": len(rows) > MAX_SAFETY_ROWS, + "categories": summarize_import_safety_failures(failures, example_limit=0), + } + + +def _safe_issue_number(value: object) -> str | None: + text = str(value).strip() + if not text or len(text) > 320 or any(ord(char) < 32 for char in text): + return None + if text.startswith(("/", "\\", "~")) or "://" in text or ".." in text: + return None + return text + + +async def _collect_failed_issue_numbers(session: AsyncSession) -> dict[str, object]: + staged = list( + ( + await session.execute( + select( + ImportedStoryArcEntry.source_issue_number_text, + ImportedStoryArcEntry.resolution_state, + ) + .where( + ImportedStoryArcEntry.source_issue_number_text.is_not(None), + ImportedStoryArcEntry.resolution_state.in_(_FAILED_RESOLUTION_STATES), + ) + .order_by(ImportedStoryArcEntry.id.desc()) + .limit(MAX_FAILED_ISSUE_NUMBER_ROWS + 1) + ) + ).all() + ) + canonical = list( + ( + await session.execute( + select(IssueStoryArc.source_issue_number_text, IssueStoryArc.resolution_state) + .where( + IssueStoryArc.source_issue_number_text.is_not(None), + IssueStoryArc.resolution_state.in_(_FAILED_RESOLUTION_STATES), + ) + .order_by(IssueStoryArc.id.desc()) + .limit(MAX_FAILED_ISSUE_NUMBER_ROWS + 1) + ) + ).all() + ) + counts: dict[tuple[str, str], int] = defaultdict(int) + for raw_number, raw_state in ( + staged[:MAX_FAILED_ISSUE_NUMBER_ROWS] + canonical[:MAX_FAILED_ISSUE_NUMBER_ROWS] + ): + number = _safe_issue_number(raw_number) + if number is not None: + counts[(number, _safe_group_key(raw_state))] += 1 + ordered = sorted(counts.items(), key=lambda item: (-item[1], item[0][0], item[0][1])) + items = [ + {"issue_number": key[0], "resolution_state": key[1], "count": count} + for key, count in ordered[:MAX_FAILED_ISSUE_NUMBER_ROWS] + ] + return { + "sample_limit": MAX_FAILED_ISSUE_NUMBER_ROWS, + "sample_truncated": ( + len(staged) > MAX_FAILED_ISSUE_NUMBER_ROWS + or len(canonical) > MAX_FAILED_ISSUE_NUMBER_ROWS + or len(ordered) > MAX_FAILED_ISSUE_NUMBER_ROWS + ), + "items": items, + } + + +async def collect_import_story_arc_diagnostics(session: AsyncSession) -> dict[str, object]: + """Return bounded counts and timing evidence without row-level private data.""" + import_total = await _count(session, ImportJob) + canonical_arc_total = await _count(session, StoryArc) + staged_arc_total = await _count(session, ImportedStoryArc) + placement_total = await _count(session, StoryArcPlacement) + action_total = await _count(session, ImportJobAction) + sync_total = await _count(session, StoryArcSyncWork) + + failure_event_predicates = ( + or_( + ImportJobLog.level == "ERROR", + ImportJobLog.event.like("%failed%"), + ImportJobLog.event.like("%retry%"), + ), + ) + return { + "schema_version": 1, + "bounds": { + "group_limit": MAX_GROUPS, + "recent_job_limit": MAX_RECENT_JOBS, + "step2_timing_limit": MAX_STEP2_TIMING_ROWS, + "cancellation_event_limit": MAX_CANCEL_EVENT_ROWS, + "safety_row_limit": MAX_SAFETY_ROWS, + "failed_issue_number_limit": MAX_FAILED_ISSUE_NUMBER_ROWS, + }, + "imports": { + "jobs": { + "total": import_total, + "by_status": await _group_counts(session, ImportJob, ImportJob.status), + "by_source_type": await _group_counts(session, ImportJob, ImportJob.source_type), + }, + "series": { + "total": await _count(session, ImportedSeries), + "by_status": await _group_counts(session, ImportedSeries, ImportedSeries.status), + }, + "files": { + "total": await _count(session, ImportedFile), + "by_status": await _group_counts(session, ImportedFile, ImportedFile.status), + }, + "policy_modes": { + "file_handling": await _group_counts( + session, ImportJob, ImportJob.file_handling_mode + ), + "effective_import_strategy": await _group_counts( + session, ImportJob, ImportJob.effective_import_strategy + ), + "effective_transfer_method": await _group_counts( + session, ImportJob, ImportJob.effective_transfer_method + ), + "source_preserved": await _group_counts( + session, ImportJob, ImportJob.source_preserved + ), + "story_arc_import_requested": await _group_counts( + session, ImportJob, ImportJob.story_arc_import_requested + ), + "story_arc_materialization_requested": await _group_counts( + session, ImportJob, ImportJob.story_arc_materialization_requested + ), + }, + "safety": await _collect_safety(session), + }, + "story_arcs": { + "canonical": { + "total": canonical_arc_total, + "by_lifecycle": await _group_counts(session, StoryArc, StoryArc.lifecycle), + "by_source_kind": await _group_counts(session, StoryArc, StoryArc.source_kind), + "sync_enabled": await _group_counts(session, StoryArc, StoryArc.sync_enabled), + }, + "canonical_entries": { + "total": await _count(session, IssueStoryArc), + "by_resolution_state": await _group_counts( + session, IssueStoryArc, IssueStoryArc.resolution_state + ), + }, + "staged": { + "total": staged_arc_total, + "by_status": await _group_counts( + session, ImportedStoryArc, ImportedStoryArc.status + ), + "by_source_kind": await _group_counts( + session, ImportedStoryArc, ImportedStoryArc.source_kind + ), + }, + "staged_entries": { + "total": await _count(session, ImportedStoryArcEntry), + "by_resolution_state": await _group_counts( + session, + ImportedStoryArcEntry, + ImportedStoryArcEntry.resolution_state, + ), + }, + "placements": { + "total": placement_total, + "by_state": await _group_counts( + session, StoryArcPlacement, StoryArcPlacement.state + ), + "by_mode": await _group_counts(session, StoryArcPlacement, StoryArcPlacement.mode), + "by_ownership": await _group_counts( + session, StoryArcPlacement, StoryArcPlacement.ownership + ), + }, + "failed_issue_numbers": await _collect_failed_issue_numbers(session), + }, + "performance": await _collect_performance(session), + "recovery": { + "control_requests": await _group_counts(session, ImportJob, ImportJob.control_request), + "jobs_with_story_arc_followup_pending": await _count( + session, + ImportJob, + predicates=(ImportJob.story_arc_placement_followup_pending.is_(True),), + ), + "jobs_waiting_for_story_arc_rollback": await _count( + session, + ImportJob, + predicates=(ImportJob.story_arc_rollback_waiting_work_id.is_not(None),), + ), + "series_pending_recovery": await _count( + session, + ImportedSeries, + predicates=(ImportedSeries.status == ImportSeriesStatus.RECOVERY_PENDING,), + ), + "action_journal": { + "total": action_total, + "by_status": await _group_counts(session, ImportJobAction, ImportJobAction.status), + "by_phase": await _group_counts(session, ImportJobAction, ImportJobAction.phase), + "by_action_type": await _group_counts( + session, ImportJobAction, ImportJobAction.action_type + ), + }, + "story_arc_sync_work": { + "total": sync_total, + "by_state": await _group_counts(session, StoryArcSyncWork, StoryArcSyncWork.state), + "by_reason": await _group_counts( + session, StoryArcSyncWork, StoryArcSyncWork.reason + ), + "claimable": await _group_counts( + session, StoryArcSyncWork, StoryArcSyncWork.claimable + ), + "cancel_requested": await _count( + session, + StoryArcSyncWork, + predicates=(StoryArcSyncWork.cancel_requested_at.is_not(None),), + ), + "attempt_count_total": int( + await session.scalar(select(func.sum(StoryArcSyncWork.attempt_count))) or 0 + ), + "attempt_count_maximum": int( + await session.scalar(select(func.max(StoryArcSyncWork.attempt_count))) or 0 + ), + "failure_categories": await _group_counts( + session, + StoryArcSyncWork, + StoryArcSyncWork.last_error_category, + predicates=(StoryArcSyncWork.last_error_category.is_not(None),), + ), + "failure_codes": await _group_counts( + session, + StoryArcSyncWork, + StoryArcSyncWork.last_error_code, + predicates=(StoryArcSyncWork.last_error_code.is_not(None),), + ), + }, + "import_failure_event_classes": await _group_counts( + session, + ImportJobLog, + ImportJobLog.event, + predicates=failure_event_predicates, + ), + }, + } diff --git a/src/pullbox/services/diagnostic_service.py b/src/pullbox/services/diagnostic_service.py index a0b1a115..b43be2ae 100644 --- a/src/pullbox/services/diagnostic_service.py +++ b/src/pullbox/services/diagnostic_service.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from datetime import UTC, datetime from pathlib import Path from typing import TYPE_CHECKING @@ -11,6 +12,9 @@ from pullbox.services.diagnostic_db_snapshot import ( create_sanitized_db_copy as _create_sanitized_db_copy, ) +from pullbox.services.diagnostic_import_collectors import ( + collect_import_story_arc_diagnostics as _collect_import_story_arc_diagnostics, +) from pullbox.services.diagnostic_log_collector import ( MAX_LOG_FILE_BYTES as _MAX_LOG_FILE_BYTES, # noqa: F401 ) @@ -48,6 +52,7 @@ from pullbox.services.diagnostic_utility_collectors import ( collect_utility_jobs as _collect_utility_jobs, ) +from pullbox.services.import_mylar3_path_reports import latest_report as _collect_mylar_path_report if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -397,9 +402,9 @@ async def create_diagnostic_package(session: AsyncSession) -> tuple[bytes, str]: # Collect all data system_info = await _collect_system_info() bootstrap_settings = await _collect_bootstrap_settings() - container_runtime = _collect_container_runtime() + container_runtime = await asyncio.to_thread(_collect_container_runtime) config = await _collect_config(session) - config_xml_snapshot = _collect_config_xml_snapshot() + config_xml_snapshot = await asyncio.to_thread(_collect_config_xml_snapshot) health = await _collect_health_status(session) health_history = await _collect_health_history(session) health_incidents = await _collect_health_incidents(session) @@ -413,6 +418,8 @@ async def create_diagnostic_package(session: AsyncSession) -> tuple[bytes, str]: disk_permissions = await _collect_disk_and_permissions(session) runtime_info = await _collect_runtime_info(session) import_jobs = await _collect_import_jobs(session) + import_story_arc_diagnostics = await _collect_import_story_arc_diagnostics(session) + mylar_path_report = await asyncio.to_thread(_collect_mylar_path_report) utility_jobs = await _collect_utility_jobs(session) utility_job_logs = await _collect_utility_job_logs( session, @@ -420,7 +427,7 @@ async def create_diagnostic_package(session: AsyncSession) -> tuple[bytes, str]: ) # Get logs directory from runtime settings - log_files = _collect_log_files(get_settings().logs_dir) + log_files = await asyncio.to_thread(_collect_log_files, get_settings().logs_dir) # Create sanitized database copy db_copy: bytes | None = None @@ -428,12 +435,13 @@ async def create_diagnostic_package(session: AsyncSession) -> tuple[bytes, str]: settings = get_settings() if ":///" in settings.db_url: db_path = Path(settings.db_url.split(":///", 1)[1]) - db_copy = _create_sanitized_db_copy(db_path) + db_copy = await asyncio.to_thread(_create_sanitized_db_copy, db_path) except Exception: logger.warning("diagnostic_db_snapshot_skipped", exc_info=True) binary_artifacts = [config_xml_snapshot] if config_xml_snapshot is not None else [] - zip_bytes = build_diagnostic_zip( + zip_bytes = await asyncio.to_thread( + build_diagnostic_zip, prefix=prefix, json_artifacts={ "system_info.json": system_info, @@ -453,6 +461,8 @@ async def create_diagnostic_package(session: AsyncSession) -> tuple[bytes, str]: "disk_and_permissions.json": disk_permissions, "runtime_info.json": runtime_info, "import_jobs.json": import_jobs, + "import_story_arc_diagnostics.json": import_story_arc_diagnostics, + "mylar_path_preflight.json": mylar_path_report, "utility_jobs.json": utility_jobs, "utility_job_logs.json": utility_job_logs, }, diff --git a/src/pullbox/services/diagnostic_storage_collectors.py b/src/pullbox/services/diagnostic_storage_collectors.py index 15f94458..60cecd0c 100644 --- a/src/pullbox/services/diagnostic_storage_collectors.py +++ b/src/pullbox/services/diagnostic_storage_collectors.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import os import shutil from pathlib import Path @@ -54,6 +55,11 @@ async def collect_disk_and_permissions(session: AsyncSession) -> dict[str, objec except Exception: pass + return await asyncio.to_thread(_probe_directories, dirs) + + +def _probe_directories(dirs: dict[str, Path]) -> dict[str, object]: + """Probe roots only; diagnostics must never enumerate the user's collection.""" output: dict[str, object] = {} for name, path in dirs.items(): info: dict[str, object] = {"path": str(path)} @@ -75,12 +81,11 @@ async def collect_disk_and_permissions(session: AsyncSession) -> dict[str, objec except OSError: pass - if path.exists() and path.is_dir(): - try: - total_size = sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) - info["dir_size_bytes"] = total_size - except (OSError, PermissionError): - pass + info["dir_size_bytes"] = None + info["dir_size_status"] = "not_collected" + info["dir_size_reason"] = ( + "Recursive directory sizing is omitted to keep diagnostics independent of library size." + ) output[name] = info diff --git a/src/pullbox/services/direct_acquisition_executor.py b/src/pullbox/services/direct_acquisition_executor.py index c7c4102d..7a3009ff 100644 --- a/src/pullbox/services/direct_acquisition_executor.py +++ b/src/pullbox/services/direct_acquisition_executor.py @@ -84,6 +84,7 @@ from pullbox.services.post_processing_operation_progress import ( project_post_processing_operation_progress, ) +from pullbox.services.story_arc_sync_queue import request_story_arc_sync_now from pullbox.tasks.post_processing_progress import ( PostProcessingPhase, _clear_post_processing, @@ -681,6 +682,16 @@ async def _post_process( force=True, final_path=str(processed.final_path), ) + try: + request_story_arc_sync_now() + except Exception: + # The completed progress write committed canonical state. Automatic + # arc placement is durable, optional follow-up work. + logger.warning( + "story_arc_sync_trigger_failed_after_direct_completion", + acquisition_id=acquisition_id, + exc_info=True, + ) return _result(attempt, artifact) async def _pause( diff --git a/src/pullbox/services/direct_artifact_pack.py b/src/pullbox/services/direct_artifact_pack.py index 327d794c..ff03b0ab 100644 --- a/src/pullbox/services/direct_artifact_pack.py +++ b/src/pullbox/services/direct_artifact_pack.py @@ -6,6 +6,7 @@ from pullbox.core.archive import ArchiveError, ArchiveReader from pullbox.core.file_safety import has_archive_member_path_traversal +from pullbox.core.issue_numbers import format_issue_number, parse_issue_number_text from pullbox.core.name_matcher import NameMatcher from pullbox.core.release_parser import parse_release_title @@ -48,7 +49,7 @@ def extract_same_series_issue_files( destination: Path, expected_issue_numbers: frozenset[str], expected_series_titles: frozenset[str], -) -> dict[float, Path]: +) -> dict[str, Path]: """Extract separately packaged issues from one contiguous direct-download pack. A pack with page images but no nested comic files is one combined comic file. @@ -94,7 +95,7 @@ def extract_same_series_issue_files( ) destination.mkdir(mode=0o700, parents=True, exist_ok=True) - extracted: dict[float, Path] = {} + extracted: dict[str, Path] = {} for member in candidates: if has_archive_member_path_traversal(member.name): raise DirectArtifactPackError( @@ -103,7 +104,11 @@ def extract_same_series_issue_files( ) parsed = parse_release_title(Path(member.name).name) issue_number = parsed.issue_number if parsed is not None else None - if issue_number is None or issue_number not in expected: + issue_number_text = parsed.issue_number_text if parsed is not None else None + if issue_number is None: + continue + exact_issue_number = issue_number_text or format_issue_number(issue_number) + if exact_issue_number not in expected: continue parsed_series_title = NameMatcher.normalize(parsed.series_name or "") if parsed else "" if parsed_series_title not in normalized_series_titles: @@ -111,13 +116,13 @@ def extract_same_series_issue_files( code="direct_pack_mixed_series", message="The direct-download pack contains files for a different series.", ) - if issue_number in extracted: + if exact_issue_number in extracted: raise DirectArtifactPackError( code="direct_pack_ambiguous_issue", message="The direct-download pack contains more than one file for the same issue.", ) suffix = Path(member.name).suffix.lower() - target = destination / f"issue-{_issue_path_token(issue_number)}{suffix}" + target = destination / f"issue-{_issue_path_token(exact_issue_number)}{suffix}" try: target.write_bytes(reader.read_file(member.name, max_bytes=member.size)) target.chmod(0o600) @@ -126,7 +131,7 @@ def extract_same_series_issue_files( code="direct_pack_extract_failed", message="Pullbox could not safely extract the direct-download pack.", ) from exc - extracted[issue_number] = target + extracted[exact_issue_number] = target missing = expected - set(extracted) if missing: @@ -137,11 +142,12 @@ def extract_same_series_issue_files( return extracted -def _normalized_issue_numbers(issue_numbers: frozenset[str]) -> set[float]: - normalized: set[float] = set() +def _normalized_issue_numbers(issue_numbers: frozenset[str]) -> set[str]: + normalized: set[str] = set() for value in issue_numbers: try: - normalized.add(float(value)) + _, exact_text = parse_issue_number_text(value) + normalized.add(exact_text) except (TypeError, ValueError) as exc: raise DirectArtifactPackError( code="direct_pack_coverage_invalid", @@ -150,8 +156,16 @@ def _normalized_issue_numbers(issue_numbers: frozenset[str]) -> set[float]: return normalized -def _issue_path_token(issue_number: float) -> str: - return str(issue_number).replace(".", "_") +def _issue_path_token(issue_number: str | float) -> str: + try: + _, exact_text = parse_issue_number_text(issue_number) + except ValueError: + exact_text = ( + format_issue_number(issue_number) + if isinstance(issue_number, float) + else str(issue_number) + ) + return exact_text.replace(".", "_") def _issue_number_from_member(name: str) -> float | None: diff --git a/src/pullbox/services/direct_artifact_post_processing.py b/src/pullbox/services/direct_artifact_post_processing.py index d901d291..5b81c0ef 100644 --- a/src/pullbox/services/direct_artifact_post_processing.py +++ b/src/pullbox/services/direct_artifact_post_processing.py @@ -14,6 +14,7 @@ from sqlalchemy import select from sqlalchemy.orm import joinedload +from pullbox.core.issue_numbers import format_issue_number from pullbox.models.download import DownloadState from pullbox.models.issue import Issue, IssueStatus from pullbox.models.library import LibraryFile @@ -168,10 +169,15 @@ async def run_direct_artifact_pack_post_processing( ) .where(Issue.series_id == initiating_issue.series_id) ) - issue_by_number = {issue.issue_number: issue for issue in issues_result.unique().scalars()} + issue_by_number = { + ( + getattr(issue, "issue_number_text", None) or format_issue_number(issue.issue_number) + ): issue + for issue in issues_result.unique().scalars() + } prepared_imports = [] - for issue_number, file_path in sorted(extracted_paths.items()): - candidate = issue_by_number.get(issue_number) + for issue_number_text, file_path in sorted(extracted_paths.items()): + candidate = issue_by_number.get(issue_number_text) if candidate is None: continue has_existing_file = getattr(candidate, "library_file", None) is not None diff --git a/src/pullbox/services/direct_search_coordinator.py b/src/pullbox/services/direct_search_coordinator.py index a4009abf..a5bd31fa 100644 --- a/src/pullbox/services/direct_search_coordinator.py +++ b/src/pullbox/services/direct_search_coordinator.py @@ -226,7 +226,7 @@ async def persist_direct_search_discoveries( """Persist redacted candidate evidence and return server-issued IDs.""" pending: list[tuple[DirectAcquisitionAttempt, DirectValidatedCandidate, bool]] = [] fingerprint_groups: list[tuple[DirectAcquisitionAttempt, list[DirectAcquisitionAttempt]]] = [] - issue_number = f"{target.issue_number:g}" + issue_number = target.effective_issue_number_text volume = _target_volume(target) for primary in (*outcome.matched, *outcome.rejected): group_attempts: list[DirectAcquisitionAttempt] = [] @@ -487,6 +487,7 @@ def validate( wanted_series=target.series_title, wanted_issue=target.issue_number, wanted_year=target.search_year, + year_context=target.year_context, wanted_issue_type=target.issue_type, alternate_names=_validation_alternate_names(target, candidate), wanted_issue_title=target.issue_title, @@ -520,7 +521,7 @@ def _is_explicit_direct_pack_for_target( candidate: DirectCandidate, target: IssueSearchTarget, ) -> bool: - target_issue = f"{target.issue_number:g}" + target_issue = target.effective_issue_number_text return ( len(candidate.parsed.issue_numbers) > 1 and target_issue in candidate.parsed.issue_numbers ) @@ -532,7 +533,7 @@ def _direct_pack_member_title( ) -> str: year = candidate.parsed.year or target.search_year suffix = f" ({year})" if year is not None else "" - return f"{candidate.parsed.series_title} #{target.issue_number:g}{suffix}" + return f"{candidate.parsed.series_title} #{target.effective_issue_number_text}{suffix}" def _validation_alternate_names( @@ -609,7 +610,7 @@ def _resolver_retry_allowed(exc: DirectProviderClientError) -> bool: def _build_intent(target: IssueSearchTarget) -> DirectSearchIntent: - issue_number = f"{target.issue_number:g}" + issue_number = target.effective_issue_number_text return DirectSearchIntent( series_title=target.series_title, normalized_title=NameMatcher.normalize(target.series_title), @@ -630,7 +631,7 @@ def _target_volume(target: IssueSearchTarget) -> str | None: """Map collection issue numbering onto provider volume coverage.""" if issue_type_family(target.issue_type) is not TypeFamily.COLLECTION: return None - return collection_title_number(target.issue_title) or f"{target.issue_number:g}" + return collection_title_number(target.issue_title) or target.effective_issue_number_text def _candidate_release( diff --git a/src/pullbox/services/download_service.py b/src/pullbox/services/download_service.py index 9b44af11..7363817c 100644 --- a/src/pullbox/services/download_service.py +++ b/src/pullbox/services/download_service.py @@ -445,10 +445,20 @@ async def add_torrent_to_client( indexer_id: int | None, download_id: int, ) -> str | None: - """Resolve opted-in Torznab descriptors before handing off to the client.""" - indexer = self._registry.get_indexer(indexer_id) if indexer_id is not None else None - if indexer is None or not bool(getattr(indexer, "browser_resolver_enabled", False)): + """Fetch HTTP torrent metadata in Pullbox; only magnets go to clients as URLs.""" + if url.lower().startswith("magnet:?"): return await client.add_torrent(url, title) + if not url.lower().startswith(("http://", "https://")): + raise ProviderError( + "download", "The torrent source URL must use HTTP, HTTPS, or magnet." + ) + indexer = self._registry.get_indexer(indexer_id) if indexer_id is not None else None + if indexer is None: + raise ProviderError( + "download", + "The originating torrent indexer is unavailable. " + "Run the search again before retrying.", + ) from pullbox.tasks.download_progress import ( clear_download_progress, @@ -457,7 +467,9 @@ async def add_torrent_to_client( fetch_descriptor = getattr(indexer, "fetch_torrent_descriptor", None) if fetch_descriptor is None: - return await client.add_torrent(url, title) + raise ProviderError( + "download", "The originating indexer cannot retrieve torrent metadata." + ) record_transient_download_stage(download_id, "Resolving torrent descriptor") async def on_attempt(event: object) -> None: diff --git a/src/pullbox/services/health_database_checks.py b/src/pullbox/services/health_database_checks.py index 25d65f1a..d6a343e1 100644 --- a/src/pullbox/services/health_database_checks.py +++ b/src/pullbox/services/health_database_checks.py @@ -24,8 +24,6 @@ _DB_CONNECTION_UNHEALTHY_MS = 1000.0 _DB_QUERY_DEGRADED_MS = 500.0 _DB_QUERY_UNHEALTHY_MS = 1500.0 -_DB_SIZE_DEGRADED_MB = 500.0 -_DB_SIZE_UNHEALTHY_MB = 1000.0 _DB_BLOAT_DEGRADED_RATIO = 0.15 _DB_BLOAT_UNHEALTHY_RATIO = 0.3 _DB_BLOAT_DEGRADED_MB = 50.0 @@ -233,25 +231,15 @@ async def check_db_size(session: AsyncSession) -> SubCheckOutcome | None: size_mb = size_bytes / (1024 * 1024) - if size_mb > _DB_SIZE_UNHEALTHY_MB: - return SubCheckOutcome( - check_name="database_size", - name="Database size", - status=HealthStatus.UNHEALTHY, - message=f"{size_mb:.0f} MB (threshold: {_DB_SIZE_UNHEALTHY_MB:.0f} MB)", - ) - if size_mb > _DB_SIZE_DEGRADED_MB: - return SubCheckOutcome( - check_name="database_size", - name="Database size", - status=HealthStatus.DEGRADED, - message=f"{size_mb:.0f} MB (threshold: {_DB_SIZE_DEGRADED_MB:.0f} MB)", - ) return SubCheckOutcome( check_name="database_size", name="Database size", status=HealthStatus.HEALTHY, - message=f"{size_mb:.1f} MB", + message=f"{size_mb:.1f} MB (informational)", + details={ + "size_bytes": size_bytes, + "classification": "informational", + }, ) diff --git a/src/pullbox/services/health_filesystem_checks.py b/src/pullbox/services/health_filesystem_checks.py index 94689343..ce06ba75 100644 --- a/src/pullbox/services/health_filesystem_checks.py +++ b/src/pullbox/services/health_filesystem_checks.py @@ -28,7 +28,7 @@ Mkstemp = Callable[..., tuple[int, str]] Close = Callable[[int], None] Unlink = Callable[[str], None] -FilesystemTargetCheck = Callable[[Path, str], tuple[SubCheckOutcome, str]] +FilesystemTargetCheck = Callable[[Path, str, bool], tuple[SubCheckOutcome, str]] async def check_filesystem( @@ -38,12 +38,18 @@ async def check_filesystem( check_target: FilesystemTargetCheck, ) -> CheckOutcome: """Check operational filesystem targets and persist path-level sub-checks.""" - paths: list[tuple[str, Path]] = [] + paths: list[tuple[str, Path, bool]] = [] result = await session.execute(select(LibraryRoot).where(LibraryRoot.enabled.is_(True))) roots = list(result.scalars().all()) for root in roots: - paths.append((f"Library Root: {root.name}", Path(root.path))) + paths.append( + ( + f"Library Root: {root.name}", + Path(root.path), + bool(root.allow_managed_writes), + ) + ) if settings: configured_targets = ( @@ -60,7 +66,7 @@ async def check_filesystem( if configured is None: continue if configured != default_path or configured.exists(): - paths.append((label, configured)) + paths.append((label, configured, True)) if not paths: return CheckOutcome( @@ -76,8 +82,13 @@ async def check_filesystem( worst = HealthStatus.HEALTHY inaccessible_or_unwritable = False - for name, path in paths: - sub_check, guidance = await asyncio.to_thread(check_target, path, name) + for name, path, require_write in paths: + sub_check, guidance = await asyncio.to_thread( + check_target, + path, + name, + require_write, + ) sub_checks.append(sub_check) if guidance: guidance_parts.append(guidance) @@ -88,6 +99,8 @@ async def check_filesystem( if inaccessible_or_unwritable: msg = "One or more paths are inaccessible or not writable" + elif any(not require_write for _name, _path, require_write in paths): + msg = "All paths meet configured access requirements" else: msg = "All paths accessible and writable" @@ -105,6 +118,7 @@ async def check_filesystem( def check_filesystem_target( path: Path, name: str, + require_write: bool, *, perf_counter: PerfCounter, scandir: Scandir, @@ -115,7 +129,10 @@ def check_filesystem_target( """Check one operational filesystem target and return a persistable sub-check.""" started = perf_counter() check_name = name - details: dict[str, Any] = {"path": str(path)} + details: dict[str, Any] = { + "path": str(path), + "required_access": "read_write" if require_write else "read", + } if not path.is_dir(): return ( @@ -144,21 +161,22 @@ def check_filesystem_target( f"'{path}' is not readable. Check filesystem permissions and mount status.", ) - try: - fd, probe = mkstemp(prefix=".pullbox-health-", dir=path) - close(fd) - unlink(probe) - except OSError as exc: - return ( - SubCheckOutcome( - check_name=check_name, - name=name, - status=HealthStatus.UNHEALTHY, - message=f"Not writable ({exc})", - details={**details, "issue": "unwritable"}, - ), - f"'{path}' is not writable. Check directory ownership and write permissions.", - ) + if require_write: + try: + fd, probe = mkstemp(prefix=".pullbox-health-", dir=path) + close(fd) + unlink(probe) + except OSError as exc: + return ( + SubCheckOutcome( + check_name=check_name, + name=name, + status=HealthStatus.UNHEALTHY, + message=f"Not writable ({exc})", + details={**details, "issue": "unwritable"}, + ), + f"'{path}' is not writable. Check directory ownership and write permissions.", + ) elapsed_ms = (perf_counter() - started) * 1000 @@ -167,7 +185,7 @@ def check_filesystem_target( check_name=check_name, name=name, status=HealthStatus.HEALTHY, - message="Readable and writable", + message="Readable and writable" if require_write else "Readable (reference-only)", details={**details, "issue": "ok"}, response_time_ms=elapsed_ms, ), diff --git a/src/pullbox/services/health_persistence.py b/src/pullbox/services/health_persistence.py index 8f625965..ed98c57e 100644 --- a/src/pullbox/services/health_persistence.py +++ b/src/pullbox/services/health_persistence.py @@ -11,7 +11,7 @@ import structlog from sqlalchemy import delete, select -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import OperationalError, PendingRollbackError from sqlalchemy.ext.asyncio import async_sessionmaker from pullbox.core.sqlite_lock import ( @@ -206,16 +206,22 @@ async def persist_health_outcomes( await active_session.commit() await session.commit() return - except OperationalError as exc: + except (OperationalError, PendingRollbackError) as exc: await active_session.rollback() - if not is_sqlite_locked_error(exc) or attempt == lock_retry_attempts: + recoverable = isinstance(exc, PendingRollbackError) or is_sqlite_locked_error(exc) + if not recoverable or attempt == lock_retry_attempts: raise delay_seconds = retry_delay(attempt) logger.warning( - "health_result_persist_retrying_after_sqlite_lock", + ( + "health_result_persist_retrying_after_pending_rollback" + if isinstance(exc, PendingRollbackError) + else "health_result_persist_retrying_after_sqlite_lock" + ), attempt=attempt, max_attempts=lock_retry_attempts, delay_seconds=delay_seconds, + failure_type=type(exc).__name__, ) finally: if manage_commit: diff --git a/src/pullbox/services/health_service.py b/src/pullbox/services/health_service.py index 97c2c05d..ccd2a8e2 100644 --- a/src/pullbox/services/health_service.py +++ b/src/pullbox/services/health_service.py @@ -148,7 +148,12 @@ async def run_all_checks( for factory, component, check_name in checks: if component == "comicvine" and skip_comicvine: continue - results = await self._safe_run(factory(), component, check_name) + results = await self._safe_run( + factory(), + component, + check_name, + session=session, + ) outcomes.extend(results) await self._persist_outcomes(session, outcomes) @@ -183,7 +188,12 @@ async def run_check( ] factory, check_name = dispatch[component] - outcomes = await self._safe_run(factory(), component, check_name) + outcomes = await self._safe_run( + factory(), + component, + check_name, + session=session, + ) await self._persist_outcomes(session, outcomes) return outcomes @@ -290,11 +300,16 @@ async def _check_filesystem( ) @staticmethod - def _check_filesystem_target(path: Path, name: str) -> tuple[SubCheckOutcome, str]: + def _check_filesystem_target( + path: Path, + name: str, + require_write: bool, + ) -> tuple[SubCheckOutcome, str]: """Check one operational filesystem target and return a persistable sub-check.""" return check_filesystem_target( path, name, + require_write, perf_counter=time.perf_counter, scandir=os.scandir, mkstemp=tempfile.mkstemp, @@ -450,6 +465,8 @@ async def _safe_run( coro: Awaitable[CheckOutcome | list[CheckOutcome]], component: str, check_name: str, + *, + session: AsyncSession | None = None, ) -> list[CheckOutcome]: """Run a check coroutine with timeout and exception isolation.""" start = time.perf_counter() @@ -471,6 +488,7 @@ async def _safe_run( return outcomes except TimeoutError: + await self._rollback_failed_check_session(session) elapsed_ms = (time.perf_counter() - start) * 1000 logger.warning("health_check_timeout", component=component, check_name=check_name) return [ @@ -488,6 +506,7 @@ async def _safe_run( ) ] except Exception as exc: + await self._rollback_failed_check_session(session) elapsed_ms = (time.perf_counter() - start) * 1000 logger.exception( "health_check_error", component=component, check_name=check_name, error=str(exc) @@ -506,6 +525,16 @@ async def _safe_run( ) ] + @staticmethod + async def _rollback_failed_check_session(session: AsyncSession | None) -> None: + """Restore a failed check transaction before later checks or persistence.""" + if session is None: + return + try: + await session.rollback() + except Exception: + logger.warning("health_check_session_rollback_failed", exc_info=True) + @staticmethod async def _persist_outcomes( session: AsyncSession, diff --git a/src/pullbox/services/import_catalog_hydration.py b/src/pullbox/services/import_catalog_hydration.py index ad192880..3265d07d 100644 --- a/src/pullbox/services/import_catalog_hydration.py +++ b/src/pullbox/services/import_catalog_hydration.py @@ -4,12 +4,21 @@ import asyncio from dataclasses import dataclass +from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any import structlog +from sqlalchemy import func, or_, update from sqlalchemy import select as sa_select +from pullbox.core.exceptions import ProviderError +from pullbox.core.library_root_resolution import preferred_managed_root_id from pullbox.models.series import IssueCatalogState, Series +from pullbox.services.import_metadata_priority import catalog_metadata_work +from pullbox.services.import_metadata_progress import ( + catalog_hydration_import_job_ids, + track_import_metadata_progress, +) if TYPE_CHECKING: from collections.abc import Awaitable, Callable @@ -18,7 +27,12 @@ logger = structlog.get_logger(__name__) +COMICVINE_BULK_BATCH_SIZE = 100 +CATALOG_HYDRATION_RETRY_DELAY = timedelta(hours=2) +_RETRYABLE_ERROR_PREFIX = "Retryable ComicVine hydration failure: " + catalog_hydration_tasks: set[asyncio.Task[None]] = set() +catalog_hydration_retry_tasks: set[asyncio.Task[None]] = set() _catalog_hydration_semaphore: asyncio.Semaphore | None = None _catalog_hydration_semaphore_loop: asyncio.AbstractEventLoop | None = None @@ -36,6 +50,30 @@ class PendingCatalogHydration: search_on_add: bool +class _CatalogHydrationDeferredError(Exception): + """Stop the current drain while a shared provider outage cools down.""" + + +def _is_retryable_provider_error(exc: Exception) -> bool: + if isinstance(exc, ProviderError): + details = exc.details if isinstance(exc.details, dict) else {} + status_code = details.get("status_code") + return bool(details.get("retryable")) or status_code in {408, 420, 429} + status_code = getattr(exc, "status_code", None) + return bool(getattr(exc, "retryable", False)) or status_code in {408, 420, 429} + + +def _is_retryable_error_text(error: str | None) -> bool: + normalized = (error or "").casefold() + return bool( + normalized.startswith(_RETRYABLE_ERROR_PREFIX.casefold()) + or "http 420" in normalized + or "http 429" in normalized + or "rate limit" in normalized + or "rate limited" in normalized + ) + + def reset_catalog_hydration_gate() -> None: """Reset the app-local hydration gate for tests and loop restarts.""" global _catalog_hydration_semaphore, _catalog_hydration_semaphore_loop @@ -50,41 +88,73 @@ def schedule_catalog_hydration( series_id: int, search_on_add: bool, ) -> None: - """Queue full catalog hydration after the Step 4 file-placement hot path.""" + """Nudge the singleton catalog drain after the Step 4 file-placement hot path.""" if session_factory is None: return - hydration_methods = _catalog_hydration_methods(series_service) - if hydration_methods is None: + if _catalog_hydration_methods(series_service) is None: + return + for retry_task in tuple(catalog_hydration_retry_tasks): + if not retry_task.done(): + retry_task.cancel() + if any(not task.done() for task in catalog_hydration_tasks): return - prefetch_comicvine_bundle, add_from_comicvine_prefetched = hydration_methods async def run_hydration() -> None: - async with catalog_hydration_gate(): + _ = series_id, search_on_add + while True: try: - await run_catalog_hydration( + await run_pending_catalog_hydration( session_factory, - series_id=series_id, - search_on_add=search_on_add, - prefetch_comicvine_bundle=prefetch_comicvine_bundle, - add_from_comicvine_prefetched=add_from_comicvine_prefetched, + series_service=series_service, ) except Exception as exc: - await mark_catalog_hydration_failed( - session_factory, - series_id=series_id, - error=str(exc), - ) logger.warning( - "import_catalog_hydration_failed", - series_id=series_id, + "import_catalog_hydration_drain_failed", error=str(exc), ) + return + await asyncio.sleep(0) + if not await load_pending_catalog_hydration(session_factory, limit=1): + await ensure_catalog_hydration_retry_scheduled( + session_factory, + series_service=series_service, + ) + return task = asyncio.create_task(run_hydration()) catalog_hydration_tasks.add(task) task.add_done_callback(catalog_hydration_tasks.discard) +async def ensure_catalog_hydration_retry_scheduled( + session_factory: async_sessionmaker[AsyncSession], + *, + series_service: Any, +) -> None: + """Wake the catalog drain when the shared provider cooldown expires.""" + retry_delay = await load_catalog_hydration_retry_delay(session_factory) + if retry_delay is None: + return + if any(not task.done() for task in catalog_hydration_retry_tasks): + return + + async def retry_after_cooldown() -> None: + await asyncio.sleep(retry_delay) + current_task = asyncio.current_task() + if current_task is not None: + catalog_hydration_retry_tasks.discard(current_task) + schedule_catalog_hydration( + session_factory, + series_service=series_service, + series_id=0, + search_on_add=False, + ) + + task = asyncio.create_task(retry_after_cooldown()) + catalog_hydration_retry_tasks.add(task) + task.add_done_callback(catalog_hydration_retry_tasks.discard) + + async def run_pending_catalog_hydration( session_factory: async_sessionmaker[AsyncSession], *, @@ -99,10 +169,24 @@ async def run_pending_catalog_hydration( pending = await load_pending_catalog_hydration(session_factory, limit=limit) recovered = 0 - async with catalog_hydration_gate(): + job_ids = await catalog_hydration_import_job_ids(session_factory) + async with ( + catalog_metadata_work(len(pending)) as priority, + track_import_metadata_progress(session_factory, job_ids=job_ids), + catalog_hydration_gate(), + ): + batch_methods = _catalog_hydration_batch_methods(series_service) + if batch_methods is not None and pending: + return await _run_pending_catalog_hydration_batch( + session_factory, + pending=pending, + batch_methods=batch_methods, + add_from_comicvine_prefetched=add_from_comicvine_prefetched, + priority=priority, + ) for request in pending: try: - await run_catalog_hydration( + hydrated = await run_catalog_hydration( session_factory, series_id=request.series_id, search_on_add=request.search_on_add, @@ -110,6 +194,18 @@ async def run_pending_catalog_hydration( add_from_comicvine_prefetched=add_from_comicvine_prefetched, ) except Exception as exc: + if _is_retryable_provider_error(exc): + await mark_catalog_hydration_deferred( + session_factory, + series_ids=[request.series_id], + error=str(exc), + ) + logger.warning( + "import_catalog_hydration_provider_deferred", + series_id=request.series_id, + error=str(exc), + ) + break await mark_catalog_hydration_failed( session_factory, series_id=request.series_id, @@ -121,6 +217,11 @@ async def run_pending_catalog_hydration( error=str(exc), ) continue + finally: + await priority.complete_one() + + if not hydrated: + continue recovered += 1 logger.info( @@ -131,6 +232,201 @@ async def run_pending_catalog_hydration( return recovered +def _catalog_hydration_batch_methods( + series_service: Any, +) -> ( + tuple[ + Callable[..., Awaitable[Any]], + Callable[..., Awaitable[Any]], + Callable[..., Awaitable[Any]], + ] + | None +): + methods = ( + getattr(series_service, "prefetch_comicvine_profiles", None), + getattr(series_service, "prefetch_comicvine_issue_catalogs", None), + getattr(series_service, "upsert_comicvine_profile", None), + ) + if not all(callable(method) for method in methods): + return None + return methods # type: ignore[return-value] + + +async def _run_pending_catalog_hydration_batch( + session_factory: async_sessionmaker[AsyncSession], + *, + pending: list[PendingCatalogHydration], + batch_methods: tuple[ + Callable[..., Awaitable[Any]], + Callable[..., Awaitable[Any]], + Callable[..., Awaitable[Any]], + ], + add_from_comicvine_prefetched: Callable[..., Awaitable[Series]], + priority: Any, +) -> int: + """Persist visible profiles first, then complete issue catalogs in bulk.""" + planned: list[tuple[PendingCatalogHydration, CatalogHydrationPlan]] = [] + for request in pending: + plan = await load_catalog_hydration_plan( + session_factory, + series_id=request.series_id, + search_on_add=request.search_on_add, + only_if_hydrating=True, + ) + if plan is None: + await priority.complete_one() + continue + planned.append((request, plan)) + + comicvine_ids = [plan.comicvine_id for _request, plan in planned] + if not comicvine_ids: + return 0 + + recovered = 0 + for start in range(0, len(planned), COMICVINE_BULK_BATCH_SIZE): + try: + recovered += await _run_planned_catalog_hydration_batch( + session_factory, + planned=planned[start : start + COMICVINE_BULK_BATCH_SIZE], + batch_methods=batch_methods, + add_from_comicvine_prefetched=add_from_comicvine_prefetched, + priority=priority, + ) + except _CatalogHydrationDeferredError: + break + return recovered + + +async def _run_planned_catalog_hydration_batch( + session_factory: async_sessionmaker[AsyncSession], + *, + planned: list[tuple[PendingCatalogHydration, CatalogHydrationPlan]], + batch_methods: tuple[ + Callable[..., Awaitable[Any]], + Callable[..., Awaitable[Any]], + Callable[..., Awaitable[Any]], + ], + add_from_comicvine_prefetched: Callable[..., Awaitable[Series]], + priority: Any, +) -> int: + """Hydrate one provider-safe batch without failing unrelated backlog rows.""" + prefetch_profiles, prefetch_catalogs, upsert_profile = batch_methods + comicvine_ids = [plan.comicvine_id for _request, plan in planned] + + try: + profiles = await prefetch_profiles(comicvine_ids) + except Exception as exc: + if _is_retryable_provider_error(exc): + await mark_catalog_hydration_deferred( + session_factory, + series_ids=[request.series_id for request, _plan in planned], + error=str(exc), + ) + for _request, _plan in planned: + await priority.complete_one() + logger.warning( + "import_catalog_hydration_provider_deferred", + affected_series=len(planned), + error=str(exc), + ) + raise _CatalogHydrationDeferredError from exc + for request, _plan in planned: + await mark_catalog_hydration_failed( + session_factory, + series_id=request.series_id, + error=str(exc), + ) + await priority.complete_one() + return 0 + + eligible: list[tuple[PendingCatalogHydration, CatalogHydrationPlan]] = [] + for request, plan in planned: + profile = profiles.get(plan.comicvine_id) + if profile is None: + await mark_catalog_hydration_failed( + session_factory, + series_id=request.series_id, + error="ComicVine did not return the requested series profile", + ) + await priority.complete_one() + continue + try: + async with session_factory() as session: + state = await session.scalar( + sa_select(Series.issue_catalog_state).where(Series.id == request.series_id) + ) + if state != IssueCatalogState.HYDRATING: + await priority.complete_one() + continue + await upsert_profile(session, plan.comicvine_id, profile) + await session.commit() + eligible.append((request, plan)) + except Exception as exc: + await mark_catalog_hydration_failed( + session_factory, + series_id=request.series_id, + error=str(exc), + ) + await priority.complete_one() + + if not eligible: + return 0 + + try: + catalogs = await prefetch_catalogs([plan.comicvine_id for _request, plan in eligible]) + except Exception as exc: + if _is_retryable_provider_error(exc): + await mark_catalog_hydration_deferred( + session_factory, + series_ids=[request.series_id for request, _plan in eligible], + error=str(exc), + ) + for _request, _plan in eligible: + await priority.complete_one() + logger.warning( + "import_catalog_hydration_provider_deferred", + affected_series=len(eligible), + error=str(exc), + ) + raise _CatalogHydrationDeferredError from exc + for request, _plan in eligible: + await mark_catalog_hydration_failed( + session_factory, + series_id=request.series_id, + error=str(exc), + ) + await priority.complete_one() + return 0 + + recovered = 0 + for request, plan in eligible: + try: + profile = profiles[plan.comicvine_id] + summaries = catalogs.get(plan.comicvine_id) + if summaries is None: + raise ValueError("ComicVine did not return the requested issue catalog") + persisted = await _persist_catalog_hydration( + session_factory, + series_id=request.series_id, + plan=plan, + series_meta=profile, + issue_summaries=summaries, + add_from_comicvine_prefetched=add_from_comicvine_prefetched, + ) + if persisted: + recovered += 1 + logger.info("import_catalog_hydration_recovered", series_id=request.series_id) + except Exception as exc: + await mark_catalog_hydration_failed( + session_factory, + series_id=request.series_id, + error=str(exc), + ) + finally: + await priority.complete_one() + return recovered + + async def load_pending_catalog_hydration( session_factory: async_sessionmaker[AsyncSession], *, @@ -138,10 +434,33 @@ async def load_pending_catalog_hydration( ) -> list[PendingCatalogHydration]: """Load incomplete catalog rows that should resume after process restart.""" async with session_factory() as session: + retry_cutoff = datetime.now(UTC) - CATALOG_HYDRATION_RETRY_DELAY + recent_provider_pause = await session.scalar( + sa_select(Series.id) + .where( + Series.issue_catalog_state == IssueCatalogState.HYDRATING, + Series.issue_catalog_error.startswith(_RETRYABLE_ERROR_PREFIX), + Series.issue_catalog_last_checked_at.isnot(None), + Series.issue_catalog_last_checked_at > retry_cutoff, + ) + .limit(1) + ) + if recent_provider_pause is not None: + return [] + + retryable_failure = or_( + Series.issue_catalog_error.startswith(_RETRYABLE_ERROR_PREFIX), + Series.issue_catalog_error.ilike("%HTTP 420%"), + Series.issue_catalog_error.ilike("%HTTP 429%"), + Series.issue_catalog_error.ilike("%rate limit%"), + ) stmt = ( sa_select(Series.id, Series.monitored) .where( - Series.issue_catalog_state == IssueCatalogState.HYDRATING, + or_( + Series.issue_catalog_state == IssueCatalogState.HYDRATING, + ((Series.issue_catalog_state == IssueCatalogState.FAILED) & retryable_failure), + ), Series.comicvine_id.isnot(None), ) .order_by(Series.id.asc()) @@ -158,6 +477,27 @@ async def load_pending_catalog_hydration( ] +async def load_catalog_hydration_retry_delay( + session_factory: async_sessionmaker[AsyncSession], +) -> float | None: + """Return seconds until the earliest active provider cooldown expires.""" + now = datetime.now(UTC) + retry_cutoff = now - CATALOG_HYDRATION_RETRY_DELAY + async with session_factory() as session: + paused_at = await session.scalar( + sa_select(func.min(Series.issue_catalog_last_checked_at)).where( + Series.issue_catalog_state == IssueCatalogState.HYDRATING, + Series.issue_catalog_error.startswith(_RETRYABLE_ERROR_PREFIX), + Series.issue_catalog_last_checked_at.isnot(None), + Series.issue_catalog_last_checked_at > retry_cutoff, + ) + ) + if paused_at is None: + return None + retry_at = paused_at + CATALOG_HYDRATION_RETRY_DELAY + return max(0.0, (retry_at - now).total_seconds()) + + def catalog_hydration_gate() -> asyncio.Semaphore: """Return the app-local lane for background catalog hydration.""" global _catalog_hydration_semaphore, _catalog_hydration_semaphore_loop @@ -192,20 +532,57 @@ async def run_catalog_hydration( search_on_add: bool, prefetch_comicvine_bundle: Callable[[int], Awaitable[tuple[Any, list[Any]]]], add_from_comicvine_prefetched: Callable[..., Awaitable[Series]], -) -> None: +) -> bool: plan = await load_catalog_hydration_plan( session_factory, series_id=series_id, search_on_add=search_on_add, + only_if_hydrating=True, ) if plan is None: - return + return False # ComicVine can be slow for giant series. Fetch outside any DB session so # the UI and active import writer keep access to the pool while we wait. series_meta, issue_summaries = await prefetch_comicvine_bundle(plan.comicvine_id) + return await _persist_catalog_hydration( + session_factory, + series_id=series_id, + plan=plan, + series_meta=series_meta, + issue_summaries=issue_summaries, + add_from_comicvine_prefetched=add_from_comicvine_prefetched, + ) + + +async def _persist_catalog_hydration( + session_factory: async_sessionmaker[AsyncSession], + *, + series_id: int, + plan: CatalogHydrationPlan, + series_meta: Any, + issue_summaries: list[Any], + add_from_comicvine_prefetched: Callable[..., Awaitable[Series]], +) -> bool: + """Persist one prefetched catalog only while its exact row still needs it.""" + async with session_factory() as hydrate_session: + # A cancellation rollback can delete the series while the provider + # request is in flight. Revalidate the exact row under the write + # transaction so the general ComicVine upsert path cannot recreate a + # series that the rollback already removed. + persisted_series_id = await hydrate_session.scalar( + sa_select(Series.id) + .where( + Series.id == series_id, + Series.comicvine_id == plan.comicvine_id, + Series.issue_catalog_state == IssueCatalogState.HYDRATING, + ) + .with_for_update() + ) + if persisted_series_id is None: + return False await add_from_comicvine_prefetched( hydrate_session, comicvine_id=plan.comicvine_id, @@ -215,6 +592,7 @@ async def run_catalog_hydration( issue_summaries=issue_summaries, ) await hydrate_session.commit() + return True async def load_catalog_hydration_plan( @@ -222,11 +600,22 @@ async def load_catalog_hydration_plan( *, series_id: int, search_on_add: bool, + only_if_hydrating: bool = False, ) -> CatalogHydrationPlan | None: async with session_factory() as session: series = await session.get(Series, series_id) if series is None: return None + retryable_failure = ( + series.issue_catalog_state == IssueCatalogState.FAILED + and _is_retryable_error_text(series.issue_catalog_error) + ) + if ( + only_if_hydrating + and series.issue_catalog_state != IssueCatalogState.HYDRATING + and not retryable_failure + ): + return None if series.comicvine_id is None: msg = "Series has no ComicVine ID" raise ValueError(msg) @@ -239,11 +628,34 @@ async def load_catalog_hydration_plan( return CatalogHydrationPlan( comicvine_id=int(series.comicvine_id), - library_root_id=series.library_root_id, + library_root_id=preferred_managed_root_id(series), search_on_add=search_on_add, ) +async def mark_catalog_hydration_deferred( + session_factory: async_sessionmaker[AsyncSession], + *, + series_ids: list[int], + error: str, +) -> None: + """Keep transient provider failures resumable and persist a shared cooldown marker.""" + if not series_ids: + return + async with session_factory() as session: + await session.execute( + update(Series) + .where(Series.id.in_(series_ids)) + .values( + issue_catalog_state=IssueCatalogState.HYDRATING, + issue_catalog_error=f"{_RETRYABLE_ERROR_PREFIX}{error}", + issue_catalog_last_synced_at=None, + issue_catalog_last_checked_at=datetime.now(UTC), + ) + ) + await session.commit() + + async def mark_catalog_hydration_failed( session_factory: async_sessionmaker[AsyncSession], *, diff --git a/src/pullbox/services/import_comicinfo_enrichment.py b/src/pullbox/services/import_comicinfo_enrichment.py index 16b7f592..290021e3 100644 --- a/src/pullbox/services/import_comicinfo_enrichment.py +++ b/src/pullbox/services/import_comicinfo_enrichment.py @@ -25,6 +25,8 @@ from pullbox.models.library import LibraryFile from pullbox.models.series import Series from pullbox.services.import_comicinfo_metadata import is_retryable_provider_error +from pullbox.services.import_metadata_priority import wait_for_comicinfo_turn +from pullbox.services.import_metadata_progress import track_import_metadata_progress if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -32,10 +34,12 @@ logger = structlog.get_logger(__name__) COMICINFO_ENRICHMENT_DIAGNOSTIC_KEY = "comicinfo_enrichment" +COMICVINE_BULK_BATCH_SIZE = 5_000 ImportComicInfoBuildPayload = Callable[..., Awaitable[dict[str, Any]]] ImportComicInfoApply = Callable[[Path, dict[str, Any]], Any] ImportComicInfoLogEvent = Callable[..., Awaitable[None]] +ImportComicInfoPrefetch = Callable[[list[int]], Awaitable[Any]] comicinfo_enrichment_tasks: set[asyncio.Task[None]] = set() _comicinfo_enrichment_semaphore: asyncio.Semaphore | None = None @@ -79,27 +83,28 @@ def schedule_import_comicinfo_enrichment( build_comicinfo_payload: ImportComicInfoBuildPayload, apply_comicinfo: ImportComicInfoApply, log_event: ImportComicInfoLogEvent, + prefetch_issue_metadata: ImportComicInfoPrefetch | None = None, ) -> None: """Queue deferred issue metadata and ComicInfo rewrites after Step 4 completes.""" if session_factory is None: return async def run_enrichment() -> None: - async with comicinfo_enrichment_gate(): - try: - await run_import_comicinfo_enrichment( - session_factory, - job_id=job_id, - build_comicinfo_payload=build_comicinfo_payload, - apply_comicinfo=apply_comicinfo, - log_event=log_event, - ) - except Exception as exc: - logger.warning( - "import_comicinfo_enrichment_failed", - job_id=job_id, - error=str(exc), - ) + try: + await run_import_comicinfo_enrichment( + session_factory, + job_id=job_id, + build_comicinfo_payload=build_comicinfo_payload, + apply_comicinfo=apply_comicinfo, + log_event=log_event, + prefetch_issue_metadata=prefetch_issue_metadata, + ) + except Exception as exc: + logger.warning( + "import_comicinfo_enrichment_failed", + job_id=job_id, + error=str(exc), + ) task = asyncio.create_task(run_enrichment()) comicinfo_enrichment_tasks.add(task) @@ -112,19 +117,20 @@ async def run_pending_import_comicinfo_enrichment( build_comicinfo_payload: ImportComicInfoBuildPayload, apply_comicinfo: ImportComicInfoApply, log_event: ImportComicInfoLogEvent, + prefetch_issue_metadata: ImportComicInfoPrefetch | None = None, ) -> int: """Refresh deferred ComicInfo metadata for all completed jobs with pending rows.""" - async with comicinfo_enrichment_gate(): - pending_job_ids = await _load_pending_import_job_ids(session_factory) - for job_id in pending_job_ids: - await run_import_comicinfo_enrichment( - session_factory, - job_id=job_id, - build_comicinfo_payload=build_comicinfo_payload, - apply_comicinfo=apply_comicinfo, - log_event=log_event, - ) - return len(pending_job_ids) + pending_job_ids = await _load_pending_import_job_ids(session_factory) + for job_id in pending_job_ids: + await run_import_comicinfo_enrichment( + session_factory, + job_id=job_id, + build_comicinfo_payload=build_comicinfo_payload, + apply_comicinfo=apply_comicinfo, + log_event=log_event, + prefetch_issue_metadata=prefetch_issue_metadata, + ) + return len(pending_job_ids) async def run_import_comicinfo_enrichment( @@ -134,10 +140,53 @@ async def run_import_comicinfo_enrichment( build_comicinfo_payload: ImportComicInfoBuildPayload, apply_comicinfo: ImportComicInfoApply, log_event: ImportComicInfoLogEvent, + prefetch_issue_metadata: ImportComicInfoPrefetch | None = None, ) -> None: """Refresh deferred ComicInfo metadata for imported files in one import job.""" + async with ( + track_import_metadata_progress(session_factory, job_ids=[job_id]), + comicinfo_enrichment_gate(), + ): + await _run_import_comicinfo_enrichment_while_fenced( + session_factory, + job_id=job_id, + build_comicinfo_payload=build_comicinfo_payload, + apply_comicinfo=apply_comicinfo, + log_event=log_event, + prefetch_issue_metadata=prefetch_issue_metadata, + ) + + +async def _run_import_comicinfo_enrichment_while_fenced( + session_factory: async_sessionmaker[AsyncSession], + *, + job_id: int, + build_comicinfo_payload: ImportComicInfoBuildPayload, + apply_comicinfo: ImportComicInfoApply, + log_event: ImportComicInfoLogEvent, + prefetch_issue_metadata: ImportComicInfoPrefetch | None = None, +) -> None: + """Run one job while holding the process-local filesystem mutation fence.""" + if not await _import_job_is_completed(session_factory, job_id=job_id): + return pending_ids = await _load_pending_imported_file_ids(session_factory, job_id=job_id) + if prefetch_issue_metadata is not None: + issue_cv_ids = await _load_pending_issue_cv_ids(session_factory, job_id=job_id) + if issue_cv_ids: + for start in range(0, len(issue_cv_ids), COMICVINE_BULK_BATCH_SIZE): + batch = issue_cv_ids[start : start + COMICVINE_BULK_BATCH_SIZE] + await wait_for_comicinfo_turn() + try: + await prefetch_issue_metadata(batch) + except Exception as exc: + logger.warning( + "import_comicinfo_metadata_batch_prefetch_failed", + job_id=job_id, + issue_count=len(batch), + error=str(exc), + ) for imported_file_id in pending_ids: + await wait_for_comicinfo_turn() try: prepared = await _prepare_pending_imported_file_with_retry( session_factory, @@ -146,6 +195,8 @@ async def run_import_comicinfo_enrichment( ) if prepared is None: continue + if not await _import_job_is_completed(session_factory, job_id=job_id): + return if inspect.iscoroutinefunction(apply_comicinfo): await apply_comicinfo(prepared.artifact_path, prepared.payload) @@ -183,6 +234,18 @@ async def run_import_comicinfo_enrichment( ) +async def _import_job_is_completed( + session_factory: async_sessionmaker[AsyncSession], + *, + job_id: int, +) -> bool: + """Read durable job state at a filesystem safe boundary.""" + async with session_factory() as session: + status = await session.scalar(sa_select(ImportJob.status).where(ImportJob.id == job_id)) + await session.rollback() + return status is ImportJobStatus.COMPLETED + + async def _load_pending_imported_file_ids( session_factory: async_sessionmaker[AsyncSession], *, @@ -191,7 +254,9 @@ async def _load_pending_imported_file_ids( async with session_factory() as session: result = await session.execute( sa_select(ImportedFile.id) + .join(ImportJob, ImportedFile.import_job_id == ImportJob.id) .where(ImportedFile.import_job_id == job_id) + .where(ImportJob.status == ImportJobStatus.COMPLETED) .where(ImportedFile.status == ImportedFileStatus.IMPORTED) ) ids: list[int] = [] @@ -202,6 +267,41 @@ async def _load_pending_imported_file_ids( return ids +async def _load_pending_issue_cv_ids( + session_factory: async_sessionmaker[AsyncSession], + *, + job_id: int, +) -> list[int]: + """Load unique provider issue IDs already recorded in pending diagnostics.""" + async with session_factory() as session: + result = await session.execute( + sa_select(ImportedFile.diagnostics) + .join(ImportJob, ImportedFile.import_job_id == ImportJob.id) + .where(ImportedFile.import_job_id == job_id) + .where(ImportJob.status == ImportJobStatus.COMPLETED) + .where(ImportedFile.status == ImportedFileStatus.IMPORTED) + .order_by(ImportedFile.id) + ) + provider_ids: list[int] = [] + seen: set[int] = set() + for diagnostics in result.scalars().all(): + if not _is_pending_comicinfo_enrichment_diagnostics(diagnostics): + continue + details = diagnostics.get(COMICINFO_ENRICHMENT_DIAGNOSTIC_KEY, {}) + raw_provider_id = details.get("issue_cv_id") if isinstance(details, dict) else None + if isinstance(raw_provider_id, bool) or not isinstance(raw_provider_id, (int, str)): + continue + try: + provider_id = int(raw_provider_id) + except (TypeError, ValueError): + continue + if provider_id <= 0 or provider_id in seen: + continue + seen.add(provider_id) + provider_ids.append(provider_id) + return provider_ids + + async def _load_pending_import_job_ids( session_factory: async_sessionmaker[AsyncSession], ) -> list[int]: @@ -288,20 +388,43 @@ async def _prepare_pending_imported_file( if not _is_pending_comicinfo_enrichment(imported_file): return None assert imported_file is not None + job_status = await session.scalar( + sa_select(ImportJob.status).where(ImportJob.id == imported_file.import_job_id) + ) + if job_status is not ImportJobStatus.COMPLETED: + return None - library_file_id = imported_file.library_file_id - if library_file_id is None: - details = imported_file.diagnostics.get(COMICINFO_ENRICHMENT_DIAGNOSTIC_KEY, {}) - if isinstance(details, dict): - library_file_id = details.get("library_file_id") + details_value = imported_file.diagnostics.get(COMICINFO_ENRICHMENT_DIAGNOSTIC_KEY, {}) + details = details_value if isinstance(details_value, dict) else {} + recorded_library_file_id = _recorded_positive_int(details, "library_file_id") + library_file_id = imported_file.library_file_id or recorded_library_file_id if library_file_id is None: raise ValueError("Deferred ComicInfo enrichment is missing library_file_id") + if ( + imported_file.library_file_id is not None + and recorded_library_file_id is not None + and imported_file.library_file_id != recorded_library_file_id + ): + raise ValueError("Deferred ComicInfo target no longer matches its queued library file") library_file = await session.get(LibraryFile, int(library_file_id)) if library_file is None: raise ValueError(f"Library file {library_file_id} no longer exists") - issue_id = imported_file.matched_issue_id or library_file.issue_id + recorded_issue_id = _recorded_positive_int(details, "issue_id") + expected_issue_id = imported_file.matched_issue_id or recorded_issue_id + if expected_issue_id is not None and library_file.issue_id != expected_issue_id: + raise ValueError("Deferred ComicInfo target no longer matches its queued issue") + + recorded_artifact_path = details.get("artifact_path") + if ( + isinstance(recorded_artifact_path, str) + and recorded_artifact_path + and Path(library_file.file_path) != Path(recorded_artifact_path) + ): + raise ValueError("Deferred ComicInfo target no longer matches its queued artifact path") + + issue_id = expected_issue_id or library_file.issue_id if issue_id is None: raise ValueError("Deferred ComicInfo enrichment is missing issue_id") @@ -313,6 +436,9 @@ async def _prepare_pending_imported_file( issue = issue_result.scalars().first() if issue is None: raise ValueError(f"Issue {issue_id} no longer exists") + recorded_issue_cv_id = _recorded_positive_int(details, "issue_cv_id") + if recorded_issue_cv_id is not None and issue.comicvine_id != recorded_issue_cv_id: + raise ValueError("Deferred ComicInfo target no longer matches its queued ComicVine issue") artifact_path = Path(library_file.file_path) payload = await build_comicinfo_payload( @@ -332,6 +458,22 @@ async def _prepare_pending_imported_file( ) +def _recorded_positive_int(details: dict[str, Any], key: str) -> int | None: + value = details.get(key) + if isinstance(value, bool): + return None + if isinstance(value, int): + parsed = value + elif isinstance(value, str): + try: + parsed = int(value) + except ValueError: + return None + else: + return None + return parsed if parsed > 0 else None + + async def _mark_pending_file_complete_with_retry( session_factory: async_sessionmaker[AsyncSession], *, @@ -362,6 +504,7 @@ async def _mark_pending_file_complete_with_retry( artifact_stat.st_mtime, tz=UTC, ) + library_file.has_comicinfo = True _set_comicinfo_enrichment_status( imported_file, status="complete", diff --git a/src/pullbox/services/import_completed_cleanup.py b/src/pullbox/services/import_completed_cleanup.py new file mode 100644 index 00000000..ed444d36 --- /dev/null +++ b/src/pullbox/services/import_completed_cleanup.py @@ -0,0 +1,1724 @@ +"""Safe, previewed recovery actions for completed collection imports.""" + +from __future__ import annotations + +import enum +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from hashlib import sha256 +from itertools import batched +from typing import TYPE_CHECKING, Any, Final +from uuid import uuid4 + +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +from sqlalchemy import and_, case, exists, func, or_, select, update +from sqlalchemy.orm import aliased + +from pullbox.core.config_resolver import get_application_secret +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.issue_numbers import parse_issue_number_text +from pullbox.core.name_matcher import NameMatcher +from pullbox.models.audit_log import AuditEventType +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobLog, + ImportJobStatus, + ImportSeriesStatus, +) +from pullbox.models.issue import Issue, IssueStatus +from pullbox.models.library import LibraryFile, LibraryFileStorageMode +from pullbox.models.series import Series +from pullbox.services.audit_service import AuditService +from pullbox.services.import_counters import recompute_file_counters, recompute_series_counters +from pullbox.services.import_deferred_recovery import load_empty_stale_series +from pullbox.services.import_known_series_recovery import load_known_series_recovery +from pullbox.services.import_review_actions import apply_safety_allow_once_to_file +from pullbox.services.import_review_recheck import retryable_failed_source_filters +from pullbox.services.import_safety_diagnostics import ImportSafetyCategory +from pullbox.services.import_story_arc_resolution import ( + refresh_story_arc_entries_for_import_files, +) +from pullbox.services.import_terminal_recovery import allows_terminal_import_recovery + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.services.import_known_series_recovery import KnownSeriesRecovery + + +_PREVIEW_TOKEN_SALT: Final = "completed-import-cleanup-v1" +_PREVIEW_TOKEN_MAX_AGE_SECONDS: Final = 15 * 60 +_PREVIEW_TOKEN_VERSION: Final = 1 +_PAGE_SIZE: Final = 500 +_EXAMPLE_LIMIT: Final = 3 + + +class CompletedImportCleanupAction(enum.StrEnum): + """Supported completed-import cleanup operations.""" + + DISMISS_MISSING_REFERENCES = "dismiss_missing_references" + SKIP_PROBABLE_COVERS = "skip_probable_covers" + SKIP_UNUSABLE_FILES = "skip_unusable_files" + ALLOW_OVERSIZED_FILES = "allow_oversized_files" + RETRY_SOURCE_INSPECTION = "retry_source_inspection" + NORMALIZE_ALREADY_OWNED = "normalize_already_owned" + ACCEPT_RECOMMENDED_CONFLICTS = "accept_recommended_conflicts" + RESOLVE_MIXED_FOLDER_FILES = "resolve_mixed_folder_files" + RECOVER_KNOWN_SERIES = "recover_known_series" + RECHECK_DEFERRED_FILES = "recheck_deferred_files" + + +@dataclass(frozen=True, slots=True) +class CompletedImportCleanupSnapshot: + """Exact identity summary for one previewed cleanup scope.""" + + affected_count: int + affected_file_count: int + min_file_id: int | None + max_file_id: int | None + max_updated_at: str | None + scope_digest: str + + +@dataclass(frozen=True, slots=True) +class CompletedImportCleanupPreview: + """User-facing bounded preview of a completed-import cleanup action.""" + + job_id: int + action: CompletedImportCleanupAction + affected_count: int + affected_file_count: int + item_unit: str + examples: tuple[str, ...] + preview_token: str + + +@dataclass(frozen=True, slots=True) +class CompletedImportCleanupResult: + """Outcome of a completed-import cleanup action.""" + + job_id: int + action: CompletedImportCleanupAction + affected_count: int + affected_file_count: int + requires_import_retry: bool + retry_file_ids: tuple[int, ...] = () + + +@dataclass(frozen=True, slots=True) +class CompletedImportCleanupFilePage: + """One bounded page of files in an actionable recovery scope.""" + + items: tuple[ImportedFile, ...] + total: int + page: int + page_size: int + total_pages: int + + +@dataclass(frozen=True, slots=True) +class CompletedImportCleanupSummary: + """Counts and examples for one results-page recovery card.""" + + affected_count: int + affected_file_count: int + examples: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _MixedFolderResolution: + """One exact, source-preserving mixed-folder ownership correction.""" + + file_id: int + source_import_series_id: int + source_import_series_name: str + target_series_id: int + target_series_title: str + target_issue_id: int + target_issue_cv_id: int | None + target_issue_number: float + target_issue_number_text: str + target_library_file_id: int | None + source_library_file_id: int | None + source_issue_id: int | None + source_library_updated_at: str | None + evidence_source: str + source_series_name: str + source_updated_at: str + + +def _serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(get_application_secret(), salt=_PREVIEW_TOKEN_SALT) + + +def _category_expression() -> Any: + return ImportedFile.diagnostics["safety_block"]["category"].as_string() + + +def _overrideable_expression() -> Any: + return ImportedFile.diagnostics["safety_block"]["overrideable"].as_boolean() + + +def _source_revalidation_category_expression() -> Any: + return ImportedFile.diagnostics["source_revalidation"]["category"].as_string() + + +def _safety_filter(*categories: ImportSafetyCategory) -> Any: + return _category_expression().in_([category.value for category in categories]) + + +def _candidate_conflict_groups(job_id: int) -> Any: + """Groups with exactly one high-confidence preferred candidate.""" + candidate = aliased(ImportedFile) + return ( + select(candidate.conflict_group_id) + .where( + candidate.import_job_id == job_id, + candidate.status == ImportedFileStatus.CONFLICT, + candidate.conflict_group_id.is_not(None), + ~exists().where(LibraryFile.issue_id == candidate.matched_issue_id), + ) + .group_by(candidate.conflict_group_id) + .having(func.sum(case((candidate.is_preferred.is_(True), 1), else_=0)) == 1) + .having( + func.sum( + case( + ( + candidate.is_preferred.is_(True) & (candidate.match_confidence == "high"), + 1, + ), + else_=0, + ) + ) + == 1 + ) + ) + + +def _fully_recoverable_conflict_series(job_id: int) -> Any: + """Series whose remaining conflicts are all safe recommended groups.""" + conflict = aliased(ImportedFile) + candidate_groups = _candidate_conflict_groups(job_id) + return ( + select(conflict.import_series_id) + .where( + conflict.import_job_id == job_id, + conflict.status == ImportedFileStatus.CONFLICT, + ) + .group_by(conflict.import_series_id) + .having( + func.sum( + case( + ( + or_( + conflict.conflict_group_id.is_(None), + ~conflict.conflict_group_id.in_(candidate_groups), + ), + 1, + ), + else_=0, + ) + ) + == 0 + ) + ) + + +def _eligible_conflict_groups(job_id: int) -> Any: + """Recommended groups that can be resumed without stranding sibling conflicts.""" + candidate = aliased(ImportedFile) + return ( + select(candidate.conflict_group_id) + .where( + candidate.import_job_id == job_id, + candidate.status == ImportedFileStatus.CONFLICT, + candidate.conflict_group_id.in_(_candidate_conflict_groups(job_id)), + candidate.import_series_id.in_(_fully_recoverable_conflict_series(job_id)), + ) + .distinct() + ) + + +def _file_filters(job_id: int, action: CompletedImportCleanupAction) -> tuple[Any, ...]: + filters: list[Any] = [ImportedFile.import_job_id == job_id] + if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: + filters.append(ImportedFile.status == ImportedFileStatus.NO_MATCH) + elif action is CompletedImportCleanupAction.DISMISS_MISSING_REFERENCES: + filters.append( + or_( + and_( + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + _safety_filter(ImportSafetyCategory.SOURCE_MISSING), + ), + and_( + ImportedFile.status == ImportedFileStatus.FAILED, + _source_revalidation_category_expression() + == ImportSafetyCategory.SOURCE_MISSING.value, + ), + ) + ) + elif action is CompletedImportCleanupAction.SKIP_PROBABLE_COVERS: + filters.extend( + [ + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + _safety_filter(ImportSafetyCategory.SINGLE_PAGE_COMIC), + ] + ) + elif action is CompletedImportCleanupAction.SKIP_UNUSABLE_FILES: + unusable_categories = ( + ImportSafetyCategory.ZERO_BYTE, + ImportSafetyCategory.ARCHIVE_NO_PAGES, + ImportSafetyCategory.UNSUPPORTED_FILE_TYPE, + ) + filters.append( + or_( + and_( + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + _safety_filter(*unusable_categories), + ), + and_( + ImportedFile.status == ImportedFileStatus.FAILED, + _source_revalidation_category_expression().in_( + [category.value for category in unusable_categories] + ), + ), + ) + ) + elif action is CompletedImportCleanupAction.ALLOW_OVERSIZED_FILES: + filters.extend( + [ + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + _safety_filter(ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT), + _overrideable_expression().is_(True), + ] + ) + elif action is CompletedImportCleanupAction.RETRY_SOURCE_INSPECTION: + retryable_categories = [ + category.value + for category in ( + ImportSafetyCategory.PERMISSION_UNREADABLE, + ImportSafetyCategory.ARCHIVE_INSPECTION_FAILED, + ImportSafetyCategory.SOURCE_CHANGED, + ) + ] + filters.append( + or_( + and_( + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + _category_expression().in_(retryable_categories), + ), + and_(*retryable_failed_source_filters(job_id)), + ) + ) + elif action is CompletedImportCleanupAction.NORMALIZE_ALREADY_OWNED: + filters.extend( + [ + ImportedFile.status == ImportedFileStatus.CONFLICT, + ImportedFile.matched_issue_id.is_not(None), + exists().where(LibraryFile.issue_id == ImportedFile.matched_issue_id), + ] + ) + elif action is CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS: + filters.extend( + [ + ImportedFile.status == ImportedFileStatus.CONFLICT, + ImportedFile.conflict_group_id.in_(_eligible_conflict_groups(job_id)), + ] + ) + elif action is CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES: + filters.append( + ImportedFile.status.in_((ImportedFileStatus.NO_MATCH, ImportedFileStatus.IMPORTED)) + ) + else: # pragma: no cover - exhaustive enum guard + raise ValidationError("Unsupported completed-import cleanup action.") + return tuple(filters) + + +def _mapping(value: object) -> Mapping[str, object]: + return value if isinstance(value, Mapping) else {} + + +def _safe_int(value: object) -> int | None: + if not isinstance(value, str | bytes | bytearray | int | float): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _mixed_folder_source_identity( + imported_file: ImportedFile, +) -> tuple[str, str | float | int, int | None, int | None, str] | None: + """Return only embedded or sidecar identity strong enough for bulk correction.""" + diagnostics = _mapping(imported_file.diagnostics) + signals = _mapping(diagnostics.get("metadata_signals")) + series_signal = str(signals.get("series_name") or "") + issue_signal = str(signals.get("issue_number") or "") + trusted_signals = {"comicinfo", "sidecar"} + if series_signal not in trusted_signals or issue_signal not in trusted_signals: + return None + + source_metadata = _mapping(diagnostics.get("source_metadata")) + comicinfo = _mapping(source_metadata.get("comicinfo")) + if series_signal == "comicinfo": + source_series_name = str(comicinfo.get("series") or "").strip() + else: + source_series_name = str(imported_file.parsed_series or "").strip() + if issue_signal == "comicinfo": + source_issue_number = comicinfo.get("number") + else: + source_issue_number = imported_file.issue_number_raw or imported_file.parsed_issue_number + if not source_series_name or not isinstance(source_issue_number, str | float | int): + return None + + trusted_series_cv_id = ( + _safe_int(diagnostics.get("comicvine_series_id")) + if str(signals.get("comicvine_series_id") or "") in trusted_signals + else None + ) + trusted_issue_cv_id = ( + imported_file.comicvine_issue_id + if str(signals.get("comicvine_issue_id") or "") in trusted_signals + else None + ) + return ( + source_series_name, + source_issue_number, + trusted_series_cv_id, + trusted_issue_cv_id, + series_signal, + ) + + +async def _load_mixed_folder_resolutions( + session: AsyncSession, + job_id: int, +) -> tuple[_MixedFolderResolution, ...]: + """Resolve exact local targets without provider calls or source-file access.""" + series_signal = ImportedFile.diagnostics["metadata_signals"]["series_name"].as_string() + issue_signal = ImportedFile.diagnostics["metadata_signals"]["issue_number"].as_string() + source_title_expression = case( + ( + series_signal == "comicinfo", + ImportedFile.diagnostics["source_metadata"]["comicinfo"]["series"].as_string(), + ), + else_=ImportedFile.parsed_series, + ) + source_rows = ( + await session.execute( + select(ImportedFile, ImportedSeries, LibraryFile) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .outerjoin(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status.in_((ImportedFileStatus.NO_MATCH, ImportedFileStatus.IMPORTED)), + series_signal.in_(("comicinfo", "sidecar")), + issue_signal.in_(("comicinfo", "sidecar")), + func.lower(func.trim(source_title_expression)) + != func.lower( + func.trim( + func.coalesce( + func.nullif(ImportedSeries.cv_title, ""), ImportedSeries.raw_series_name + ) + ) + ), + ) + .order_by(ImportedFile.id) + ) + ).all() + if not source_rows: + return () + + current_library_by_file_id: dict[int, LibraryFile] = {} + for imported_file, _imported_series, library_file in source_rows: + if ( + imported_file.status is ImportedFileStatus.IMPORTED + and library_file is not None + and library_file.storage_mode is LibraryFileStorageMode.REFERENCED + and library_file.file_path == imported_file.file_path + and library_file.issue_id == imported_file.matched_issue_id + ): + current_library_by_file_id[int(imported_file.id)] = library_file + + source_candidates: list[ + tuple[ImportedFile, ImportedSeries, str, str, int | None, int | None, str] + ] = [] + normalized_titles: set[str] = set() + trusted_series_cv_ids: set[int] = set() + for imported_file, imported_series, _library_file in source_rows: + if ( + imported_file.status is ImportedFileStatus.IMPORTED + and int(imported_file.id) not in current_library_by_file_id + ): + continue + identity = _mixed_folder_source_identity(imported_file) + if identity is None: + continue + source_title, raw_number, series_cv_id, issue_cv_id, evidence_source = identity + normalized_source_title = NameMatcher.normalize(source_title) + parent_title = imported_series.cv_title or imported_series.raw_series_name + if not normalized_source_title or normalized_source_title == NameMatcher.normalize( + parent_title + ): + continue + try: + _numeric_number, exact_number = parse_issue_number_text(raw_number) + except ValueError: + continue + source_candidates.append( + ( + imported_file, + imported_series, + source_title, + exact_number, + series_cv_id, + issue_cv_id, + evidence_source, + ) + ) + normalized_titles.add(normalized_source_title) + if series_cv_id is not None: + trusted_series_cv_ids.add(series_cv_id) + if not source_candidates: + return () + + local_series = list((await session.scalars(select(Series))).all()) + series_by_cv_id = { + int(series.comicvine_id): series + for series in local_series + if series.comicvine_id is not None and int(series.comicvine_id) in trusted_series_cv_ids + } + series_by_title: dict[str, list[Series]] = {} + for series in local_series: + normalized = NameMatcher.normalize(series.title) + if normalized in normalized_titles: + series_by_title.setdefault(normalized, []).append(series) + + candidate_targets: list[ + tuple[ImportedFile, ImportedSeries, str, str, str, Series, int | None] + ] = [] + target_series_ids: set[int] = set() + for ( + imported_file, + imported_series, + source_title, + exact_number, + series_cv_id, + issue_cv_id, + evidence_source, + ) in source_candidates: + target_series = series_by_cv_id.get(series_cv_id) if series_cv_id is not None else None + if target_series is None: + title_matches = series_by_title.get(NameMatcher.normalize(source_title), []) + if len(title_matches) != 1: + continue + target_series = title_matches[0] + if imported_series.series_id == target_series.id: + continue + candidate_targets.append( + ( + imported_file, + imported_series, + source_title, + exact_number, + evidence_source, + target_series, + issue_cv_id, + ) + ) + target_series_ids.add(int(target_series.id)) + if not candidate_targets: + return () + + target_issues = [ + issue + for ids in batched(sorted(target_series_ids), 400) + for issue in (await session.scalars(select(Issue).where(Issue.series_id.in_(ids)))).all() + ] + issues_by_cv_id = { + int(issue.comicvine_id): issue for issue in target_issues if issue.comicvine_id is not None + } + issues_by_number: dict[tuple[int, str], list[Issue]] = {} + for issue in target_issues: + issues_by_number.setdefault( + (int(issue.series_id), issue.effective_issue_number_text), [] + ).append(issue) + + resolved_targets: list[tuple[ImportedFile, ImportedSeries, str, str, Series, Issue]] = [] + for ( + imported_file, + imported_series, + source_title, + exact_number, + evidence_source, + target_series, + issue_cv_id, + ) in candidate_targets: + target_issue: Issue | None = ( + issues_by_cv_id.get(issue_cv_id) if issue_cv_id is not None else None + ) + if target_issue is not None and target_issue.series_id != target_series.id: + target_issue = None + if target_issue is None: + number_matches = issues_by_number.get((int(target_series.id), exact_number), []) + if len(number_matches) != 1: + continue + target_issue = number_matches[0] + resolved_targets.append( + ( + imported_file, + imported_series, + source_title, + evidence_source, + target_series, + target_issue, + ) + ) + if not resolved_targets: + return () + + target_issue_ids = {int(item[5].id) for item in resolved_targets} + owned_files_by_issue_id: dict[int, list[int]] = {} + for ids in batched(sorted(target_issue_ids), 400): + for library_file in ( + await session.scalars(select(LibraryFile).where(LibraryFile.issue_id.in_(ids))) + ).all(): + if library_file.issue_id is not None: + owned_files_by_issue_id.setdefault(int(library_file.issue_id), []).append( + int(library_file.id) + ) + files_by_target_issue: dict[int, list[int]] = {} + for imported_file, _series, _title, _source, _target_series, issue in resolved_targets: + files_by_target_issue.setdefault(int(issue.id), []).append(int(imported_file.id)) + + resolutions: list[_MixedFolderResolution] = [] + for ( + imported_file, + imported_series, + source_title, + evidence_source, + target_series, + issue, + ) in resolved_targets: + existing_library_file_ids = owned_files_by_issue_id.get(int(issue.id), []) + if len(existing_library_file_ids) > 1: + continue + library_file_id = existing_library_file_ids[0] if existing_library_file_ids else None + if library_file_id is None and len(files_by_target_issue[int(issue.id)]) != 1: + continue + current_library_file = current_library_by_file_id.get(int(imported_file.id)) + resolutions.append( + _MixedFolderResolution( + file_id=int(imported_file.id), + source_import_series_id=int(imported_series.id), + source_import_series_name=imported_series.raw_series_name, + target_series_id=int(target_series.id), + target_series_title=target_series.title, + target_issue_id=int(issue.id), + target_issue_cv_id=( + int(issue.comicvine_id) if issue.comicvine_id is not None else None + ), + target_issue_number=float(issue.issue_number), + target_issue_number_text=issue.effective_issue_number_text, + target_library_file_id=library_file_id, + source_library_file_id=( + int(current_library_file.id) if current_library_file is not None else None + ), + source_issue_id=( + int(current_library_file.issue_id) + if current_library_file is not None + and current_library_file.issue_id is not None + else None + ), + source_library_updated_at=( + current_library_file.updated_at.isoformat(timespec="microseconds") + if current_library_file is not None + else None + ), + evidence_source=evidence_source, + source_series_name=source_title, + source_updated_at=imported_file.updated_at.isoformat(timespec="microseconds"), + ) + ) + return tuple(resolutions) + + +async def _load_completed_job(session: AsyncSession, job_id: int) -> ImportJob: + job = await session.get(ImportJob, job_id, populate_existing=True) + if job is None: + raise NotFoundError("ImportJob", job_id) + if not allows_terminal_import_recovery(job): + raise ValidationError( + "Job must have a COMPLETED canonical import with no pending control or rollback " + "work for recovery cleanup" + ) + return job + + +def _safe_example_name(value: str) -> str: + normalized = value.replace("\\", "/").rstrip("/") + leaf = normalized.rsplit("/", maxsplit=1)[-1] + safe_leaf = "".join(character for character in leaf if character >= " " and character != "\x7f") + return (safe_leaf or "File")[:200] + + +async def _load_snapshot( + session: AsyncSession, + job_id: int, + action: CompletedImportCleanupAction, +) -> CompletedImportCleanupSnapshot: + if action is CompletedImportCleanupAction.RECOVER_KNOWN_SERIES: + plans = await load_known_series_recovery(session, job_id) + digest = sha256() + for plan in plans: + digest.update(f"{plan.file_id}|{plan.cv_id}|{plan.evidence_digest}\n".encode()) + return CompletedImportCleanupSnapshot( + len(plans), + len(plans), + plans[0].file_id if plans else None, + plans[-1].file_id if plans else None, + None, + digest.hexdigest(), + ) + if action is CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES: + resolutions = await _load_mixed_folder_resolutions(session, job_id) + digest = sha256() + for resolution in resolutions: + digest.update( + ( + f"{resolution.file_id}|{resolution.target_series_id}|" + f"{resolution.target_issue_id}|{resolution.target_library_file_id or 0}|" + f"{resolution.source_library_file_id or 0}|" + f"{resolution.source_library_updated_at or ''}|" + f"{resolution.source_updated_at}\n" + ).encode() + ) + file_ids = [resolution.file_id for resolution in resolutions] + return CompletedImportCleanupSnapshot( + affected_count=len(resolutions), + affected_file_count=len(resolutions), + min_file_id=min(file_ids) if file_ids else None, + max_file_id=max(file_ids) if file_ids else None, + max_updated_at=max( + (resolution.source_updated_at for resolution in resolutions), + default=None, + ), + scope_digest=digest.hexdigest(), + ) + filters = _file_filters(job_id, action) + stale_series = ( + await load_empty_stale_series(session, job_id) + if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES + else [] + ) + aggregate = ( + await session.execute( + select( + func.count(ImportedFile.id), + func.min(ImportedFile.id), + func.max(ImportedFile.id), + func.max(ImportedFile.updated_at), + ).where(*filters) + ) + ).one() + file_count = int(aggregate[0] or 0) + if file_count == 0 and not stale_series: + return CompletedImportCleanupSnapshot(0, 0, None, None, None, sha256().hexdigest()) + + digest = sha256() + group_ids: set[int] = set() + if file_count: + result = await session.stream( + select( + ImportedFile.id, + ImportedFile.conflict_group_id, + ImportedFile.updated_at, + ) + .where(*filters) + .order_by(ImportedFile.id) + .execution_options(yield_per=20_000) + ) + try: + async for rows in result.partitions(20_000): + for file_id, conflict_group_id, updated_at in rows: + digest_line = ( + f"file|{int(file_id)}|{int(conflict_group_id or 0)}|" + f"{updated_at.isoformat()}\n" + ) + digest.update(digest_line.encode()) + if conflict_group_id is not None: + group_ids.add(int(conflict_group_id)) + finally: + await result.close() + for item in stale_series: + digest.update(f"series|{int(item.id)}|{item.updated_at.isoformat()}\n".encode()) + affected_count = ( + len(group_ids) + if action is CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS + else file_count + ) + if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: + affected_count = len(stale_series) + int( + await session.scalar( + select(func.count(func.distinct(ImportedFile.file_path))).where(*filters) + ) + or 0 + ) + updated_at_values = [ + item.updated_at.isoformat(timespec="microseconds") for item in stale_series + ] + if aggregate[3] is not None: + updated_at_values.append(aggregate[3].isoformat(timespec="microseconds")) + return CompletedImportCleanupSnapshot( + affected_count=affected_count, + affected_file_count=file_count, + min_file_id=int(aggregate[1]) if aggregate[1] is not None else None, + max_file_id=int(aggregate[2]) if aggregate[2] is not None else None, + max_updated_at=max(updated_at_values, default=None), + scope_digest=digest.hexdigest(), + ) + + +async def count_completed_import_cleanup_scope( + session: AsyncSession, + job_id: int, + action: CompletedImportCleanupAction, +) -> tuple[int, int]: + """Return action and file counts without hashing the full preview scope.""" + if action is CompletedImportCleanupAction.RECOVER_KNOWN_SERIES: + count = len(await load_known_series_recovery(session, job_id)) + return count, count + if action is CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES: + count = len(await _load_mixed_folder_resolutions(session, job_id)) + return count, count + filters = _file_filters(job_id, action) + file_count = int( + (await session.scalar(select(func.count(ImportedFile.id)).where(*filters))) or 0 + ) + if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: + paths = int( + await session.scalar( + select(func.count(func.distinct(ImportedFile.file_path))).where(*filters) + ) + or 0 + ) + return paths + len(await load_empty_stale_series(session, job_id)), file_count + if action is not CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS: + return file_count, file_count + group_count = int( + ( + await session.scalar( + select(func.count(func.distinct(ImportedFile.conflict_group_id))).where(*filters) + ) + ) + or 0 + ) + return group_count, file_count + + +async def list_completed_import_cleanup_files( + session: AsyncSession, + job_id: int, + action: CompletedImportCleanupAction, + *, + page: int = 1, + page_size: int = 25, +) -> CompletedImportCleanupFilePage: + """Return a bounded, deterministic page for user review.""" + await _load_completed_job(session, job_id) + normalized_page = max(1, int(page)) + normalized_page_size = min(max(1, int(page_size)), 100) + if action in { + CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES, + CompletedImportCleanupAction.RECOVER_KNOWN_SERIES, + }: + eligible_file_ids = await _identity_recovery_file_ids(session, job_id, action) + total = len(eligible_file_ids) + total_pages = max(1, (total + normalized_page_size - 1) // normalized_page_size) + normalized_page = min(normalized_page, total_pages) + page_file_ids = eligible_file_ids[ + (normalized_page - 1) * normalized_page_size : normalized_page * normalized_page_size + ] + items_by_id = { + int(item.id): item + for item in ( + await session.scalars( + select(ImportedFile).where(ImportedFile.id.in_(page_file_ids)) + ) + ).all() + } + items = tuple(items_by_id[file_id] for file_id in page_file_ids) + return CompletedImportCleanupFilePage( + items=items, + total=total, + page=normalized_page, + page_size=normalized_page_size, + total_pages=total_pages, + ) + filters = _file_filters(job_id, action) + if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: + filters = ( + ImportedFile.id.in_( + select(func.min(ImportedFile.id)).where(*filters).group_by(ImportedFile.file_path) + ), + ) + total = int((await session.scalar(select(func.count(ImportedFile.id)).where(*filters))) or 0) + total_pages = max(1, (total + normalized_page_size - 1) // normalized_page_size) + normalized_page = min(normalized_page, total_pages) + items = tuple( + ( + await session.scalars( + select(ImportedFile) + .where(*filters) + .order_by(ImportedFile.id) + .offset((normalized_page - 1) * normalized_page_size) + .limit(normalized_page_size) + ) + ).all() + ) + return CompletedImportCleanupFilePage( + items=items, + total=total, + page=normalized_page, + page_size=normalized_page_size, + total_pages=total_pages, + ) + + +async def list_completed_import_cleanup_examples( + session: AsyncSession, + job_id: int, + action: CompletedImportCleanupAction, + *, + limit: int = _EXAMPLE_LIMIT, +) -> tuple[str, ...]: + """Return sanitized example filenames without hydrating the full scope.""" + if action in { + CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES, + CompletedImportCleanupAction.RECOVER_KNOWN_SERIES, + }: + file_ids = (await _identity_recovery_file_ids(session, job_id, action))[:limit] + names_by_id = { + int(file_id): file_name + for file_id, file_name in ( + await session.execute( + select(ImportedFile.id, ImportedFile.file_name).where( + ImportedFile.id.in_(file_ids) + ) + ) + ).all() + } + return tuple(_safe_example_name(names_by_id[file_id]) for file_id in file_ids) + filters = _file_filters(job_id, action) + if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: + filters = ( + ImportedFile.id.in_( + select(func.min(ImportedFile.id)).where(*filters).group_by(ImportedFile.file_path) + ), + ) + names = ( + await session.scalars( + select(ImportedFile.file_name) + .where(*filters) + .order_by(ImportedFile.id) + .limit(min(max(1, int(limit)), 10)) + ) + ).all() + return tuple(_safe_example_name(name) for name in names) + + +async def summarize_completed_import_cleanup_scope( + session: AsyncSession, + job_id: int, + action: CompletedImportCleanupAction, + *, + example_limit: int = _EXAMPLE_LIMIT, +) -> CompletedImportCleanupSummary: + """Load a recovery-card summary without resolving mixed folders twice.""" + normalized_limit = min(max(1, int(example_limit)), 10) + if action not in { + CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES, + CompletedImportCleanupAction.RECOVER_KNOWN_SERIES, + }: + affected_count, affected_file_count = await count_completed_import_cleanup_scope( + session, + job_id, + action, + ) + examples = ( + await list_completed_import_cleanup_examples( + session, + job_id, + action, + limit=normalized_limit, + ) + if affected_count + else () + ) + return CompletedImportCleanupSummary( + affected_count=affected_count, + affected_file_count=affected_file_count, + examples=examples, + ) + + eligible_file_ids = await _identity_recovery_file_ids(session, job_id, action) + file_ids = eligible_file_ids[:normalized_limit] + names_by_id = { + int(file_id): file_name + for file_id, file_name in ( + await session.execute( + select(ImportedFile.id, ImportedFile.file_name).where(ImportedFile.id.in_(file_ids)) + ) + ).all() + } + return CompletedImportCleanupSummary( + affected_count=len(eligible_file_ids), + affected_file_count=len(eligible_file_ids), + examples=tuple(_safe_example_name(names_by_id[file_id]) for file_id in file_ids), + ) + + +async def _identity_recovery_file_ids( + session: AsyncSession, + job_id: int, + action: CompletedImportCleanupAction, +) -> list[int]: + if action is CompletedImportCleanupAction.RECOVER_KNOWN_SERIES: + return [plan.file_id for plan in await load_known_series_recovery(session, job_id)] + return [plan.file_id for plan in await _load_mixed_folder_resolutions(session, job_id)] + + +def _snapshot_payload(snapshot: CompletedImportCleanupSnapshot) -> dict[str, object]: + return { + "affected_count": snapshot.affected_count, + "affected_file_count": snapshot.affected_file_count, + "min_file_id": snapshot.min_file_id, + "max_file_id": snapshot.max_file_id, + "max_updated_at": snapshot.max_updated_at, + "scope_digest": snapshot.scope_digest, + } + + +def _load_token(token: str) -> Mapping[str, object]: + try: + payload = _serializer().loads(token, max_age=_PREVIEW_TOKEN_MAX_AGE_SECONDS) + except SignatureExpired as exc: + raise ValidationError("The cleanup preview expired. Preview the action again.") from exc + except BadSignature as exc: + raise ValidationError("The cleanup preview is invalid. Preview the action again.") from exc + if not isinstance(payload, Mapping): + raise ValidationError("The cleanup preview is invalid. Preview the action again.") + return payload + + +def _snapshot_from_payload(payload: Mapping[str, object]) -> CompletedImportCleanupSnapshot: + raw = payload.get("snapshot") + if not isinstance(raw, Mapping): + raise ValidationError("The cleanup preview is invalid. Preview the action again.") + try: + affected_count = int(raw["affected_count"]) + affected_file_count = int(raw["affected_file_count"]) + min_file_id = int(raw["min_file_id"]) if raw.get("min_file_id") is not None else None + max_file_id = int(raw["max_file_id"]) if raw.get("max_file_id") is not None else None + except (KeyError, TypeError, ValueError) as exc: + raise ValidationError("The cleanup preview is invalid. Preview the action again.") from exc + max_updated_at = raw.get("max_updated_at") + scope_digest = raw.get("scope_digest") + if not isinstance(max_updated_at, (str, type(None))) or not isinstance(scope_digest, str): + raise ValidationError("The cleanup preview is invalid. Preview the action again.") + return CompletedImportCleanupSnapshot( + affected_count, + affected_file_count, + min_file_id, + max_file_id, + max_updated_at, + scope_digest, + ) + + +async def preview_completed_import_cleanup( + session: AsyncSession, + job_id: int, + action: CompletedImportCleanupAction, + *, + actor_id: int, +) -> CompletedImportCleanupPreview: + """Return a bounded preview and actor-bound confirmation token.""" + job = await _load_completed_job(session, job_id) + snapshot = await _load_snapshot(session, job_id, action) + if snapshot.affected_count == 0: + raise ValidationError("No files are eligible for this cleanup action.") + examples = await list_completed_import_cleanup_examples( + session, + job_id, + action, + ) + token = str( + _serializer().dumps( + { + "version": _PREVIEW_TOKEN_VERSION, + "job_id": job.id, + "action": action.value, + "actor_id": actor_id, + "snapshot": _snapshot_payload(snapshot), + } + ) + ) + return CompletedImportCleanupPreview( + job_id=job.id, + action=action, + affected_count=snapshot.affected_count, + affected_file_count=snapshot.affected_file_count, + item_unit=( + "group" + if action is CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS + else "file" + ), + examples=examples, + preview_token=token, + ) + + +def _validate_token( + token: str, + *, + job_id: int, + action: CompletedImportCleanupAction, + actor_id: int, +) -> CompletedImportCleanupSnapshot: + payload = _load_token(token) + if ( + payload.get("version") != _PREVIEW_TOKEN_VERSION + or payload.get("job_id") != job_id + or payload.get("action") != action.value + or payload.get("actor_id") != actor_id + ): + raise ValidationError("The cleanup preview does not match this job and action.") + return _snapshot_from_payload(payload) + + +def _mark_skipped(file: ImportedFile, *, action: CompletedImportCleanupAction) -> None: + diagnostics = dict(file.diagnostics or {}) + diagnostics["completed_import_cleanup"] = { + "action": action.value, + "resolved_at": datetime.now(UTC).isoformat(), + "source_preserved": True, + } + file.status = ImportedFileStatus.SKIPPED + file.include_in_import = False + file.error_message = None + file.match_method = "completed_import_cleanup" + file.diagnostics = diagnostics + + +def _prepare_source_retry(file: ImportedFile) -> None: + diagnostics = dict(file.diagnostics or {}) + raw_block = diagnostics.get("safety_block") + raw_revalidation = diagnostics.get("source_revalidation") + if isinstance(raw_block, Mapping): + retry_evidence = dict(raw_block) + diagnostics.pop("safety_block", None) + elif isinstance(raw_revalidation, Mapping) and raw_revalidation.get("retryable") is True: + retry_evidence = dict(raw_revalidation) + else: + raise ValidationError("A selected source failure no longer has safety evidence.") + diagnostics["source_revalidation"] = { + **retry_evidence, + "kind": "source_revalidation", + "retryable": True, + "source": "completed_import_cleanup", + } + file.status = ImportedFileStatus.FAILED + file.include_in_import = False + file.diagnostics = diagnostics + + +async def _apply_file_action( + session: AsyncSession, + job: ImportJob, + action: CompletedImportCleanupAction, +) -> tuple[set[int], tuple[int, ...], bool]: + affected_series_ids: set[int] = set() + affected_file_ids: list[int] = [] + requires_import_retry = False + cursor = 0 + while True: + files = list( + ( + await session.scalars( + select(ImportedFile) + .where(*_file_filters(job.id, action), ImportedFile.id > cursor) + .order_by(ImportedFile.id) + .limit(_PAGE_SIZE) + ) + ).all() + ) + if not files: + break + cursor = int(files[-1].id) + page_file_ids: list[int] = [] + for file in files: + affected_series_ids.add(int(file.import_series_id)) + affected_file_ids.append(int(file.id)) + page_file_ids.append(int(file.id)) + if action in { + CompletedImportCleanupAction.DISMISS_MISSING_REFERENCES, + CompletedImportCleanupAction.SKIP_PROBABLE_COVERS, + CompletedImportCleanupAction.SKIP_UNUSABLE_FILES, + }: + _mark_skipped(file, action=action) + elif action is CompletedImportCleanupAction.ALLOW_OVERSIZED_FILES: + apply_safety_allow_once_to_file(file, retry_import=True) + requires_import_retry = True + elif action is CompletedImportCleanupAction.RETRY_SOURCE_INSPECTION: + _prepare_source_retry(file) + requires_import_retry = True + elif action is CompletedImportCleanupAction.NORMALIZE_ALREADY_OWNED: + file.status = ImportedFileStatus.ALREADY_OWNED + file.include_in_import = False + file.error_message = None + else: # pragma: no cover - conflict groups use a separate path + raise ValidationError("Unsupported file cleanup action.") + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job.id, + import_file_ids=page_file_ids, + ) + await session.flush() + return affected_series_ids, tuple(affected_file_ids), requires_import_retry + + +async def _apply_recommended_conflicts( + session: AsyncSession, + job: ImportJob, +) -> set[int]: + eligible_group_ids = [ + int(group_id) + for group_id in ( + await session.scalars( + select(ImportedFile.conflict_group_id) + .where( + ImportedFile.import_job_id == job.id, + ImportedFile.status == ImportedFileStatus.CONFLICT, + ImportedFile.conflict_group_id.in_(_eligible_conflict_groups(job.id)), + ) + .distinct() + .order_by(ImportedFile.conflict_group_id) + ) + ).all() + if group_id is not None + ] + if not eligible_group_ids: + return set() + affected_series_ids = set( + await session.scalars( + select(ImportedFile.import_series_id) + .where( + ImportedFile.import_job_id == job.id, + ImportedFile.status == ImportedFileStatus.CONFLICT, + ImportedFile.conflict_group_id.in_(eligible_group_ids), + ) + .distinct() + ) + ) + await session.execute( + update(ImportedFile) + .where( + ImportedFile.import_job_id == job.id, + ImportedFile.status == ImportedFileStatus.CONFLICT, + ImportedFile.conflict_group_id.in_(eligible_group_ids), + ImportedFile.is_preferred.is_(False), + ) + .values(status=ImportedFileStatus.SKIPPED, include_in_import=False) + ) + await session.execute( + update(ImportedFile) + .where( + ImportedFile.import_job_id == job.id, + ImportedFile.status == ImportedFileStatus.CONFLICT, + ImportedFile.conflict_group_id.in_(eligible_group_ids), + ImportedFile.is_preferred.is_(True), + ImportedFile.match_confidence == "high", + ) + .values(status=ImportedFileStatus.CONFIRMED, include_in_import=True) + ) + await session.flush() + return {int(series_id) for series_id in affected_series_ids} + + +async def _apply_mixed_folder_resolutions( + session: AsyncSession, + job: ImportJob, +) -> tuple[set[int], set[int]]: + """Rebucket exact embedded identities while preserving every source artifact.""" + resolutions = await _load_mixed_folder_resolutions(session, int(job.id)) + if not resolutions: + return set(), set() + + target_series_ids = {resolution.target_series_id for resolution in resolutions} + target_series_by_id = { + int(series.id): series + for series in ( + await session.scalars(select(Series).where(Series.id.in_(target_series_ids))) + ).all() + } + target_issue_by_id = { + int(issue.id): issue + for issue in ( + await session.scalars( + select(Issue).where( + Issue.id.in_({resolution.target_issue_id for resolution in resolutions}) + ) + ) + ).all() + } + source_library_by_id = { + int(library_file.id): library_file + for library_file in ( + await session.scalars( + select(LibraryFile).where( + LibraryFile.id.in_( + { + resolution.source_library_file_id + for resolution in resolutions + if resolution.source_library_file_id is not None + } + ) + ) + ) + ).all() + } + source_issue_by_id = { + int(issue.id): issue + for issue in ( + await session.scalars( + select(Issue).where( + Issue.id.in_( + { + resolution.source_issue_id + for resolution in resolutions + if resolution.source_issue_id is not None + } + ) + ) + ) + ).all() + } + source_series_by_id = { + int(series.id): series + for series in ( + await session.scalars( + select(Series).where( + Series.id.in_({issue.series_id for issue in source_issue_by_id.values()}) + ) + ) + ).all() + } + source_issue_owner_counts = { + int(issue_id): int(owner_count) + for issue_id, owner_count in ( + await session.execute( + select(LibraryFile.issue_id, func.count(LibraryFile.id)) + .where(LibraryFile.issue_id.in_(set(source_issue_by_id))) + .group_by(LibraryFile.issue_id) + ) + ).all() + if issue_id is not None + } + imported_target_by_series_id: dict[int, ImportedSeries] = {} + existing_target_rows = list( + ( + await session.scalars( + select(ImportedSeries) + .where( + ImportedSeries.import_job_id == job.id, + ImportedSeries.series_id.in_(target_series_ids), + ) + .order_by(ImportedSeries.id) + ) + ).all() + ) + for imported_series in existing_target_rows: + if imported_series.series_id is not None: + imported_target_by_series_id.setdefault(int(imported_series.series_id), imported_series) + + affected_series_ids: set[int] = set() + retry_series_ids: set[int] = set() + affected_file_ids: list[int] = [] + for resolution in resolutions: + target_import_series = imported_target_by_series_id.get(resolution.target_series_id) + if target_import_series is None: + target_series = target_series_by_id[resolution.target_series_id] + target_import_series = ImportedSeries( + import_job_id=job.id, + raw_series_name=target_series.title, + raw_year=target_series.year_start, + file_count=0, + sample_paths=[], + has_files=True, + cv_id=target_series.comicvine_id, + cv_title=target_series.title, + cv_year=target_series.year_start, + cv_match_score=1.0, + cv_match_method="completed_import_mixed_folder_recovery", + status=ImportSeriesStatus.DUPLICATE, + selected_for_import=False, + series_id=target_series.id, + diagnostics={ + "kind": "completed_import_mixed_folder_recovery", + "existing_series_id": target_series.id, + "source_preserved": True, + }, + ) + session.add(target_import_series) + await session.flush() + imported_target_by_series_id[resolution.target_series_id] = target_import_series + + imported_file = await session.get(ImportedFile, resolution.file_id) + if imported_file is None: # pragma: no cover - signed snapshot guards deletion + raise ValidationError("A mixed-folder file disappeared. Preview the action again.") + affected_file_ids.append(int(imported_file.id)) + diagnostics = dict(imported_file.diagnostics or {}) + diagnostics["completed_import_cleanup"] = { + "action": CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES.value, + "resolved_at": datetime.now(UTC).isoformat(), + "source_preserved": True, + "evidence_source": resolution.evidence_source, + "source_import_series_id": resolution.source_import_series_id, + "source_import_series_name": resolution.source_import_series_name, + "source_series_name": resolution.source_series_name, + "target_import_series_id": target_import_series.id, + "target_series_id": resolution.target_series_id, + "target_series_title": resolution.target_series_title, + "target_issue_id": resolution.target_issue_id, + "target_issue_number": resolution.target_issue_number_text, + } + imported_file.import_series_id = int(target_import_series.id) + imported_file.parsed_series = resolution.target_series_title + imported_file.parsed_issue_number = resolution.target_issue_number + imported_file.issue_number_raw = resolution.target_issue_number_text + imported_file.matched_issue_id = resolution.target_issue_id + imported_file.matched_issue_cv_id = resolution.target_issue_cv_id + imported_file.match_confidence = "high" + imported_file.match_method = "completed_import_metadata_reassignment" + imported_file.conflict_group_id = None + imported_file.duplicate_group_id = None + imported_file.duplicate_of_file_id = None + imported_file.is_preferred = False + imported_file.error_message = None + imported_file.diagnostics = diagnostics + source_library_file = ( + source_library_by_id.get(resolution.source_library_file_id) + if resolution.source_library_file_id is not None + else None + ) + target_issue = target_issue_by_id[resolution.target_issue_id] + if source_library_file is not None: + source_issue_id_before = source_library_file.issue_id + previous_issue = ( + source_issue_by_id.get(resolution.source_issue_id) + if resolution.source_issue_id is not None + else None + ) + if source_library_file.issue_id == resolution.target_issue_id: + imported_file.status = ImportedFileStatus.IMPORTED + imported_file.include_in_import = False + imported_file.library_file_id = source_library_file.id + target_issue.status = IssueStatus.OWNED + elif ( + resolution.target_library_file_id is not None + and resolution.target_library_file_id != source_library_file.id + ): + imported_file.status = ImportedFileStatus.ALREADY_OWNED + imported_file.include_in_import = False + imported_file.library_file_id = resolution.target_library_file_id + await session.delete(source_library_file) + target_issue.status = IssueStatus.OWNED + else: + source_library_file.issue_id = resolution.target_issue_id + imported_file.status = ImportedFileStatus.IMPORTED + imported_file.include_in_import = False + imported_file.library_file_id = source_library_file.id + target_issue.status = IssueStatus.OWNED + if previous_issue is not None and previous_issue.id != resolution.target_issue_id: + previous_series = source_series_by_id.get(int(previous_issue.series_id)) + remaining_owners = source_issue_owner_counts.get(int(previous_issue.id), 0) + if source_issue_id_before == previous_issue.id: + remaining_owners = max(0, remaining_owners - 1) + source_issue_owner_counts[int(previous_issue.id)] = remaining_owners + if remaining_owners: + previous_issue.status = IssueStatus.OWNED + else: + previous_issue.status = ( + IssueStatus.WANTED + if previous_series is not None and previous_series.monitored + else IssueStatus.SKIPPED + ) + elif resolution.target_library_file_id is not None: + imported_file.status = ImportedFileStatus.ALREADY_OWNED + imported_file.include_in_import = False + imported_file.library_file_id = resolution.target_library_file_id + target_issue.status = IssueStatus.OWNED + else: + imported_file.status = ImportedFileStatus.CONFIRMED + imported_file.include_in_import = True + retry_series_ids.add(int(target_import_series.id)) + target_import_series.status = ImportSeriesStatus.DUPLICATE + target_import_series.selected_for_import = True + target_import_series.error_message = None + + affected_series_ids.update( + {resolution.source_import_series_id, int(target_import_series.id)} + ) + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job.id, + import_file_ids=affected_file_ids, + ) + await session.flush() + return affected_series_ids, retry_series_ids + + +async def _prepare_series_for_retry( + session: AsyncSession, + job: ImportJob, + series_ids: set[int], +) -> bool: + if not series_ids: + return False + remaining_conflicts = set( + await session.scalars( + select(ImportedFile.import_series_id) + .where( + ImportedFile.import_series_id.in_(series_ids), + ImportedFile.status == ImportedFileStatus.CONFLICT, + ) + .distinct() + ) + ) + retry_series_ids = sorted(series_ids - {int(value) for value in remaining_conflicts}) + if not retry_series_ids: + return False + await session.execute( + update(ImportedSeries) + .where(ImportedSeries.id.in_(retry_series_ids)) + .values( + status=ImportSeriesStatus.CONFIRMED, + selected_for_import=True, + error_message=None, + ) + ) + job.status = ImportJobStatus.IMPORTING + job.error_message = None + return True + + +async def _apply_known_series_recovery(session: AsyncSession, job: ImportJob) -> set[int]: + plans = await load_known_series_recovery(session, job.id) + affected: set[int] = set() + for batch in batched(plans, 400): + items = { + item.id: item + for item in ( + await session.scalars( + select(ImportedSeries).where( + ImportedSeries.id.in_({plan.series_id for plan in batch}) + ) + ) + ).all() + } + files = { + file.id: file + for file in ( + await session.scalars( + select(ImportedFile).where( + ImportedFile.id.in_([plan.file_id for plan in batch]) + ) + ) + ).all() + } + for plan in batch: + item = items.get(plan.series_id) + file = files.get(plan.file_id) + if item is None or file is None: + raise ValidationError("Recovery evidence disappeared. Preview the action again.") + _apply_known_series_file(item, file, plan, affected) + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job.id, + import_file_ids=list(files), + ) + await session.flush() + return affected + + +def _apply_known_series_file( + item: ImportedSeries, + file: ImportedFile, + plan: KnownSeriesRecovery, + affected: set[int], +) -> None: + if item.id not in affected: + candidate = dict(item.diagnostics or {}).get("selected_candidate") + candidate = candidate if isinstance(candidate, dict) else {} + item.cv_id = plan.cv_id + item.cv_match_method = plan.match_method + item.cv_match_score = 1.0 + item.cv_title = item.raw_series_name + item.cv_year = item.raw_year + item.cv_issue_count = candidate.get("issue_count") + item.diagnostics = { + **dict(item.diagnostics or {}), + "previous_reason": "trusted_source_identity_conflict", + "reason": "known_series_recovered", + "file_identity_review_required": True, + } + affected.add(item.id) + file.status = ImportedFileStatus.CONFIRMED + file.include_in_import = True + file.matched_issue_cv_id = int(plan.summary["provider_id"]) + file.match_confidence = "high" + file.match_method = "comicvine_id" + file.error_message = None + file.diagnostics = { + **dict(file.diagnostics or {}), + "target_issue_summary": plan.summary, + "completed_import_cleanup": { + "action": CompletedImportCleanupAction.RECOVER_KNOWN_SERIES.value, + "evidence_digest": plan.evidence_digest, + "source_preserved": True, + "resolved_at": datetime.now(UTC).isoformat(), + }, + } + + +async def apply_completed_import_cleanup( + session: AsyncSession, + job_id: int, + action: CompletedImportCleanupAction, + *, + actor_id: int, + preview_token: str, + actor_username: str | None = None, + source_ip: str | None = None, +) -> CompletedImportCleanupResult: + """Apply exactly the previewed cleanup scope without touching source files.""" + job = await _load_completed_job(session, job_id) + preview_snapshot = _validate_token( + preview_token, + job_id=job_id, + action=action, + actor_id=actor_id, + ) + current_snapshot = await _load_snapshot(session, job_id, action) + if current_snapshot != preview_snapshot: + raise ValidationError("The cleanup scope changed. Preview the action again.") + + if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: + from pullbox.services.import_retry_helpers import require_retained_import_destination + + require_retained_import_destination(job) + stale_series_ids = [int(item.id) for item in await load_empty_stale_series(session, job_id)] + job.progress_snapshot = { + **dict(job.progress_snapshot or {}), + "deferred_recovery": { + "state": "queued", + "run_id": uuid4().hex, + "series_ids": [], + "stale_series_ids": stale_series_ids, + "actor_id": actor_id, + }, + "mode": "import", + "phase": "deferred_recovery", + "progress": 0, + "message": "Queued deferred file recovery...", + } + job.status = ImportJobStatus.IMPORTING + job.error_message = None + affected_series_ids = set() + affected_file_ids: tuple[int, ...] = () + requires_import_retry = True + elif action is CompletedImportCleanupAction.RECOVER_KNOWN_SERIES: + affected_series_ids = await _apply_known_series_recovery(session, job) + affected_file_ids = () + requires_import_retry = await _prepare_series_for_retry(session, job, affected_series_ids) + elif action is CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS: + affected_series_ids = await _apply_recommended_conflicts(session, job) + affected_file_ids = () + requires_import_retry = await _prepare_series_for_retry(session, job, affected_series_ids) + elif action is CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES: + affected_series_ids, retry_series_ids = await _apply_mixed_folder_resolutions(session, job) + affected_file_ids = () + requires_import_retry = await _prepare_series_for_retry(session, job, retry_series_ids) + else: + affected_series_ids, affected_file_ids, requires_import_retry = await _apply_file_action( + session, job, action + ) + if action is CompletedImportCleanupAction.ALLOW_OVERSIZED_FILES: + requires_import_retry = await _prepare_series_for_retry( + session, job, affected_series_ids + ) + + if action is not CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: + await recompute_file_counters(session, job, series_ids=sorted(affected_series_ids)) + await recompute_series_counters(session, job) + result = CompletedImportCleanupResult( + job_id=job.id, + action=action, + affected_count=preview_snapshot.affected_count, + affected_file_count=preview_snapshot.affected_file_count, + requires_import_retry=requires_import_retry, + retry_file_ids=( + affected_file_ids + if action is CompletedImportCleanupAction.RETRY_SOURCE_INSPECTION + else () + ), + ) + item_unit = ( + "group" + if action is CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS + else "follow-up item" + if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES + else "file" + ) + session.add( + ImportJobLog( + import_job_id=job.id, + level="INFO", + event="import_completed_cleanup_applied", + message=( + f"Applied {action.value} to {result.affected_count} " + f"{item_unit}" + f"{'s' if result.affected_count != 1 else ''}." + ), + data={ + "action": action.value, + "affected_count": result.affected_count, + "affected_file_count": result.affected_file_count, + "requires_import_retry": result.requires_import_retry, + "source_preserved": True, + }, + ) + ) + await AuditService.log_event( + session, + AuditEventType.IMPORT_RECOVERY_BULK_ACTION, + source_ip=source_ip, + user_id=actor_id, + username=actor_username, + detail="Completed import recovery action applied.", + metadata={ + "job_id": job.id, + "action": action.value, + "affected_count": result.affected_count, + "affected_file_count": result.affected_file_count, + "source_preserved": True, + }, + ) + await session.flush() + return result diff --git a/src/pullbox/services/import_confirm_policy.py b/src/pullbox/services/import_confirm_policy.py index 41b9d4f3..4932d2aa 100644 --- a/src/pullbox/services/import_confirm_policy.py +++ b/src/pullbox/services/import_confirm_policy.py @@ -6,10 +6,18 @@ from pullbox.core.exceptions import ValidationError from pullbox.core.library_policy import ( + load_effective_library_ingest_policy, load_library_ingest_policy, load_search_on_add_default, ) +from pullbox.models.import_job import ImportFileHandlingMode +from pullbox.models.library import LibraryRoot from pullbox.services.import_policy_snapshot import apply_ingest_policy_to_import_job +from pullbox.services.import_retry_helpers import require_retained_import_destination +from pullbox.services.import_root_policy_activation import ( + apply_future_root_policy_to_ingest_policy, +) +from pullbox.services.library_root_management import validate_managed_library_root if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -24,16 +32,42 @@ async def apply_confirm_import_policy( request: ConfirmImportRequest, ) -> None: """Apply global import policy and confirm-time overrides to an import job.""" + require_retained_import_destination(job) if request.monitored is not None: job.monitored = request.monitored if request.target_library_root_id is not None: + if ( + job.future_layout_requested + and job.target_library_root_id is not None + and request.target_library_root_id != job.target_library_root_id + ): + raise ValidationError( + "The target library root cannot change after future layout setup." + ) + requested_root = await session.get(LibraryRoot, request.target_library_root_id) + if requested_root is None: + raise ValidationError("The selected managed library root does not exist.") + await validate_managed_library_root(requested_root) job.target_library_root_id = request.target_library_root_id - ingest_policy = await load_library_ingest_policy(session) + ingest_policy = ( + await load_effective_library_ingest_policy(session, job.target_library_root_id) + if job.target_library_root_id is not None + else await load_library_ingest_policy(session) + ) + if job.future_layout_requested and job.future_root_policy_snapshot: + ingest_policy = apply_future_root_policy_to_ingest_policy( + ingest_policy, + job.future_root_policy_snapshot, + source_import_job_id=job.id, + ) search_on_add = await load_search_on_add_default(session) + handling_mode = job.file_handling_mode or ImportFileHandlingMode.MANAGED_COPY if request.search_on_add is not None and request.search_on_add != search_on_add: raise ValidationError("Search on add is now controlled by the global import policy.") - if request.move_to_library is False: + if handling_mode == ImportFileHandlingMode.IN_PLACE and request.move_to_library is True: + raise ValidationError("move_to_library=true conflicts with the selected in-place mode.") + if handling_mode == ImportFileHandlingMode.MANAGED_COPY and request.move_to_library is False: raise ValidationError( "Collection import now always creates library artifacts. " "The deprecated move_to_library=false override is no longer supported." diff --git a/src/pullbox/services/import_confirmation.py b/src/pullbox/services/import_confirmation.py index e68b919f..a30ed809 100644 --- a/src/pullbox/services/import_confirmation.py +++ b/src/pullbox/services/import_confirmation.py @@ -18,6 +18,18 @@ ImportSeriesStatus, ) from pullbox.models.issue import Issue +from pullbox.services.import_job_execution_items import ( + ensure_target_issue_summary_for_import_file, +) +from pullbox.services.import_managed_copy_preflight import ( + ManagedCopyPreflightError, + reopen_review_after_managed_copy_preflight_failure, + validate_managed_copy_preflight, +) +from pullbox.services.import_split_series import ( + require_preferred_managed_root_for_selected_split_series, +) +from pullbox.services.import_story_arc_review import confirm_import_story_arcs from pullbox.services.import_workflow_state import initialize_progress_snapshot if TYPE_CHECKING: @@ -84,6 +96,16 @@ async def confirm_import_job( items = await _load_confirmed_series(session, job_id, request) + await require_preferred_managed_root_for_selected_split_series( + session, + job, + preferred_library_root_id=( + request.target_library_root_id + if request.target_library_root_id is not None + else job.target_library_root_id + ), + ) + for item in items: item.status = ImportSeriesStatus.CONFIRMED item.selected_for_import = False @@ -101,21 +123,48 @@ async def confirm_import_job( await apply_confirm_policy(session, job, request) duplicate_selected_count = await _count_selected_duplicate_files(session, job_id) - if not items and duplicate_selected_count == 0: + confirmed_story_arc_count = await confirm_import_story_arcs( + session, + job_id, + story_arc_ids=request.story_arc_ids, + decisions=[ + ( + decision.imported_story_arc_id, + decision.action, + decision.proposed_story_arc_id, + ) + for decision in request.story_arc_decisions + ], + ) + if not items and duplicate_selected_count == 0 and confirmed_story_arc_count == 0: raise ValidationError( - "Select at least one matched series or duplicate importable file before importing" + "Select at least one matched series, duplicate importable file, or story arc " + "before importing" ) + try: + capacity_snapshot = await validate_managed_copy_preflight( + session, + job, + stage="confirmation", + ) + except ManagedCopyPreflightError as exc: + await reopen_review_after_managed_copy_preflight_failure(session, job, exc) + raise + job.status = ImportJobStatus.IMPORTING job.control_request = ImportControlRequest.NONE - job.progress_snapshot = initialize_progress_snapshot( + progress_snapshot = initialize_progress_snapshot( job, mode="import", phase="queued", progress=0, - message="Preparing the selected series for import...", + message="Preparing the selected import items...", status=ImportJobStatus.IMPORTING, ) + if capacity_snapshot is not None: + progress_snapshot["managed_copy_capacity"] = capacity_snapshot.as_dict() + job.progress_snapshot = progress_snapshot await session.flush() await log_event( @@ -125,10 +174,12 @@ async def confirm_import_job( "import_confirmed", message=( f"User confirmed {len(items)} series and " - f"{duplicate_selected_count} duplicate-series files for import" + f"{duplicate_selected_count} duplicate-series files and " + f"{confirmed_story_arc_count} story arcs for import" ), confirmed_count=len(items), duplicate_file_count=duplicate_selected_count, + story_arc_count=confirmed_story_arc_count, ) return job @@ -160,11 +211,65 @@ async def _load_confirmed_series( item.selected_for_import = True await session.flush() + persisted_item_ids = [item.id for item in persisted_items] + series_ids_with_files = ( + set( + ( + await session.scalars( + sa_select(ImportedFile.import_series_id) + .where(ImportedFile.import_series_id.in_(persisted_item_ids)) + .distinct() + ) + ).all() + ) + if persisted_item_ids + else set() + ) + importable_series_ids = ( + set( + ( + await session.scalars( + sa_select(ImportedFile.import_series_id) + .where( + ImportedFile.import_series_id.in_(persisted_item_ids), + ImportedFile.status.in_( + [ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED] + ), + ) + .distinct() + ) + ).all() + ) + if persisted_item_ids + else set() + ) + conflict_series_ids = ( + set( + ( + await session.scalars( + sa_select(ImportedFile.import_series_id) + .where( + ImportedFile.import_series_id.in_(persisted_item_ids), + ImportedFile.status == ImportedFileStatus.CONFLICT, + ) + .distinct() + ) + ).all() + ) + if persisted_item_ids + else set() + ) + valid_items: list[ImportedSeries] = [] invalid_items: list[ImportedSeries] = [] has_unresolved_conflicts = False for item in persisted_items: - if item.status == ImportSeriesStatus.MATCHED and (item.files_conflict or 0) == 0: + no_persisted_files = item.id not in series_ids_with_files + if item.status == ImportSeriesStatus.MATCHED and ( + item.id in importable_series_ids + or no_persisted_files + or item.id not in conflict_series_ids + ): valid_items.append(item) continue if (item.files_conflict or 0) > 0: @@ -261,6 +366,11 @@ async def _confirm_matched_files( ) ) for matched_file in matched_result.scalars().all(): + if not ensure_target_issue_summary_for_import_file(matched_file): + matched_file.status = ImportedFileStatus.NO_MATCH + matched_file.include_in_import = False + affected_series_ids.add(matched_file.import_series_id) + continue matched_file.status = ImportedFileStatus.CONFIRMED affected_series_ids.add(matched_file.import_series_id) await session.flush() diff --git a/src/pullbox/services/import_content_inspection.py b/src/pullbox/services/import_content_inspection.py new file mode 100644 index 00000000..96ed5543 --- /dev/null +++ b/src/pullbox/services/import_content_inspection.py @@ -0,0 +1,65 @@ +"""Import-only comic content review, without extracting page payloads.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import structlog + +from pullbox.core.archive import ArchiveError, ArchiveReader +from pullbox.core.file_safety import FileSafetyError +from pullbox.core.page_sources.base import canonical_page_names +from pullbox.services.import_safety_diagnostics import build_import_safety_diagnostics + +if TYPE_CHECKING: + from pathlib import Path + + from pullbox.core.file_safety import FileSafetyInspection + + +logger = structlog.get_logger(__name__) + + +def inspect_import_content(path: Path, inspection: FileSafetyInspection) -> dict[str, object]: + """Reuse ZIP inventory; other comic archives need only their member headers. + + PDF and EPUB are not image archives and must not use this page heuristic. + Call from a worker thread, just like archive safety inspection. + """ + if path.suffix.lower() not in {".cbz", ".cbr", ".cb7", ".cbt"}: + return {} + report = next((report for report in inspection.archives if report.archive_path == path), None) + if report is not None and report.page_count is not None: + page_count = report.page_count + else: + try: + members = ArchiveReader(path).list_members() + except ArchiveError as exc: + logger.warning( + "import_archive_content_inspection_failed", + file_name=path.name, + error=str(exc), + error_type=type(exc).__name__, + ) + raise FileSafetyError("Archive inspection failed", details=[str(path)]) from exc + page_count = len( + canonical_page_names( + [ + member.name + for member in members + if member.is_regular_file and not member.is_link and member.size > 0 + ] + ) + ) + diagnostics: dict[str, object] = { + "content_inspection": {"version": 1, "page_count": page_count}, + } + if page_count < 2: + code = "archive_no_pages" if page_count == 0 else "single_page_comic" + diagnostics["file_safety"] = build_import_safety_diagnostics( + code, + code=code, + kind=code, + source="import_content", + ) + return diagnostics diff --git a/src/pullbox/services/import_counters.py b/src/pullbox/services/import_counters.py index f63b0057..1dc0f3f8 100644 --- a/src/pullbox/services/import_counters.py +++ b/src/pullbox/services/import_counters.py @@ -163,7 +163,7 @@ async def recompute_file_counters( ImportedFileStatus.SAFETY_BLOCKED, 0, ) + job_status_counts.get(ImportedFileStatus.SAFETY_APPROVED, 0) - job.total_files_found = ( + classified_file_count = ( job.total_files_matched + job.total_files_duplicate + job.total_files_already_owned @@ -173,5 +173,10 @@ async def recompute_file_counters( + job.total_files_failed + total_files_safety_blocked ) + job.total_files_found = max( + int(job.total_files_found or 0), + int(job.scan_total_files or 0), + classified_file_count, + ) await session.flush() diff --git a/src/pullbox/services/import_cv_search.py b/src/pullbox/services/import_cv_search.py index 0c049278..26f0714f 100644 --- a/src/pullbox/services/import_cv_search.py +++ b/src/pullbox/services/import_cv_search.py @@ -29,6 +29,11 @@ async def search_with_retry( year: int | None, ) -> list[SeriesSearchResult]: """Search ComicVine with retry on transient provider failures.""" + if getattr(provider, "is_local_catalog", False) is True: + results, _ = await provider.search_series_globally( + query, max_results=_IMPORT_GLOBAL_SEARCH_LIMIT + ) + return list(results) last_provider_error: str | None = None saw_successful_response = False for attempt in range(_MAX_RETRIES): diff --git a/src/pullbox/services/import_deferred_recovery.py b/src/pullbox/services/import_deferred_recovery.py new file mode 100644 index 00000000..3c3fdb77 --- /dev/null +++ b/src/pullbox/services/import_deferred_recovery.py @@ -0,0 +1,675 @@ +"""Reconcile completed import decisions using physical files and exact identities.""" + +from __future__ import annotations + +import re +from collections import defaultdict +from dataclasses import dataclass +from datetime import UTC, datetime +from itertools import batched +from typing import TYPE_CHECKING, Any + +from sqlalchemy import select + +from pullbox.core.name_matcher import NameMatcher +from pullbox.core.release_parser import normalize_issue_number, parse_release_title +from pullbox.core.source_metadata import _extract_issue_id_from_notes, _extract_issue_id_from_web +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobLog, + ImportSeriesStatus, +) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile +from pullbox.models.series import IssueCatalogState, Series +from pullbox.services.import_source_metadata import source_metadata_for_import_file +from pullbox.services.import_terminal_recovery import allows_terminal_import_recovery + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql import Select + + +def positive_id(value: object) -> int | None: + try: + result = int(str(value)) + except (TypeError, ValueError): + return None + return result if result > 0 else None + + +def provider_ids(file: ImportedFile) -> set[int]: + """Retain disagreements instead of silently preferring one saved identity.""" + summary = file.diagnostics.get("target_issue_summary") + summary = summary if isinstance(summary, dict) else {} + source = file.diagnostics.get("source_metadata") + source = source if isinstance(source, dict) else {} + comicinfo = source.get("comicinfo") + comicinfo = comicinfo if isinstance(comicinfo, dict) else {} + embedded_ids = { + value + for value in ( + _extract_issue_id_from_web(comicinfo.get("web")), + _extract_issue_id_from_notes(comicinfo.get("notes")), + ) + if value is not None + } + if len(embedded_ids) > 1: + return embedded_ids + signals = file.diagnostics.get("metadata_signals") + signals = signals if isinstance(signals, dict) else {} + source_id = file.comicvine_issue_id + if len(embedded_ids) == 1 and signals.get("comicvine_issue_id") == "mylar3": + source_id = next(iter(embedded_ids)) + values = (source_id, file.matched_issue_cv_id, summary.get("provider_id"), *embedded_ids) + return {positive_id(value) or -1 for value in values if value is not None} + + +def _unresolved_identity_conflicts(file: ImportedFile) -> bool: + source = file.diagnostics.get("source_metadata") + source = source if isinstance(source, dict) else {} + conflicts = [ + *(source.get("identity_conflicts") or []), + *(file.diagnostics.get("identity_conflicts") or []), + ] + signals = file.diagnostics.get("metadata_signals") + signals = signals if isinstance(signals, dict) else {} + ids = provider_ids(file) + for conflict in conflicts: + if not isinstance(conflict, dict): + return True + if conflict.get("field") == "comicvine_series_id": + continue + if ( + conflict.get("field") == "comicvine_issue_id" + and signals.get("comicvine_issue_id") == "mylar3" + and len(ids) == 1 + and positive_id(conflict.get("first")) == file.comicvine_issue_id + and positive_id(conflict.get("conflicting")) in ids + ): + continue + return True + return False + + +def protected_file(file: ImportedFile, item: ImportedSeries) -> bool: + diagnostics = dict(file.diagnostics or {}) + return bool( + file.status is not ImportedFileStatus.NO_MATCH + or file.library_file_id is not None + or file.include_in_import + or item.user_selected_cv_id is not None + or (file.match_method or "").startswith(("manual", "orphan_recovery")) + or any( + diagnostics.get(key) + for key in ( + "safety_block", + "safety_exception", + "source_revalidation", + "safety_review", + ) + ) + or _unresolved_identity_conflicts(file) + or ( + file.conflict_group_id is not None + and not ( + file.is_preferred + and file.match_confidence == "high" + and file.matched_issue_cv_id is not None + ) + ) + or len(provider_ids(file)) > 1 + or -1 in provider_ids(file) + ) + + +def same_source(left: ImportedFile, right: ImportedFile | LibraryFile) -> bool: + if left.file_path != right.file_path or left.file_size != right.file_size: + return False + first, second = dict(left.source_signature or {}), dict(right.source_signature or {}) + # Persisted path alone is not proof that a file survived unchanged. + # Device IDs change when container mounts are recreated. Size and mtime are portable. + keys = ("size", "size_bytes", "mtime_ns", "content_digest", "content_digest_algorithm") + common = [key for key in keys if key in first and key in second] + return bool("mtime_ns" in common and all(first[key] == second[key] for key in common)) + + +def _titles(series: Series) -> set[str]: + return {NameMatcher.normalize(name) for name in (series.title, *(series.alternate_names or []))} + + +def _source_agrees( + file: ImportedFile, + item: ImportedSeries, + series: Series, + issue: Issue, + *, + require_issue_number: bool = True, +) -> bool: + metadata = source_metadata_for_import_file(item, file) + if metadata.series_name and NameMatcher.normalize(metadata.series_name) not in _titles(series): + return False + issue_numbers: list[float] = [] + if metadata.issue_number is not None: + issue_numbers.append(metadata.issue_number) + hint = metadata.diagnostics.get("archive_entry_issue_hint") + if isinstance(hint, dict) and hint.get("confidence") == "strong": + hint_number = normalize_issue_number(hint.get("issue_number")) + if hint_number is None: + return False + issue_numbers.append(hint_number) + if (require_issue_number and not issue_numbers) or any( + number != issue.issue_number for number in issue_numbers + ): + return False + # Type evidence from a release or ComicInfo must not turn an Annual into #1. + raw_type = file.diagnostics.get("source_issue_type") + return not raw_type or raw_type == issue.issue_type.value + + +def strict_filename_target( + file: ImportedFile, item: ImportedSeries, series: Series, issues: list[Issue] +) -> Issue | None: + """Reparse only after identifying the series, requiring independent agreement.""" + if series.issue_catalog_state is not IssueCatalogState.COMPLETE: + return None + source = file.diagnostics.get("source_metadata") + if isinstance(source, dict) and source.get("identity_conflicts"): + return None + name = re.sub(r"\(converted\)", "", file.file_name, flags=re.IGNORECASE) + name = re.sub(r"(?<=\D)\.(\d+)\.(?=\s|\()", r" \1 ", name) + parsed = parse_release_title( + name, expected_series=(series.title, *(series.alternate_names or [])) + ) + if ( + parsed is None + or parsed.is_pack + or parsed.volume is not None + or parsed.issue_number is None + or parsed.year is None + or NameMatcher.normalize(parsed.series_name or "") not in _titles(series) + or (file.parsed_series and NameMatcher.normalize(file.parsed_series) not in _titles(series)) + or ( + file.parsed_issue_number is not None and file.parsed_issue_number != parsed.issue_number + ) + ): + return None + candidates = [issue for issue in issues if issue.issue_number == parsed.issue_number] + if len(candidates) != 1: + return None + issue = candidates[0] + if ( + issue.release_date is None + or abs(issue.release_date.year - parsed.year) > 1 + or parsed.issue_type != issue.issue_type + or (file.parsed_year is not None and abs(file.parsed_year - issue.release_date.year) > 1) + or not _source_agrees(file, item, series, issue, require_issue_number=False) + ): + return None + return issue + + +@dataclass(frozen=True) +class DeferredRecoveryPlan: + file_id: int + action: str + canonical_file_id: int | None = None + issue_id: int | None = None + reason: str = "" + + +async def load_deferred_rows( + session: AsyncSession, job_id: int +) -> list[tuple[ImportedFile, ImportedSeries]]: + rows: list[tuple[ImportedFile, ImportedSeries]] = [] + cursor = 0 + while True: + batch = ( + await session.execute( + select(ImportedFile, ImportedSeries) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.NO_MATCH, + ImportedFile.id > cursor, + ) + .order_by(ImportedFile.id) + .limit(500) + ) + ).all() + if not batch: + return rows + rows.extend((file, item) for file, item in batch) + cursor = batch[-1][0].id + + +async def plan_deferred_recovery( + session: AsyncSession, job_id: int, *, running: bool = False +) -> tuple[DeferredRecoveryPlan, ...]: + """Return a read-only plan for deterministic local recovery.""" + job = await session.get(ImportJob, job_id) + if job is None or (not running and not allows_terminal_import_recovery(job)): + return () + rows = await load_deferred_rows(session, job_id) + groups: dict[str, list[tuple[ImportedFile, ImportedSeries]]] = defaultdict(list) + for file, item in rows: + groups[file.file_path].append((file, item)) + cv_ids = set().union(*(provider_ids(file) for file, _ in rows)) if rows else set() + local_series_ids = {item.series_id for _, item in rows if item.series_id is not None} + targets: dict[int, tuple[Issue, Series]] = {} + issues_by_series: dict[int, list[Issue]] = defaultdict(list) + series_by_id: dict[int, Series] = {} + for ids in batched(sorted(local_series_ids), 300): + for issue, series in ( + await session.execute( + select(Issue, Series) + .join(Series, Series.id == Issue.series_id) + .where(Series.id.in_(ids)) + ) + ).all(): + issues_by_series[series.id].append(issue) + series_by_id[series.id] = series + if issue.comicvine_id is not None: + targets[issue.comicvine_id] = issue, series + for ids in batched(sorted(cv_ids - targets.keys()), 300): + for issue, series in ( + await session.execute( + select(Issue, Series) + .join(Series, Series.id == Issue.series_id) + .where(Issue.comicvine_id.in_(ids)) + ) + ).all(): + if issue.comicvine_id is not None: + targets[issue.comicvine_id] = issue, series + + registered: dict[str, LibraryFile] = {} + owned: dict[int, LibraryFile] = {} + for ids in batched( + sorted( + {issue.id for issue, _ in targets.values()} + | {issue.id for issues in issues_by_series.values() for issue in issues} + ), + 300, + ): + for library in await session.scalars( + select(LibraryFile).where(LibraryFile.issue_id.in_(ids)) + ): + if library.issue_id is not None: + owned[library.issue_id] = library + registered[library.file_path] = library + for paths in batched(sorted(groups), 300): + for library in await session.scalars( + select(LibraryFile).where(LibraryFile.file_path.in_(paths)) + ): + registered[library.file_path] = library + + plans: list[DeferredRecoveryPlan] = [] + for path, cohort in groups.items(): + if any(protected_file(file, item) for file, item in cohort): + continue + identities = set().union(*(provider_ids(file) for file, _ in cohort)) + if len(identities) > 1: + continue + # Prefer the row whose parent already owns the target catalog. + cohort.sort(key=lambda pair: (pair[1].series_id is None, pair[0].id)) + file, item = cohort[0] + if any(not same_source(file, other) for other, _ in cohort[1:]): + continue + provider_id = next(iter(identities), None) + target = targets.get(provider_id) if provider_id is not None else None + if target is not None: + issue, series = target + if any(not _source_agrees(other, parent, series, issue) for other, parent in cohort): + continue + if any(other.matched_issue_id not in (None, issue.id) for other, _ in cohort): + continue + elif provider_id is None and item.series_id in series_by_id: + series = series_by_id[item.series_id] + issue = strict_filename_target(file, item, series, issues_by_series[series.id]) + target = (issue, series) if issue is not None else None + + registered_file = registered.get(path) + if registered_file is not None: + if ( + target is None + or registered_file.issue_id != target[0].id + or not same_source(file, registered_file) + ): + continue + plans.append(DeferredRecoveryPlan(file.id, "already_registered", issue_id=target[0].id)) + elif target is not None: + issue = target[0] + plans.append( + DeferredRecoveryPlan( + file.id, + "owned_variant" if issue.id in owned else "exact_target", + issue_id=issue.id, + reason="provider_id" if provider_id else "strict_filename", + ) + ) + for other, _ in cohort[1:]: + plans.append( + DeferredRecoveryPlan(other.id, "duplicate_reference", canonical_file_id=file.id) + ) + + # Two distinct physical files targeting one unowned issue still need a choice. + pending: dict[int, list[int]] = defaultdict(list) + for index, plan in enumerate(plans): + if plan.action == "exact_target" and plan.issue_id is not None: + pending[plan.issue_id].append(index) + ambiguous = {index for indices in pending.values() if len(indices) > 1 for index in indices} + return tuple( + sorted( + (plan for index, plan in enumerate(plans) if index not in ambiguous), + key=lambda plan: plan.file_id, + ) + ) + + +def candidate_series_ids(file: ImportedFile, item: ImportedSeries) -> set[int]: + """Use trusted saved provider identities as candidates, never as proof of ownership.""" + diagnostics = dict(file.diagnostics or {}) + signals = diagnostics.get("metadata_signals") + signals = signals if isinstance(signals, dict) else {} + ids: set[int] = set() + if signals.get("comicvine_series_id") in {"mylar3", "comicinfo", "sidecar", "folder_sidecar"}: + cv_id = positive_id(diagnostics.get("comicvine_series_id")) + if cv_id is not None: + ids.add(cv_id) + source = diagnostics.get("source_metadata") + if isinstance(source, dict): + for conflict in source.get("identity_conflicts") or []: + if isinstance(conflict, dict) and conflict.get("field") == "comicvine_series_id": + ids.update( + value + for raw in (conflict.get("first"), conflict.get("conflicting")) + if (value := positive_id(raw)) is not None + ) + candidate = dict(item.diagnostics or {}).get("selected_candidate") + if isinstance(candidate, dict) and candidate.get("match_method") in { + "mylar3_cv_id", + "comicinfo_cv_id", + "folder_cv_id", + }: + cv_id = positive_id(candidate.get("cv_id")) + if cv_id is not None: + ids.add(cv_id) + retained = diagnostics.get("deferred_recovery_candidates") + if isinstance(retained, list): + ids.update(value for raw in retained if (value := positive_id(raw)) is not None) + return ids + + +def issue_summary(issue: Issue) -> dict[str, Any]: + return { + "provider_id": str(issue.comicvine_id) if issue.comicvine_id else None, + "issue_number": issue.issue_number, + "issue_number_text": issue.issue_number_text or str(issue.issue_number), + "title": issue.title, + "release_date": issue.release_date.isoformat() if issue.release_date else None, + "cover_url": issue.cover_url, + "issue_type": issue.issue_type.value, + } + + +def apply_proven_identity( + file: ImportedFile, + *, + issue_cv_id: int | None, + series_cv_id: int | None, + summary: dict[str, Any], +) -> None: + """Retain superseded evidence while replacing only an already-proven identity.""" + diagnostics = dict(file.diagnostics or {}) + previous = dict(diagnostics) + source = dict(diagnostics.get("source_metadata") or {}) + source.pop("identity_conflicts", None) + diagnostics.pop("identity_conflicts", None) + diagnostics.update( + source_metadata=source, + comicvine_series_id=series_cv_id, + kind="deferred_exact_identity", + target_issue_summary=summary, + deferred_recovery_previous_diagnostics=previous, + ) + file.diagnostics = diagnostics + file.comicvine_issue_id = issue_cv_id + + +async def apply_deferred_recovery( + session: AsyncSession, job: ImportJob, *, running: bool = False +) -> dict[str, int]: + """Apply locally proven decisions; materialization remains ordinary Step 4 work.""" + from pullbox.services.import_story_arc_resolution import ( + refresh_story_arc_entries_for_import_files, + ) + + if not running and not allows_terminal_import_recovery(job): + return {} + plans = await plan_deferred_recovery(session, job.id, running=running) + counts: dict[str, int] = defaultdict(int) + affected: set[int] = set() + targets: dict[int, ImportedSeries] = {} + for batch in batched(plans, 300): + files = { + file.id: file + for file in await session.scalars( + select(ImportedFile).where(ImportedFile.id.in_([plan.file_id for plan in batch])) + ) + } + for plan in batch: + file = files[plan.file_id] + parent = await session.get(ImportedSeries, file.import_series_id) + assert parent is not None + file.diagnostics = { + **file.diagnostics, + "deferred_recovery_candidates": sorted(candidate_series_ids(file, parent)), + } + affected.add(parent.id) + evidence = { + "action": plan.action, + "reason": plan.reason, + "source_import_series_id": parent.id, + "previous_error": file.error_message, + "source_preserved": True, + "resolved_at": datetime.now(UTC).isoformat(), + } + if plan.action == "duplicate_reference": + canonical = await session.get(ImportedFile, plan.canonical_file_id) + assert canonical is not None + canonical_parent = await session.get(ImportedSeries, canonical.import_series_id) + assert canonical_parent is not None + candidates = candidate_series_ids(file, parent) | candidate_series_ids( + canonical, canonical_parent + ) + canonical.diagnostics = { + **canonical.diagnostics, + "deferred_recovery_candidates": sorted(candidates), + } + file.duplicate_of_file_id = canonical.id + file.status = ImportedFileStatus.SKIPPED + evidence["canonical_file_id"] = canonical.id + else: + issue = await session.get(Issue, plan.issue_id) + assert issue is not None + series = await session.get(Series, issue.series_id) + assert series is not None + file.matched_issue_id = issue.id + file.matched_issue_cv_id = issue.comicvine_id + if plan.action == "already_registered": + file.status = ImportedFileStatus.ALREADY_OWNED + elif plan.action == "owned_variant": + file.status = ImportedFileStatus.CONFLICT + file.error_message = "This issue is already owned. Review this alternate file." + else: + target = targets.get(series.id) + if target is None: + # Isolate selected files from unrelated ready rows in the old parent. + target = ImportedSeries( + import_job_id=job.id, + raw_series_name=series.title, + raw_year=series.year_start, + cv_id=series.comicvine_id, + cv_title=series.title, + cv_year=series.year_start, + cv_match_method="deferred_exact_identity", + cv_match_score=1.0, + series_id=series.id, + has_files=True, + status=ImportSeriesStatus.CONFIRMED, + selected_for_import=True, + diagnostics={"kind": "deferred_recovery", "source_preserved": True}, + ) + session.add(target) + await session.flush() + targets[series.id] = target + file.import_series_id = target.id + affected.add(target.id) + file.status = ImportedFileStatus.CONFIRMED + file.include_in_import = True + file.match_method = "completed_import_exact_target" + file.match_confidence = "high" + file.parsed_issue_number = issue.issue_number + file.issue_number_raw = issue.issue_number_text + file.conflict_group_id = None + apply_proven_identity( + file, + issue_cv_id=issue.comicvine_id, + series_cv_id=series.comicvine_id, + summary=issue_summary(issue), + ) + evidence["target_issue_id"] = issue.id + if plan.action != "exact_target": + file.include_in_import = False + if plan.action != "owned_variant": + file.error_message = None + file.diagnostics = {**file.diagnostics, "deferred_recovery": evidence} + counts[plan.action] += 1 + await refresh_story_arc_entries_for_import_files( + session, import_job_id=job.id, import_file_ids=list(files) + ) + await session.flush() + + stale_series_ids: tuple[int, ...] | None = None + if running: + state = dict(dict(job.progress_snapshot or {}).get("deferred_recovery") or {}) + raw_ids = state.get("stale_series_ids") + stale_series_ids = ( + tuple(int(value) for value in raw_ids) + if isinstance(raw_ids, list) and all(isinstance(value, int) for value in raw_ids) + else () + ) + counts["stale_series"] = await archive_empty_stale_series( + session, + job.id, + series_ids=stale_series_ids, + ) + await refresh_recovered_groups(session, job, affected) + snapshot = dict(job.progress_snapshot or {}) + recovery = dict(snapshot.get("deferred_recovery") or {}) + recovery["series_ids"] = sorted( + set(recovery.get("series_ids", [])) | {item.id for item in targets.values()} + ) + job.progress_snapshot = {**snapshot, "deferred_recovery": recovery} + session.add( + ImportJobLog( + import_job_id=job.id, + level="INFO", + event="import_deferred_recovery_local", + message="Reconciled deferred file records.", + data={"counts": dict(counts), "source_preserved": True}, + ) + ) + await session.flush() + return dict(counts) + + +async def refresh_recovered_groups( + session: AsyncSession, job: ImportJob, affected: set[int] +) -> None: + """Remove finished matching groups while keeping their audit records.""" + from pullbox.services.import_counters import recompute_file_counters, recompute_series_counters + + if affected: + await recompute_file_counters(session, job, series_ids=sorted(affected)) + # Groups containing only completed decisions no longer need a matching task. + actionable = { + ImportedFileStatus.NO_MATCH, + ImportedFileStatus.CONFLICT, + ImportedFileStatus.FAILED, + ImportedFileStatus.SAFETY_BLOCKED, + ImportedFileStatus.SAFETY_APPROVED, + ImportedFileStatus.MATCHED, + ImportedFileStatus.CONFIRMED, + ImportedFileStatus.PENDING, + } + for ids in batched(sorted(affected), 300): + open_ids = set( + await session.scalars( + select(ImportedFile.import_series_id) + .where( + ImportedFile.import_series_id.in_(ids), ImportedFile.status.in_(actionable) + ) + .distinct() + ) + ) + for item in await session.scalars( + select(ImportedSeries).where(ImportedSeries.id.in_(ids)) + ): + if item.id not in open_ids and item.status in { + ImportSeriesStatus.NO_MATCH, + ImportSeriesStatus.RECOVERY_PENDING, + }: + item.status = ImportSeriesStatus.SKIPPED + item.selected_for_import = False + item.diagnostics = { + **item.diagnostics, + "follow_up_resolved": "all_files_handled", + } + await recompute_series_counters(session, job) + + +def empty_stale_series_query(job_id: int) -> Select[tuple[ImportedSeries]]: + """Return empty missing-location rows eligible for follow-up archival.""" + return select(ImportedSeries).where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status.in_( + (ImportSeriesStatus.NO_MATCH, ImportSeriesStatus.RECOVERY_PENDING) + ), + ImportedSeries.user_selected_cv_id.is_(None), + ImportedSeries.diagnostics["reason"].as_string().in_(("path_missing", "source_missing")), + ~select(ImportedFile.id).where(ImportedFile.import_series_id == ImportedSeries.id).exists(), + ) + + +async def load_empty_stale_series( + session: AsyncSession, + job_id: int, +) -> list[ImportedSeries]: + """Load empty stale series deterministically for signed cleanup previews.""" + return list(await session.scalars(empty_stale_series_query(job_id).order_by(ImportedSeries.id))) + + +async def archive_empty_stale_series( + session: AsyncSession, + job_id: int, + *, + series_ids: tuple[int, ...] | None = None, +) -> int: + """Retain empty missing Mylar locations in history rather than active matching.""" + query = empty_stale_series_query(job_id) + if series_ids is not None: + query = query.where(ImportedSeries.id.in_(series_ids)) + items = list(await session.scalars(query.order_by(ImportedSeries.id))) + for item in items: + item.status = ImportSeriesStatus.SKIPPED + item.selected_for_import = False + item.diagnostics = { + **item.diagnostics, + "follow_up_resolved": "stale_empty_reference", + "archived_at": datetime.now(UTC).isoformat(), + } + return len(items) diff --git a/src/pullbox/services/import_deferred_recovery_execution.py b/src/pullbox/services/import_deferred_recovery_execution.py new file mode 100644 index 00000000..f5137dbc --- /dev/null +++ b/src/pullbox/services/import_deferred_recovery_execution.py @@ -0,0 +1,421 @@ +"""Durable background preparation for completed-import file recovery.""" + +from __future__ import annotations + +from collections import Counter, defaultdict +from dataclasses import asdict +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +from sqlalchemy import select + +from pullbox.core.exceptions import JobPausedError, NotFoundError, ProviderError, ValidationError +from pullbox.core.name_matcher import NameMatcher +from pullbox.models.import_job import ( + ImportControlRequest, + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobLog, + ImportJobStatus, + ImportSeriesStatus, +) +from pullbox.models.issue import Issue +from pullbox.schemas.import_job import ImportProgressEvent +from pullbox.services.import_counters import recompute_file_counters, recompute_series_counters +from pullbox.services.import_deferred_recovery import ( + apply_deferred_recovery, + apply_proven_identity, + candidate_series_ids, + load_deferred_rows, + positive_id, + protected_file, + provider_ids, + refresh_recovered_groups, +) +from pullbox.services.import_source_metadata import source_metadata_for_import_file +from pullbox.services.import_workflow_state import ( + emit_live_progress, + emit_progress, + raise_if_job_cancelled, +) + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.providers.base import IssueSummary + from pullbox.services.metadata_service import MetadataService + + +def recovery_state(job: ImportJob) -> dict[str, Any]: + return dict(dict(job.progress_snapshot or {}).get("deferred_recovery") or {}) + + +def save_recovery_state(job: ImportJob, state: dict[str, Any]) -> None: + job.progress_snapshot = {**dict(job.progress_snapshot or {}), "deferred_recovery": state} + + +def _catalog_summary_payload(summary: IssueSummary) -> dict[str, Any]: + """Return a durable JSON-safe catalog checkpoint payload.""" + payload = asdict(summary) + source_cutoff_at = payload.get("source_cutoff_at") + if isinstance(source_cutoff_at, datetime): + payload["source_cutoff_at"] = source_cutoff_at.isoformat() + return payload + + +async def cancel_deferred_preparation(session: AsyncSession, job: ImportJob) -> bool: + """Stop this recovery pass, retaining the original and any completed imports.""" + state = recovery_state(job) + if state.get("state") not in {"queued", "catalogs", "prepared"}: + return False + ids = state.get("series_ids", []) + for item in await session.scalars( + select(ImportedSeries).where( + ImportedSeries.import_job_id == job.id, ImportedSeries.id.in_(ids) + ) + ): + pending = list( + await session.scalars( + select(ImportedFile).where( + ImportedFile.import_series_id == item.id, + ImportedFile.status.in_( + (ImportedFileStatus.CONFIRMED, ImportedFileStatus.MATCHED) + ), + ) + ) + ) + for file in pending: + file.status = ImportedFileStatus.NO_MATCH + file.include_in_import = False + if pending: + item.status = ImportSeriesStatus.RECOVERY_PENDING + item.selected_for_import = False + await recompute_file_counters(session, job, series_ids=ids) + await recompute_series_counters(session, job) + state["state"] = "cancelled" + save_recovery_state(job, state) + job.status = ImportJobStatus.COMPLETED + job.control_request = ImportControlRequest.NONE + job.error_message = None + job.progress_snapshot = { + **job.progress_snapshot, + "status": "completed", + "phase": "done", + "progress": 100, + "message": "Recovery stopped. Completed imports and source files were preserved.", + } + session.add( + ImportJobLog( + import_job_id=job.id, + level="INFO", + event="import_deferred_recovery_cancelled", + message="Recovery stopped without rolling back the original import.", + data={"source_preserved": True}, + ) + ) + await session.commit() + return True + + +async def _catalog_candidates(session: AsyncSession, job: ImportJob) -> dict[str, list[int]]: + candidates: dict[str, set[int]] = defaultdict(set) + rows = await load_deferred_rows(session, job.id) + for file, item in rows: + if protected_file(file, item) or len(provider_ids(file)) != 1: + continue + issue_cv_id = next(iter(provider_ids(file))) + for series_cv_id in candidate_series_ids(file, item): + candidates[str(series_cv_id)].add(issue_cv_id) + # Never fetch catalogs for an issue whose canonical local ownership is already known. + from itertools import batched + + known: set[int] = set() + for ids in batched(sorted(set().union(*candidates.values()) if candidates else set()), 300): + known.update( + value + for value in await session.scalars( + select(Issue.comicvine_id).where(Issue.comicvine_id.in_(ids)) + ) + if value is not None + ) + return {key: sorted(values - known) for key, values in candidates.items() if values - known} + + +def _catalog_target_agrees( + file: ImportedFile, item: ImportedSeries, target: dict[str, Any] +) -> bool: + metadata = source_metadata_for_import_file(item, file) + summary = target["summary"] + source_title = metadata.series_name or "" + hint = metadata.diagnostics.get("archive_entry_issue_hint") + if ( + isinstance(hint, dict) + and hint.get("confidence") == "strong" + and hint.get("issue_number") != summary["issue_number"] + ): + return False + return bool( + source_title + and NameMatcher.normalize(source_title) == NameMatcher.normalize(target["title"]) + and metadata.issue_number is not None + and metadata.issue_number == summary["issue_number"] + and ( + not file.diagnostics.get("source_issue_type") + or file.diagnostics["source_issue_type"] == summary["issue_type"] + ) + and file.matched_issue_id is None + ) + + +async def _prepare_catalog_targets(session: AsyncSession, job: ImportJob) -> int: + """Only stage unique issue membership with agreeing file evidence.""" + state = recovery_state(job) + matches = state.get("matches", {}) + rows = await load_deferred_rows(session, job.id) + eligible: list[tuple[ImportedFile, ImportedSeries, dict[str, Any]]] = [] + for file, item in rows: + ids = provider_ids(file) + if protected_file(file, item) or len(ids) != 1: + continue + options = matches.get(str(next(iter(ids))), []) + # Membership in conflicting candidate catalogs is ambiguous even when titles agree. + if len(options) != 1 or not _catalog_target_agrees(file, item, options[0]): + continue + eligible.append((file, item, options[0])) + counts = Counter(str(target["summary"]["provider_id"]) for _, _, target in eligible) + targets: dict[int, ImportedSeries] = {} + affected: set[int] = set() + ready = 0 + for file, original, target in eligible: + issue_cv_id = positive_id(target["summary"]["provider_id"]) + if issue_cv_id is None or counts[str(issue_cv_id)] != 1: + continue + if await session.scalar(select(Issue.id).where(Issue.comicvine_id == issue_cv_id)): + continue + target_cv_id = int(target["cv_id"]) + target_item = targets.get(target_cv_id) + if target_item is None: + target_item = ImportedSeries( + import_job_id=job.id, + raw_series_name=target["title"], + raw_year=target["year"], + cv_id=target_cv_id, + cv_title=target["title"], + cv_year=target["year"], + cv_match_score=1.0, + cv_match_method="deferred_catalog_identity", + status=ImportSeriesStatus.CONFIRMED, + selected_for_import=True, + has_files=True, + diagnostics={"kind": "deferred_recovery", "source_preserved": True}, + ) + session.add(target_item) + await session.flush() + targets[target_cv_id] = target_item + affected.update((original.id, target_item.id)) + file.import_series_id = target_item.id + file.status = ImportedFileStatus.CONFIRMED + file.matched_issue_cv_id = issue_cv_id + file.include_in_import = True + file.match_confidence = "high" + file.match_method = "completed_import_exact_target" + file.error_message = None + file.conflict_group_id = None + apply_proven_identity( + file, issue_cv_id=issue_cv_id, series_cv_id=target_cv_id, summary=target["summary"] + ) + file.diagnostics = { + **file.diagnostics, + "target_issue_summary": target["summary"], + "deferred_recovery": { + "action": "catalog_identity", + "source_import_series_id": original.id, + "target_series_cv_id": target_cv_id, + "source_preserved": True, + "resolved_at": datetime.now(UTC).isoformat(), + }, + } + ready += 1 + if affected: + from pullbox.services.import_story_arc_resolution import ( + refresh_story_arc_entries_for_import_files, + ) + + await refresh_recovered_groups(session, job, affected) + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job.id, + import_file_ids=[ + file.id for file, _, _ in eligible if file.status is ImportedFileStatus.CONFIRMED + ], + ) + state["series_ids"] = sorted( + set(state.get("series_ids", [])) | {item.id for item in targets.values()} + ) + save_recovery_state(job, state) + await session.flush() + return ready + + +async def prepare_deferred_recovery( + session: AsyncSession, + job_id: int, + *, + metadata_service: MetadataService, + progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None = None, +) -> bool: + """Resume a saved recovery request, preparing exact files for normal execution.""" + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + state = recovery_state(job) + if not state or state.get("state") in {"prepared", "completed", "cancelled"}: + return False + if job.status is not ImportJobStatus.IMPORTING: + raise ValidationError("Deferred recovery must run inside the import worker.") + + revision_state = {"value": int(job.progress_revision or 0)} + + def progress_event(current: int, total: int, message: str) -> ImportProgressEvent: + return ImportProgressEvent( + job_id=job_id, + status=ImportJobStatus.IMPORTING, + mode="import", + phase="deferred_recovery", + progress=round(15 * current / max(total, 1)), + message=message, + current_file_stage="deferred_recovery", + current_file_progress_current=current, + current_file_progress_total=total, + current_file_progress_pct=round(100 * current / max(total, 1)), + current_file_progress_unit="catalogs", + ) + + async def report( + current: int, + total: int, + message: str, + *, + durable: bool = True, + check_control: bool = True, + ) -> None: + if check_control: + await raise_if_job_cancelled(session, job_id) + event = progress_event(current, total, message) + if durable: + event.progress_revision = revision_state["value"] + 1 + await emit_progress(session, job, event, progress_callback) + revision_state["value"] = event.progress_revision + return + # A read-only control check still opens a transaction. Close it before + # provider I/O, then publish live progress without rewriting the large + # durable recovery checkpoint a second time. + await session.commit() + await emit_live_progress( + job, + event, + progress_callback=progress_callback, + revision_state=revision_state, + started_at=job.import_started_at, + ) + + if state.get("state") == "queued": + await report(0, 1, "Reconciling deferred files with the existing library...") + local_counts = await apply_deferred_recovery(session, job, running=True) + state = recovery_state(job) + state.update( + state="catalogs", + local_counts=local_counts, + candidates=await _catalog_candidates(session, job), + completed=[], + matches={}, + ) + save_recovery_state(job, state) + await session.commit() + + candidates = state["candidates"] + completed = set(state.get("completed", [])) + for cv_id_text, needed_ids in sorted(candidates.items()): + if cv_id_text in completed: + continue + await report( + len(completed), + len(candidates), + f"Checking series catalog {len(completed) + 1} of {len(candidates)}...", + durable=False, + ) + # The live progress report closes its read transaction before provider I/O. + cv_id = int(cv_id_text) + try: + series = await metadata_service.get_series_metadata(cv_id) + summaries = await metadata_service.get_issue_summaries_for_series(cv_id) + except NotFoundError: + series = None + summaries = [] + except ProviderError as exc: + job.error_message = ( + "Metadata is temporarily unavailable. Resume recovery when it is available." + ) + await report(len(completed), len(candidates), job.error_message) + raise JobPausedError(job.error_message) from exc + await raise_if_job_cancelled(session, job_id) + if series is not None and positive_id(series.provider_id) == cv_id: + needed = set(needed_ids) + for summary in summaries: + if positive_id(summary.provider_id) not in needed: + continue + entries = state["matches"].setdefault(str(summary.provider_id), []) + entries.append( + { + "cv_id": cv_id, + "title": series.title, + "year": series.year_start, + "summary": _catalog_summary_payload(summary), + } + ) + completed.add(cv_id_text) + state["completed"] = sorted(completed) + save_recovery_state(job, state) + await report( + len(completed), + len(candidates), + f"Checked series catalog {len(completed)} of {len(candidates)}.", + check_control=False, + ) + + await report(len(completed), max(len(candidates), 1), "Preparing verified files for import...") + catalog_count = await _prepare_catalog_targets(session, job) + state = recovery_state(job) + state.update(state="prepared", catalog_files_prepared=catalog_count) + job.error_message = None + if not state.get("series_ids"): + state["state"] = "completed" + job.status = ImportJobStatus.COMPLETED + job.progress_snapshot = { + **job.progress_snapshot, + "status": "completed", + "phase": "done", + "progress": 100, + "message": "Recovery completed. Remaining files still need review.", + } + save_recovery_state(job, state) + session.add( + ImportJobLog( + import_job_id=job_id, + level="INFO", + event="import_deferred_recovery_prepared", + message="Deferred recovery preparation completed.", + data={ + "local_counts": state.get("local_counts", {}), + "catalogs_checked": len(completed), + "catalog_files_prepared": catalog_count, + "source_preserved": True, + }, + ) + ) + await session.commit() + return True diff --git a/src/pullbox/services/import_duplicate_copies.py b/src/pullbox/services/import_duplicate_copies.py index d0e7b073..caf19745 100644 --- a/src/pullbox/services/import_duplicate_copies.py +++ b/src/pullbox/services/import_duplicate_copies.py @@ -30,7 +30,7 @@ def __call__( ContentHashFunc = Callable[[str], str | None] DuplicateSeriesPredicate = Callable[[ImportedSeries | None], bool] DuplicateTargetKeyFunc = Callable[[ImportedFile], tuple[str, int | float] | None] -FileSortKeyFunc = Callable[[ImportedFile], tuple[int, int, int, int]] +FileSortKeyFunc = Callable[[ImportedFile], tuple[int, int, int, int, int]] NormalizedReleaseNameFunc = Callable[[str], str] diff --git a/src/pullbox/services/import_duplicates.py b/src/pullbox/services/import_duplicates.py index 20809c0e..0a92144c 100644 --- a/src/pullbox/services/import_duplicates.py +++ b/src/pullbox/services/import_duplicates.py @@ -208,9 +208,14 @@ def _raw_name_has_scanner_suffix(raw_name: str | None, cv_title: str | None) -> return bool(_SCANNER_STYLE_SUFFIX_RE.fullmatch(suffix)) -def preferred_file_sort_key(item: ImportedFile) -> tuple[int, int, int, int]: +def preferred_file_sort_key(item: ImportedFile) -> tuple[int, int, int, int, int]: """Prefer richer metadata, stronger confidence, larger files, then older IDs.""" + cross_folder = dict(item.diagnostics or {}).get("mylar3_cross_folder_reconciliation") + canonical_mylar_reference = ( + isinstance(cross_folder, dict) and cross_folder.get("role") == "canonical" + ) return ( + 1 if canonical_mylar_reference else 0, 1 if item.has_comicinfo else 0, confidence_rank(item.match_confidence), item.file_size, diff --git a/src/pullbox/services/import_file_execution.py b/src/pullbox/services/import_file_execution.py index f8aefbad..38240b72 100644 --- a/src/pullbox/services/import_file_execution.py +++ b/src/pullbox/services/import_file_execution.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import os import shutil from contextlib import suppress from dataclasses import dataclass @@ -15,12 +16,34 @@ from sqlalchemy import select as sa_select from sqlalchemy.orm import joinedload -from pullbox.core.exceptions import JobCancelledError, JobPausedError +from pullbox.core.exceptions import ( + ConfigurationError, + ImportDestinationValidationError, + JobCancelledError, + JobPausedError, +) from pullbox.core.file_ops import LibraryFileRegistrationOutcome from pullbox.core.file_safety import classify_resource_safety_exception -from pullbox.models.import_job import ImportedFile, ImportedFileStatus, ImportJobAction +from pullbox.core.import_resources import bounded_async_map +from pullbox.core.issue_numbers import ( + issue_number_text_matches_numeric, + normalize_issue_number_text, +) +from pullbox.core.library_file_ownership import ( + ReferencedFileValidationError, + build_file_identity_signature, + build_managed_placement_signature, + validate_file_identity_signature, +) +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportFileHandlingMode, + ImportJobAction, + ImportSourceType, +) from pullbox.models.issue import Issue, IssueStatus, IssueType -from pullbox.models.library import LibraryFile, MatchConfidence +from pullbox.models.library import LibraryFile, LibraryFileStorageMode, MatchConfidence from pullbox.models.series import Series from pullbox.services.import_file_match_targets import ( PROVIDER_MISSING_ISSUE_PLACEHOLDER_KIND, @@ -31,6 +54,12 @@ load_issue_lookup_for_series, ) from pullbox.services.import_folder_adoption import apply_import_series_folder_adoption +from pullbox.services.import_job_actions import seed_action_sequence_cache +from pullbox.services.import_placement_recovery import ( + has_completed_direct_move_placement_record, +) +from pullbox.services.import_referenced_sources import revalidate_mylar_in_place_file_root +from pullbox.services.import_safety_diagnostics import build_import_safety_diagnostics from pullbox.utilities.settings import restore_file_from_utility_trash if TYPE_CHECKING: @@ -72,52 +101,314 @@ @dataclass(frozen=True, slots=True) class _PlaceholderIssueTarget: issue_number: float + issue_number_text: str | None issue_type: IssueType issue_title: str | None metadata_source: str +@dataclass(frozen=True, slots=True) +class _VerifiedLibraryAdoption: + rollback_snapshot: dict[str, object] + + +def _library_adoption_mapping(value: object) -> dict[str, object] | None: + if value is None: + return None + if not isinstance(value, dict): + raise ConfigurationError("Clean-library adoption evidence is invalid. Preview it again.") + return dict(value) + + +def _library_adoption_positive_int(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ConfigurationError(f"Clean-library adoption {label} is invalid. Preview it again.") + return value + + +async def _load_verified_library_adoption( + session: AsyncSession, + *, + job: ImportJob, + imported_file: ImportedFile, + issue: Issue, +) -> _VerifiedLibraryAdoption | None: + evidence = _library_adoption_mapping( + dict(imported_file.diagnostics or {}).get("library_adoption") + ) + if evidence is None: + return None + if evidence.get("schema_version") != 1 or evidence.get("source_preserved") is not True: + raise ConfigurationError("Clean-library adoption evidence is invalid. Preview it again.") + if evidence.get("source_storage_mode") != LibraryFileStorageMode.REFERENCED.value: + raise ConfigurationError("Only referenced library files can be adopted safely.") + + source_imported_file_id = _library_adoption_positive_int( + evidence.get("source_imported_file_id"), + "source import file", + ) + source_library_file_id = _library_adoption_positive_int( + evidence.get("source_library_file_id"), + "source library file", + ) + source_library_root_id = _library_adoption_positive_int( + evidence.get("source_library_root_id"), + "source library root", + ) + source_job_id = _library_adoption_positive_int( + evidence.get("source_import_job_id"), + "source import job", + ) + if ( + job.file_handling_mode is not ImportFileHandlingMode.MANAGED_COPY + or not job.source_preserved + ): + raise ConfigurationError( + "Clean-library adoption evidence is attached to the wrong import job." + ) + source_path = evidence.get("source_path") + expected_signature = evidence.get("source_signature") + if not isinstance(source_path, str) or not source_path: + raise ConfigurationError("Clean-library adoption source path is invalid.") + if not isinstance(expected_signature, dict) or not expected_signature: + raise ConfigurationError("Clean-library adoption source evidence is missing.") + + source_imported_file = await session.get(ImportedFile, source_imported_file_id) + source_library_file = await session.get(LibraryFile, source_library_file_id) + source_series = await session.get(Series, issue.series_id) + if ( + source_imported_file is None + or source_imported_file.import_job_id != source_job_id + or source_imported_file.status is not ImportedFileStatus.IMPORTED + or source_imported_file.library_file_id != source_library_file_id + or source_imported_file.matched_issue_id != issue.id + or source_imported_file.file_path != source_path + or source_series is None + ): + raise ConfigurationError( + "The original import reference changed after preview. Preview it again." + ) + if ( + source_library_file is None + or source_library_file.storage_mode is not LibraryFileStorageMode.REFERENCED + or source_library_file.issue_id != issue.id + or source_library_file.library_root_id != source_library_root_id + or source_library_file.file_path != source_path + or imported_file.file_path != source_path + or dict(source_library_file.source_signature or {}) != expected_signature + ): + raise ConfigurationError( + "The referenced library file changed after preview. Preview it again." + ) + current_signature = await asyncio.to_thread( + build_file_identity_signature, + Path(source_path), + ) + validate_file_identity_signature(expected_signature, current_signature) + + return _VerifiedLibraryAdoption( + rollback_snapshot={ + "schema_version": 1, + "source_imported_file_id": source_imported_file_id, + "source_library_file_id": source_library_file_id, + "file_path": source_library_file.file_path, + "file_name": source_library_file.file_name, + "file_size": source_library_file.file_size, + "file_format": source_library_file.file_format.value, + "file_hash": source_library_file.file_hash, + "file_modified_at": source_library_file.file_modified_at.isoformat(), + "match_confidence": source_library_file.match_confidence.value, + "parsed_series": source_library_file.parsed_series, + "parsed_issue_number": source_library_file.parsed_issue_number, + "parsed_year": source_library_file.parsed_year, + "parsed_publisher": source_library_file.parsed_publisher, + "has_comicinfo": source_library_file.has_comicinfo, + "naming_snapshot": dict(source_library_file.naming_snapshot or {}), + "storage_mode": source_library_file.storage_mode.value, + "source_signature": dict(source_library_file.source_signature or {}), + "issue_id": source_library_file.issue_id, + "library_root_id": source_library_file.library_root_id, + "source_series_id": source_series.id, + "previous_series_path": source_series.path, + "previous_series_library_root_id": source_series.library_root_id, + "previous_series_preferred_library_root_id": (source_series.preferred_library_root_id), + } + ) + + def _cleanup_failed_library_artifact( *, destination_path: Path | None, original_source: Path | None, original_trash_path: Path | None, transfer_method: str | None, + storage_mode: str | None, created_series_folder: bool, created_series_folder_path: Path | None, -) -> None: + expected_destination_signature: dict[str, object] | None, + created_directory_paths: tuple[Path, ...] = (), + directory_ownership_boundary_path: Path | None = None, +) -> bool: """Best-effort cleanup when import fails after the library artifact was placed.""" + if storage_mode == "referenced" or transfer_method == "leave_in_place": + return False + + destination_matches = _destination_matches_signature( + destination_path, + expected_destination_signature, + ) + destination_exists = bool(destination_path is not None and os.path.lexists(destination_path)) + destination_preserved = destination_exists and not destination_matches + source_reappeared = bool( + transfer_method == "move" + and original_source is not None + and os.path.lexists(original_source) + and ( + ( + destination_path is not None + and os.path.lexists(destination_path) + and original_source.resolve(strict=False) != destination_path.resolve(strict=False) + ) + or (original_trash_path is not None and os.path.lexists(original_trash_path)) + ) + ) + if source_reappeared: + # The source path may now contain a different user-owned artifact. Never + # let failure cleanup replace it or discard a proven destination/trash copy. + return True + if ( original_trash_path is not None and original_source is not None - and original_trash_path.exists() + and os.path.lexists(original_trash_path) ): restore_file_from_utility_trash(original_trash_path, original_source) - if destination_path is not None and destination_path.exists(): + if ( + destination_matches + and destination_path is not None + and os.path.lexists(destination_path) + ): destination_path.unlink(missing_ok=True) elif ( transfer_method in {"move", "leave_in_place"} + and destination_matches and destination_path is not None - and destination_path.exists() and original_source is not None and destination_path != original_source ): original_source.parent.mkdir(parents=True, exist_ok=True) shutil.move(str(destination_path), str(original_source)) - elif destination_path is not None and destination_path.exists(): + elif destination_matches and destination_path is not None: destination_path.unlink(missing_ok=True) - if ( - created_series_folder - and created_series_folder_path is not None - and created_series_folder_path.exists() - ): + if not destination_preserved: + owned_directories: tuple[Path, ...] = () + if created_directory_paths and directory_ownership_boundary_path is not None: + owned_directories = _validated_created_directory_paths( + created_directory_paths, + boundary_path=directory_ownership_boundary_path, + destination_parent=destination_path.parent + if destination_path is not None + else None, + ) + elif ( + created_series_folder + and created_series_folder_path is not None + and destination_path is not None + ): + try: + if created_series_folder_path.resolve( + strict=False + ) == destination_path.parent.resolve(strict=False): + owned_directories = (created_series_folder_path,) + except (OSError, RuntimeError): + owned_directories = () + for directory in sorted( + set(owned_directories), + key=lambda path: len(path.parts), + reverse=True, + ): + try: + directory.rmdir() + except (FileNotFoundError, OSError): + continue + return destination_preserved + + +def _validated_created_directory_paths( + paths: tuple[Path, ...], + *, + boundary_path: Path, + destination_parent: Path | None, +) -> tuple[Path, ...]: + """Return only exact import-owned descendants on the destination chain.""" + if destination_parent is None: + return () + try: + boundary_resolved = boundary_path.resolve(strict=False) + destination_resolved = destination_parent.resolve(strict=False) + destination_relative = destination_resolved.relative_to(boundary_resolved) + except (OSError, RuntimeError, ValueError): + return () + if not destination_relative.parts: + return () + + validated: list[Path] = [] + for path in paths: try: - next(created_series_folder_path.iterdir()) - except StopIteration: - created_series_folder_path.rmdir() - except OSError: - pass + path_resolved = path.resolve(strict=False) + relative = path_resolved.relative_to(boundary_resolved) + destination_resolved.relative_to(path_resolved) + except (OSError, RuntimeError, ValueError): + continue + if relative.parts: + validated.append(path) + return tuple(validated) + + +def _destination_matches_signature( + destination_path: Path | None, + expected_signature: dict[str, object] | None, +) -> bool: + if destination_path is None or not os.path.lexists(destination_path) or not expected_signature: + return False + try: + return build_managed_placement_signature(destination_path) == expected_signature + except (ConfigurationError, OSError, RuntimeError, ValueError): + return False + + +def _revalidate_managed_import_source( + source_path: Path, + expected_signature: dict[str, object], +) -> None: + try: + current_signature = build_file_identity_signature(source_path) + except (OSError, RuntimeError, ValueError) as exc: + raise ReferencedFileValidationError( + "source_missing", + "Managed-copy source is missing or unavailable. Rescan before retrying.", + ) from exc + validate_file_identity_signature(expected_signature, current_signature) + + +def _mylar_managed_source_boundary(source_path: Path, series_folder: str) -> Path: + """Use the series folder only when it actually contains this recorded issue.""" + if not series_folder: + return source_path.parent + candidate = Path(series_folder) + try: + lexical_source = source_path.expanduser().absolute() + lexical_candidate = candidate.expanduser().absolute() + resolved_source = source_path.expanduser().resolve(strict=False) + resolved_candidate = candidate.expanduser().resolve(strict=False) + except (OSError, RuntimeError, ValueError): + return source_path.parent + if lexical_source.is_relative_to(lexical_candidate) and resolved_source.is_relative_to( + resolved_candidate + ): + return candidate + return source_path.parent async def _load_issue_for_processing( @@ -143,6 +434,7 @@ def _resolve_import_file_issue_id( imp_file: ImportedFile, *, cv_id_to_issue_id: dict[int, int], + exact_number_to_issue_id: dict[str, int], number_to_issue_id: dict[float, int], ) -> int | None: if imp_file.matched_issue_id is not None: @@ -160,8 +452,19 @@ def _resolve_import_file_issue_id( placeholder_target = _placeholder_issue_target_from_diagnostics(imp_file) if placeholder_target is not None: + if placeholder_target.issue_number_text is not None: + return exact_number_to_issue_id.get(placeholder_target.issue_number_text) return number_to_issue_id.get(placeholder_target.issue_number) + if imp_file.issue_number_raw and imp_file.parsed_issue_number is not None: + with suppress(ValueError): + exact_issue_number = normalize_issue_number_text(imp_file.issue_number_raw) + if issue_number_text_matches_numeric( + imp_file.parsed_issue_number, + exact_issue_number, + ): + return exact_number_to_issue_id.get(exact_issue_number) + if imp_file.parsed_issue_number is not None: return number_to_issue_id.get(imp_file.parsed_issue_number) @@ -188,7 +491,7 @@ async def _requires_serial_file_processing( resolved_series_id: int, importable_files: list[ImportedFile], ) -> bool: - cv_id_to_issue, number_to_issue = await load_issue_lookup_for_series( + cv_id_to_issue, exact_number_to_issue, number_to_issue = await load_issue_lookup_for_series( session, resolved_series_id, ) @@ -202,6 +505,11 @@ async def _requires_serial_file_processing( for issue_number, issue in number_to_issue.items() if issue.id is not None } + exact_number_to_issue_id = { + issue_number: issue.id + for issue_number, issue in exact_number_to_issue.items() + if issue.id is not None + } seen_issue_ids: set[int] = set() seen_file_names: set[str] = set() for imp_file in importable_files: @@ -213,6 +521,7 @@ async def _requires_serial_file_processing( resolved_issue_id = _resolve_import_file_issue_id( imp_file, cv_id_to_issue_id=cv_id_to_issue_id, + exact_number_to_issue_id=exact_number_to_issue_id, number_to_issue_id=number_to_issue_id, ) if resolved_issue_id is None: @@ -241,9 +550,16 @@ def _placeholder_issue_target_from_diagnostics( issue_type = IssueType(str(diagnostics.get("target_issue_type"))) except (TypeError, ValueError): return None + issue_number_text: str | None = None + if imp_file.issue_number_raw: + with suppress(ValueError): + normalized_text = normalize_issue_number_text(imp_file.issue_number_raw) + if issue_number_text_matches_numeric(issue_number, normalized_text): + issue_number_text = normalized_text issue_title = diagnostics.get("target_issue_title") return _PlaceholderIssueTarget( issue_number=issue_number, + issue_number_text=issue_number_text, issue_type=issue_type, issue_title=str(issue_title) if issue_title else None, metadata_source=( @@ -269,12 +585,24 @@ async def _ensure_placeholder_issue_targets( return False issues_result = await session.execute(sa_select(Issue).where(Issue.series_id == series_id)) - existing_by_number = {issue.issue_number: issue for issue in issues_result.scalars().all()} + existing_issues = list(issues_result.scalars().all()) + existing_by_exact = {issue.effective_issue_number_text: issue for issue in existing_issues} + existing_by_number: dict[float, list[Issue]] = {} + for issue in existing_issues: + existing_by_number.setdefault(issue.issue_number, []).append(issue) created_count = 0 max_target_issue_number = 0 for target in placeholder_targets: - existing = existing_by_number.get(target.issue_number) + if target.issue_number_text is not None: + existing = existing_by_exact.get(target.issue_number_text) + else: + numeric_candidates = existing_by_number.get(target.issue_number, []) + if len(numeric_candidates) > 1: + continue + existing = numeric_candidates[0] if numeric_candidates else None if existing is not None: + if target.issue_number_text is not None and existing.issue_number_text is None: + existing.issue_number_text = target.issue_number_text if existing.issue_type == IssueType.ISSUE: existing.issue_type = target.issue_type if target.issue_title and not existing.title: @@ -291,8 +619,11 @@ async def _ensure_placeholder_issue_targets( status=IssueStatus.SKIPPED, metadata_source=target.metadata_source, ) + if target.issue_number_text is not None: + issue.issue_number_text = target.issue_number_text session.add(issue) - existing_by_number[target.issue_number] = issue + existing_by_exact[issue.effective_issue_number_text] = issue + existing_by_number.setdefault(target.issue_number, []).append(issue) created_count += 1 if created_count: @@ -300,7 +631,7 @@ async def _ensure_placeholder_issue_targets( if series is not None: series.issue_count = max( int(series.issue_count or 0), - len(existing_by_number), + len(existing_issues) + created_count, max_target_issue_number, ) await session.flush() @@ -453,6 +784,7 @@ async def process_import_series_files( move_to_trash: MoveToTrashFunc, report_file_progress: ReportFileProgressFunc | None = None, defer_comicinfo_enrichment: bool = False, + revalidate_managed_sources: bool = False, file_worker_count: int = 1, session_factory: async_sessionmaker[AsyncSession] | None = None, _file_ids_override: list[int] | None = None, @@ -483,8 +815,12 @@ async def process_import_series_files( raise ValueError("Resolved series id is required before importing files") item_id = item.id job_id = job.id - move_to_library = bool(job.move_to_library) - transfer_method = job.effective_transfer_method or job.transfer_method + in_place = job.file_handling_mode == ImportFileHandlingMode.IN_PLACE + move_to_library = not in_place + transfer_method = ( + "leave_in_place" if in_place else job.effective_transfer_method or job.transfer_method + ) + storage_mode = LibraryFileStorageMode.REFERENCED if in_place else LibraryFileStorageMode.MANAGED target_library_root_id = job.target_library_root_id update_embedded_comicinfo_from_match = bool(job.update_embedded_comicinfo_from_match) ingest_policy = await load_ingest_policy(session, job) @@ -537,7 +873,6 @@ async def process_import_series_files( ) or 0 ) - semaphore = asyncio.Semaphore(min(effective_worker_count, len(importable_file_ids))) async def locked_record_action( session: AsyncSession, @@ -550,6 +885,11 @@ async def locked_record_action( nonlocal next_action_sequence async with record_action_lock: next_action_sequence += 1 + seed_action_sequence_cache( + session, + int(job.id), + last_sequence=next_action_sequence - 1, + ) action = await record_action( session, job, @@ -562,7 +902,7 @@ async def locked_record_action( return action async def process_one_file(imp_file_id: int) -> tuple[int, int]: - async with semaphore, session_factory() as worker_session: + async with session_factory() as worker_session: worker_job = await worker_session.get(type(job), job_id) worker_item = await worker_session.get(type(item), item_id) if worker_job is None or worker_item is None: @@ -588,6 +928,7 @@ async def process_one_file(imp_file_id: int) -> tuple[int, int]: move_to_trash=move_to_trash, report_file_progress=report_file_progress, defer_comicinfo_enrichment=defer_comicinfo_enrichment, + revalidate_managed_sources=revalidate_managed_sources, file_worker_count=1, session_factory=None, _file_ids_override=[imp_file_id], @@ -597,11 +938,16 @@ async def process_one_file(imp_file_id: int) -> tuple[int, int]: _setup_placeholder_targets=False, ) - results = await asyncio.gather( - *(process_one_file(int(imp_file_id)) for imp_file_id in importable_file_ids) - ) - files_imported = sum(imported for imported, _failed in results) - files_failed = sum(failed for _imported, failed in results) + files_imported = 0 + files_failed = 0 + async with bounded_async_map( + process_one_file, + importable_file_ids, + workers=effective_worker_count, + ) as results: + async for imported, failed in results: + files_imported += imported + files_failed += failed reloaded_item = await session.get(type(item), item_id) if reloaded_item is not None: item = reloaded_item @@ -628,7 +974,7 @@ async def process_one_file(imp_file_id: int) -> tuple[int, int]: await session.flush() return files_imported, files_failed - cv_id_to_issue, number_to_issue = await load_issue_lookup_for_series( + cv_id_to_issue, exact_number_to_issue, number_to_issue = await load_issue_lookup_for_series( session, resolved_series_id, ) @@ -642,14 +988,23 @@ async def process_one_file(imp_file_id: int) -> tuple[int, int]: for issue_number, issue in number_to_issue.items() if issue.id is not None } + exact_number_to_issue_id = { + issue_number: issue.id + for issue_number, issue in exact_number_to_issue.items() + if issue.id is not None + } media_settings = await load_media_settings(session, job) skip_existing_enabled = media_settings["skip_existing_files"].lower() == "true" - trash_dir = await load_trash_dir(session, job) permission_policy = await load_permission_policy(session, job) - trash_dir.mkdir(parents=True, exist_ok=True) issue_ids = { - issue.id for issue in [*cv_id_to_issue.values(), *number_to_issue.values()] if issue.id + issue.id + for issue in [ + *cv_id_to_issue.values(), + *exact_number_to_issue.values(), + *number_to_issue.values(), + ] + if issue.id } owned_issue_ids = ( await _load_owned_issue_ids(session, issue_ids) if skip_existing_enabled else set() @@ -670,9 +1025,15 @@ async def process_one_file(imp_file_id: int) -> tuple[int, int]: ) prepared: PreparedImportFile | None = None placed_destination_path: Path | None = None + placed_destination_signature: dict[str, object] | None = None placed_series_folder_created = False placed_series_folder_path: Path | None = None + placed_created_directory_paths: tuple[Path, ...] = () + placed_directory_ownership_boundary_path: Path | None = None + placed_storage_mode = "referenced" if not move_to_library else "managed" original_trash_path: Path | None = None + source_transfer_method = transfer_method + completed_placement_recovery_pending = False imp_file = await _load_imported_file_for_processing(session, imp_file_id) if imp_file is None: continue @@ -681,8 +1042,46 @@ async def process_one_file(imp_file_id: int) -> tuple[int, int]: raise ValueError("Import job disappeared during file processing") imp_file_name = imp_file.file_name imp_file_path = imp_file.file_path + registration_library_root_id = None if in_place else target_library_root_id try: await raise_if_cancelled(session, job_id) + if in_place and current_job.source_type == ImportSourceType.MYLAR3: + registration_library_root_id = await revalidate_mylar_in_place_file_root( + session, + Path(imp_file.file_path), + dict(imp_file.source_signature or {}), + ) + elif not in_place and revalidate_managed_sources: + try: + await asyncio.to_thread( + _revalidate_managed_import_source, + Path(imp_file.file_path), + dict(imp_file.source_signature or {}), + ) + except ReferencedFileValidationError as exc: + can_reach_recovery = ( + exc.reason == "source_missing" + and await has_completed_direct_move_placement_record( + session, + job_id=job_id, + imported_file_id=int(imp_file.id), + source_path=Path(imp_file.file_path), + ) + ) + if not can_reach_recovery: + raise + completed_placement_recovery_pending = True + + if in_place: + source_scan_root = None + elif current_job.source_type == ImportSourceType.FILESYSTEM: + source_scan_root = Path(current_job.source_path) + else: + source_folder = str(item.source_folder or "").strip() + source_scan_root = _mylar_managed_source_boundary( + Path(imp_file.file_path), + source_folder, + ) def _build_current_file_reporter( current_imp_file: ImportedFile, @@ -732,6 +1131,7 @@ async def report_current_file( resolved_issue_id = _resolve_import_file_issue_id( imp_file, cv_id_to_issue_id=cv_id_to_issue_id, + exact_number_to_issue_id=exact_number_to_issue_id, number_to_issue_id=number_to_issue_id, ) if resolved_issue_id is None: @@ -756,7 +1156,18 @@ async def report_current_file( imp_file.match_confidence or "", MatchConfidence.MEDIUM ) - if skip_existing_enabled and resolved_issue.id in owned_issue_ids: + library_adoption = await _load_verified_library_adoption( + session, + job=current_job, + imported_file=imp_file, + issue=resolved_issue, + ) + + if ( + library_adoption is None + and skip_existing_enabled + and resolved_issue.id in owned_issue_ids + ): imp_file.status = ImportedFileStatus.SKIPPED await log_event( session, @@ -824,6 +1235,10 @@ async def report_current_file( source_path=imp_file.file_path, ) if prepared.converted: + if move_to_library and transfer_method == "move": + # The moved artifact is a disposable conversion workspace; + # Collection Import still preserves the original source. + source_transfer_method = "copy" await log_event( session, job_id, @@ -835,7 +1250,7 @@ async def report_current_file( source_file_name=Path(prepared.original_source).name, prepared_file_name=Path(prepared.registration_source).name, ) - if effective_embedded_comicinfo: + if effective_embedded_comicinfo and not completed_placement_recovery_pending: prepared_source_path = Path(prepared.registration_source) comicinfo_payload_cache_key = (resolved_issue.id, str(prepared_source_path)) comicinfo_payload = comicinfo_payload_cache.get(comicinfo_payload_cache_key) @@ -859,7 +1274,11 @@ async def report_current_file( resolved_issue, confidence, move_to_library=move_to_library, - library_root_id=target_library_root_id, + storage_mode=storage_mode, + expected_source_signature=( + dict(imp_file.source_signature) if in_place else None + ), + library_root_id=registration_library_root_id, transfer_method=transfer_method, normalize_to_cbz=False, update_embedded_comicinfo_from_match=effective_embedded_comicinfo, @@ -873,9 +1292,39 @@ async def report_current_file( comicinfo_progress_callback=( _report_current_file if report_file_progress else None ), + recovery_imported_file_id=int(imp_file.id), + recovery_original_source_path=Path(imp_file.file_path), + replace_existing_library_file=library_adoption is not None, + replacement_trash_dir=None, + preserve_replaced_artifact=library_adoption is not None, + source_scan_root=source_scan_root, + strict_import_target=not in_place, ) library_file, registration = _registration_outcome(registration_result) + if library_adoption is not None: + adopted_series = await session.get(Series, resolved_issue.series_id) + if adopted_series is None: # pragma: no cover - issue FK guarantees this + raise ConfigurationError("The clean-library series no longer exists.") + adopted_series.path = str(Path(library_file.file_path).parent) + adopted_series.library_root_id = library_file.library_root_id + adopted_series.preferred_library_root_id = library_file.library_root_id + library_adoption.rollback_snapshot.update( + { + "installed_series_path": adopted_series.path, + "installed_series_library_root_id": adopted_series.library_root_id, + "installed_series_preferred_library_root_id": ( + adopted_series.preferred_library_root_id + ), + } + ) + library_file.has_comicinfo = bool( + library_file.has_comicinfo + or imp_file.has_comicinfo + or comicinfo_payload is not None + ) placed_destination_path = Path(library_file.file_path) + placed_destination_signature = dict(library_file.source_signature or {}) + placed_storage_mode = library_file.storage_mode.value placed_series_folder_created = ( bool(registration.series_folder_created) if registration is not None else False ) @@ -884,6 +1333,12 @@ async def report_current_file( if registration is not None else placed_destination_path.parent ) + placed_created_directory_paths = ( + registration.created_directory_paths if registration is not None else () + ) + placed_directory_ownership_boundary_path = ( + registration.directory_ownership_boundary_path if registration is not None else None + ) final_file_name = placed_destination_path.name if comicinfo_payload is not None: await log_event( @@ -909,13 +1364,6 @@ async def report_current_file( unit="steps", live_only=True, ) - if prepared.converted and transfer_method == "move": - original_trash_path = await asyncio.to_thread( - move_to_trash, - prepared.original_source, - trash_dir, - relative_path=Path("imports") / prepared.original_source.name, - ) if report_file_progress is not None: await report_file_progress( imp_file=imp_file, @@ -943,6 +1391,7 @@ async def report_current_file( "issue_id": resolved_issue.id, "issue_cv_id": resolved_issue.comicvine_id, "library_file_id": library_file.id, + "artifact_path": library_file.file_path, "queued_at": datetime.now(UTC).isoformat(), } imp_file.diagnostics = diagnostics @@ -978,8 +1427,12 @@ async def report_current_file( "imported_file_id": imp_file.id, "library_file_id": library_file.id, "destination_path": library_file.file_path, + "destination_signature": dict(library_file.source_signature or {}), "original_source_path": str(prepared.original_source), - "transfer_method": (transfer_method if move_to_library else "leave_in_place"), + "transfer_method": ( + source_transfer_method if move_to_library else "leave_in_place" + ), + "storage_mode": placed_storage_mode, "original_trash_path": ( str(original_trash_path) if original_trash_path is not None else "" ), @@ -996,10 +1449,23 @@ async def report_current_file( if registration is not None and registration.series_folder_path is not None else "" ), + "created_directory_paths": [ + str(path) for path in placed_created_directory_paths + ], + "directory_ownership_boundary_path": ( + str(placed_directory_ownership_boundary_path) + if placed_directory_ownership_boundary_path is not None + else None + ), "permission_restores": _permission_restore_payload( registration, original_source=prepared.original_source, - transfer_method=(transfer_method if move_to_library else "leave_in_place"), + transfer_method=( + source_transfer_method if move_to_library else "leave_in_place" + ), + ), + "adopted_reference": ( + library_adoption.rollback_snapshot if library_adoption is not None else None ), }, ) @@ -1079,15 +1545,20 @@ async def report_current_file( await asyncio.to_thread(cleanup_prepared_file, prepared) await session.rollback() placeholder_progress_live_only = False + destination_preserved_for_review = False try: - await asyncio.to_thread( + destination_preserved_for_review = await asyncio.to_thread( _cleanup_failed_library_artifact, destination_path=placed_destination_path, original_source=prepared.original_source if prepared is not None else None, original_trash_path=original_trash_path, - transfer_method=transfer_method, + transfer_method=source_transfer_method, + storage_mode=placed_storage_mode, created_series_folder=placed_series_folder_created, created_series_folder_path=placed_series_folder_path, + expected_destination_signature=placed_destination_signature, + created_directory_paths=placed_created_directory_paths, + directory_ownership_boundary_path=(placed_directory_ownership_boundary_path), ) except Exception: logger.exception( @@ -1106,11 +1577,74 @@ async def report_current_file( reloaded_item = await session.get(type(item), item_id) if reloaded_item is not None: item = reloaded_item + if destination_preserved_for_review: + diagnostics = dict(imp_file.diagnostics or {}) + diagnostics["destination_preservation"] = { + "kind": "destination_preserved_for_review", + "code": "destination_changed_after_placement", + "retryable": False, + "overrideable": False, + } + imp_file.diagnostics = diagnostics logger.debug( "import_file_failed", file_path=imp_file_path, error=str(exc), ) + if isinstance(exc, ImportDestinationValidationError): + diagnostics = dict(imp_file.diagnostics or {}) + diagnostics["destination_review"] = { + "kind": "managed_destination_review", + "code": exc.reason, + "reason": ( + "The planned managed-library destination is not a new, disjoint path. " + "Review the existing artifact or choose another destination." + ), + "retryable": False, + "overrideable": False, + } + imp_file.status = ImportedFileStatus.FAILED + imp_file.include_in_import = False + imp_file.error_message = str(exc) + imp_file.diagnostics = diagnostics + files_failed += 1 + await log_event( + session, + job_id, + "WARNING", + "import_file_destination_review_required", + message=f"Managed destination needs review: {imp_file_name}", + source_path=imp_file_path, + reason=exc.reason, + ) + await session.commit() + continue + if isinstance(exc, ReferencedFileValidationError): + diagnostics = dict(imp_file.diagnostics or {}) + diagnostics["source_revalidation"] = build_import_safety_diagnostics( + str(exc), + kind="source_revalidation", + code=exc.reason, + source="source_revalidation", + overrideable_hint=False, + ) + imp_file.status = ImportedFileStatus.FAILED + imp_file.include_in_import = False + imp_file.error_message = str(exc) + imp_file.diagnostics = diagnostics + files_failed += 1 + await log_event( + session, + job_id, + "WARNING", + "import_file_source_revalidation_failed", + message=f"Source changed after scan; rescan before retry: {imp_file_name}", + source_path=imp_file_path, + reason=exc.reason, + ) + await session.commit() + continue + resource_block = classify_resource_safety_exception(exc) if resource_block is not None: diagnostics = dict(imp_file.diagnostics or {}) diff --git a/src/pullbox/services/import_file_execution_protocols.py b/src/pullbox/services/import_file_execution_protocols.py index 8b37af23..ba00bbbe 100644 --- a/src/pullbox/services/import_file_execution_protocols.py +++ b/src/pullbox/services/import_file_execution_protocols.py @@ -15,7 +15,7 @@ from pullbox.core.library_policy import LibraryIngestPolicy from pullbox.models.import_job import ImportedFile, ImportJob, ImportJobAction from pullbox.models.issue import Issue - from pullbox.models.library import LibraryFile, MatchConfidence + from pullbox.models.library import LibraryFile, LibraryFileStorageMode, MatchConfidence from pullbox.services.import_file_preparation import PreparedImportFile @@ -107,6 +107,8 @@ async def __call__( confidence: MatchConfidence, *, move_to_library: bool, + storage_mode: LibraryFileStorageMode, + expected_source_signature: dict[str, object] | None, library_root_id: int | None, transfer_method: str | None, normalize_to_cbz: bool | None = None, @@ -119,7 +121,14 @@ async def __call__( | None = None, comicinfo_progress_callback: Callable[[str, int, int, str], Awaitable[None] | None] | None = None, + recovery_imported_file_id: int | None = None, + recovery_original_source_path: Path | None = None, placement_started_callback: Callable[..., Awaitable[None] | None] | None = None, + replace_existing_library_file: bool = False, + replacement_trash_dir: Path | None = None, + preserve_replaced_artifact: bool = False, + source_scan_root: Path | None = None, + strict_import_target: bool = False, ) -> LibraryFile | LibraryFileRegistrationOutcome: ... diff --git a/src/pullbox/services/import_file_interruptible_ops.py b/src/pullbox/services/import_file_interruptible_ops.py index 82f591f6..d4b13fba 100644 --- a/src/pullbox/services/import_file_interruptible_ops.py +++ b/src/pullbox/services/import_file_interruptible_ops.py @@ -105,6 +105,7 @@ async def materialize_import_cbz_with_comicinfo_interruptible( comicinfo_payload: dict[str, Any], *, transfer_method: str, + temp_path: Path | None = None, progress_callback: ProgressCallback | None = None, raise_if_cancelled_immediately: RaiseIfCancelledImmediately, materializer: MaterializeCbzWithComicInfoInterruptible | None = None, @@ -116,6 +117,7 @@ async def materialize_import_cbz_with_comicinfo_interruptible( target_path, comicinfo_payload, transfer_method=transfer_method, + temp_path=temp_path, cancellation_check=lambda: raise_if_cancelled_immediately(session, int(job.id)), progress_callback=progress_callback, ) diff --git a/src/pullbox/services/import_file_issue_signals.py b/src/pullbox/services/import_file_issue_signals.py index e7feb62e..f681c748 100644 --- a/src/pullbox/services/import_file_issue_signals.py +++ b/src/pullbox/services/import_file_issue_signals.py @@ -5,6 +5,10 @@ import re from typing import TYPE_CHECKING +from pullbox.core.issue_numbers import ( + issue_number_text_matches_numeric, + normalize_issue_number_text, +) from pullbox.core.release_parser import normalize_issue_number, parse_release_title if TYPE_CHECKING: @@ -48,6 +52,30 @@ def filename_issue_number(imp_file: ImportedFile) -> float | None: return parsed.issue_number if parsed is not None else None +def comicinfo_issue_number_raw(imp_file: ImportedFile) -> str | float | int | None: + """Return the issue designation cached from ComicInfo.xml during discovery.""" + diagnostics = imp_file.diagnostics if isinstance(imp_file.diagnostics, dict) else {} + source_metadata = diagnostics.get("source_metadata") + comicinfo = source_metadata.get("comicinfo") if isinstance(source_metadata, dict) else None + value = comicinfo.get("number") if isinstance(comicinfo, dict) else None + return value if isinstance(value, str | float | int) else None + + +def comicinfo_issue_number(imp_file: ImportedFile) -> float | None: + """Return a numeric compatibility value from saved ComicInfo.xml evidence.""" + raw_value = comicinfo_issue_number_raw(imp_file) + normalized = normalize_issue_number(raw_value) + if normalized is not None: + return normalized + if not isinstance(raw_value, str): + return None + bracketed_total = re.fullmatch( + r"\s*([+-]?(?:\d+(?:\.\d*)?|\.\d+))\s*\[\s*\d+\s*\]\s*", + raw_value, + ) + return normalize_issue_number(bracketed_total.group(1)) if bracketed_total else None + + def candidate_issue_number(imp_file: ImportedFile) -> float | None: """Return the best issue-number signal available for target lookup.""" if imp_file.parsed_issue_number is not None: @@ -55,4 +83,37 @@ def candidate_issue_number(imp_file: ImportedFile) -> float | None: parsed_issue_number = filename_issue_number(imp_file) if parsed_issue_number is not None: return parsed_issue_number - return volume_issue_number(imp_file) + volume_number = volume_issue_number(imp_file) + if volume_number is not None: + return volume_number + comicinfo_number = comicinfo_issue_number(imp_file) + if comicinfo_number is not None: + return comicinfo_number + diagnostics = imp_file.diagnostics if isinstance(imp_file.diagnostics, dict) else {} + comicinfo_raw = comicinfo_issue_number_raw(imp_file) + comicinfo_number_missing = comicinfo_raw is None or ( + isinstance(comicinfo_raw, str) and not comicinfo_raw.strip() + ) + if ( + comicinfo_number_missing + and diagnostics.get("source_issue_type") == "one_shot" + and diagnostics.get("issue_count_hint") == 1 + and imp_file.matched_issue_cv_id is not None + ): + return 1.0 + return None + + +def candidate_issue_number_text(imp_file: ImportedFile) -> str | None: + """Return a validated exact issue designation when the source preserved one.""" + issue_number = candidate_issue_number(imp_file) + raw_value = imp_file.issue_number_raw or comicinfo_issue_number_raw(imp_file) + if not raw_value or issue_number is None: + return None + try: + normalized = normalize_issue_number_text(raw_value) + except ValueError: + return None + if not issue_number_text_matches_numeric(issue_number, normalized): + return None + return normalized diff --git a/src/pullbox/services/import_file_match_candidates.py b/src/pullbox/services/import_file_match_candidates.py index fc36cf8e..0162558b 100644 --- a/src/pullbox/services/import_file_match_candidates.py +++ b/src/pullbox/services/import_file_match_candidates.py @@ -15,7 +15,10 @@ ) from pullbox.models.issue import IssueType from pullbox.services.import_embedded_title_match import embedded_issue_number_title_match -from pullbox.services.import_file_issue_signals import candidate_issue_number +from pullbox.services.import_file_issue_signals import ( + candidate_issue_number, + candidate_issue_number_text, +) from pullbox.services.import_file_match_targets import ( PROVIDER_MISSING_ISSUE_PLACEHOLDER_METHOD, PROVIDER_ZERO_ISSUE_PLACEHOLDER_METHOD, @@ -175,6 +178,30 @@ def select_file_match_candidate( if imp_file.comicvine_issue_id is not None: return None + exact_issue_number = candidate_issue_number_text(imp_file) + if exact_issue_number is not None and target_index.exact_number_map: + exact_target = target_index.exact_number_map.get(exact_issue_number) + if exact_target is None: + return None + matched_issue_id, matched_issue_cv_id, has_library_file, matched_issue, issue_title = ( + exact_target + ) + target_issue_number = ( + matched_issue.issue_number + if matched_issue is not None + else candidate_issue_number(imp_file) + ) + return FileMatchCandidate( + matched_issue_id=matched_issue_id, + matched_issue_cv_id=matched_issue_cv_id, + target_issue_number=target_issue_number, + has_library_file=has_library_file, + matched_issue=matched_issue, + target_issue_title=issue_title, + confidence="high" if series_high_confidence else "medium", + method="issue_number", + ) + issue_number = candidate_issue_number(imp_file) if issue_number is not None and issue_number in target_index.number_map: matched_issue_id, matched_issue_cv_id, has_library_file, matched_issue, issue_title = ( diff --git a/src/pullbox/services/import_file_match_outcomes.py b/src/pullbox/services/import_file_match_outcomes.py index 717f720c..079743fd 100644 --- a/src/pullbox/services/import_file_match_outcomes.py +++ b/src/pullbox/services/import_file_match_outcomes.py @@ -8,6 +8,7 @@ from pullbox.core.release_parser import parse_release_title from pullbox.models.import_job import ImportedFileStatus from pullbox.models.issue import IssueType +from pullbox.services.import_file_issue_signals import candidate_issue_number_text from pullbox.services.import_file_match_targets import ( PROVIDER_MISSING_ISSUE_PLACEHOLDER_KIND, PROVIDER_MISSING_ISSUE_PLACEHOLDER_METHOD, @@ -44,6 +45,22 @@ class FileMatchLogEvent: level: str = "DEBUG" +_SOURCE_EVIDENCE_DIAGNOSTIC_KEYS = ( + "source_issue_type", + "comicvine_series_id", + "series_status", + "issue_count_hint", + "metadata_signals", + "source_metadata", + "mylar3_cross_folder_reconciliation", +) + + +def _source_evidence_diagnostics(imp_file: ImportedFile) -> dict[str, Any]: + diagnostics = dict(imp_file.diagnostics or {}) + return {key: diagnostics[key] for key in _SOURCE_EVIDENCE_DIAGNOSTIC_KEYS if key in diagnostics} + + def apply_matched_file_outcome( imp_file: ImportedFile, imp_series: ImportedSeries, @@ -53,6 +70,7 @@ def apply_matched_file_outcome( duplicate_target_state: DuplicateTargetStateFunc, ) -> FileMatchLogEvent: """Apply accepted file-match status/diagnostics and return its log event.""" + source_evidence = _source_evidence_diagnostics(imp_file) imp_file.matched_issue_id = match_candidate.matched_issue_id imp_file.matched_issue_cv_id = match_candidate.matched_issue_cv_id imp_file.match_confidence = match_candidate.confidence @@ -63,10 +81,13 @@ def apply_matched_file_outcome( if match_candidate.has_library_file: imp_file.status = ImportedFileStatus.ALREADY_OWNED imp_file.include_in_import = False - imp_file.diagnostics = _duplicate_file_diagnostics( - matched_issue, - target_state="already_owned", - ) + imp_file.diagnostics = { + **source_evidence, + **_duplicate_file_diagnostics( + matched_issue, + target_state="already_owned", + ), + } return FileMatchLogEvent( name="import_duplicate_file_already_owned", message=f"Duplicate file already owned: {imp_file.file_name}", @@ -81,10 +102,13 @@ def apply_matched_file_outcome( target_state = duplicate_target_state(matched_issue) imp_file.status = ImportedFileStatus.MATCHED imp_file.include_in_import = False - imp_file.diagnostics = _duplicate_file_diagnostics( - matched_issue, - target_state=target_state, - ) + imp_file.diagnostics = { + **source_evidence, + **_duplicate_file_diagnostics( + matched_issue, + target_state=target_state, + ), + } return FileMatchLogEvent( name="import_duplicate_file_importable_match", message=f"Duplicate file matched to {target_state} issue: {imp_file.file_name}", @@ -109,6 +133,7 @@ def apply_matched_file_outcome( ): provisional = match_candidate.method == PROVIDER_MISSING_ISSUE_PLACEHOLDER_METHOD imp_file.diagnostics = { + **source_evidence, "kind": ( PROVIDER_MISSING_ISSUE_PLACEHOLDER_KIND if provisional @@ -131,11 +156,14 @@ def apply_matched_file_outcome( ), } else: - imp_file.diagnostics = _target_issue_summary_diagnostics( - imp_file, - imp_series, - match_candidate, - ) + imp_file.diagnostics = { + **source_evidence, + **_target_issue_summary_diagnostics( + imp_file, + imp_series, + match_candidate, + ), + } return FileMatchLogEvent( name="import_file_match_detail", message=f"File matched: {imp_file.file_name}", @@ -174,6 +202,16 @@ def _target_issue_summary_diagnostics( "cover_url": None, "issue_type": issue_type.value, } + issue_number_text = ( + match_candidate.matched_issue.effective_issue_number_text + if match_candidate.matched_issue is not None + else candidate_issue_number_text(imp_file) + ) + numeric_issue_text = ( + str(int(issue_number)) if float(issue_number).is_integer() else str(float(issue_number)) + ) + if issue_number_text is not None and issue_number_text != numeric_issue_text: + target_summary["issue_number_text"] = issue_number_text return { "target_issue_summary": target_summary, } @@ -292,6 +330,7 @@ def apply_unmatched_file_outcome( ) next_diagnostics = metadata_conflict or { "kind": "duplicate_series_file", + "reason": "duplicate_series_no_importable_target", "target_state": ( "no_importable_targets" if duplicate_merge_profile is not None and not duplicate_merge_profile.actionable @@ -309,6 +348,10 @@ def apply_unmatched_file_outcome( else 0 ), } + next_diagnostics.setdefault( + "reason", + str(next_diagnostics.get("kind") or "duplicate_series_issue_target_not_found"), + ) imp_file.diagnostics = {**existing_diagnostics, **next_diagnostics} informational_only = ( duplicate_merge_profile is not None and not duplicate_merge_profile.actionable @@ -341,9 +384,23 @@ def apply_unmatched_file_outcome( ) imp_file.diagnostics = ( - {**existing_diagnostics, **metadata_conflict} + { + **existing_diagnostics, + **metadata_conflict, + "reason": str( + metadata_conflict.get("reason") + or metadata_conflict.get("kind") + or "issue_target_not_found" + ), + } if metadata_conflict is not None - else existing_diagnostics + else { + **existing_diagnostics, + "reason": "issue_target_not_found", + "rejection_reason": ( + "No issue target matched the available file name and metadata evidence." + ), + } ) return FileMatchLogEvent( name=( diff --git a/src/pullbox/services/import_file_match_results.py b/src/pullbox/services/import_file_match_results.py index bc53b33f..9a5ab42b 100644 --- a/src/pullbox/services/import_file_match_results.py +++ b/src/pullbox/services/import_file_match_results.py @@ -19,6 +19,9 @@ _TRUSTED_SERIES_MATCH_METHODS = frozenset({"mylar3_cv_id", "comicinfo_cv_id", "folder_cv_id"}) +_SOURCE_LAYOUT_REVIEW_MESSAGE = ( + "This file does not fit the selected source layout. Review its series before importing." +) @dataclass(frozen=True, slots=True) @@ -67,8 +70,88 @@ def apply_file_match_series_summary( metadata_conflict_files = [ f for f in files if dict(f.diagnostics or {}).get("kind") == "metadata_conflict" ] + trusted_identity_conflict_files = [ + f + for f in metadata_conflict_files + if dict(f.diagnostics or {}).get("conflict_type") == "trusted_source_identity_conflict" + ] + source_layout_review_files = [ + f for f in files if dict(f.diagnostics or {}).get("kind") == "source_layout_review" + ] invalidation_diagnostics: dict[str, Any] | None = None if ( + not duplicate_series + and imp_series.status == ImportSeriesStatus.MATCHED + and trusted_identity_conflict_files + and imp_series.cv_match_method == "mylar3_cv_id" + ): + diagnostics = dict(imp_series.diagnostics or {}) + diagnostics.update( + { + "file_identity_conflict_count": len(trusted_identity_conflict_files), + "file_identity_review_required": True, + "conflicting_files": [ + { + "file_name": imp_file.file_name, + "rejection_reason": dict(imp_file.diagnostics or {}).get( + "rejection_reason" + ), + } + for imp_file in trusted_identity_conflict_files + ], + } + ) + imp_series.diagnostics = diagnostics + elif ( + not duplicate_series + and imp_series.status == ImportSeriesStatus.MATCHED + and trusted_identity_conflict_files + ): + identity_conflicts: list[dict[str, object]] = [] + for imp_file in trusted_identity_conflict_files: + raw_conflicts = dict(imp_file.diagnostics or {}).get("identity_conflicts") + if not isinstance(raw_conflicts, list): + continue + for conflict_item in raw_conflicts: + if isinstance(conflict_item, dict) and conflict_item not in identity_conflicts: + identity_conflicts.append(dict(conflict_item)) + invalidation_diagnostics = { + **dict(imp_series.diagnostics or {}), + "kind": "series_conflict", + "reason": "trusted_source_identity_conflict", + "raw_name": imp_series.raw_series_name, + "raw_year": imp_series.raw_year, + "normalized_query": NameMatcher.normalize(imp_series.raw_series_name), + "threshold": cv_match_threshold, + "identity_conflicts": identity_conflicts, + "top_candidates": [], + "conflicting_files": [ + { + "file_name": imp_file.file_name, + "rejection_reason": dict(imp_file.diagnostics or {}).get("rejection_reason"), + } + for imp_file in trusted_identity_conflict_files + ], + } + imp_series.status = ImportSeriesStatus.NO_MATCH + imp_series.diagnostics = invalidation_diagnostics + clear_auto_cv_match_fields(imp_series) + elif ( + not duplicate_series + and imp_series.status == ImportSeriesStatus.MATCHED + and source_layout_review_files + ): + invalidation_diagnostics = { + **dict(imp_series.diagnostics or {}), + "kind": "source_layout_review", + "reason": "selected_layout_no_match", + "rejection_reason": _SOURCE_LAYOUT_REVIEW_MESSAGE, + "source_layout_review_files": len(source_layout_review_files), + "unmatched_files": [file.file_name for file in source_layout_review_files], + } + imp_series.status = ImportSeriesStatus.NO_MATCH + imp_series.diagnostics = invalidation_diagnostics + elif ( not duplicate_series and imp_series.cv_match_method not in _TRUSTED_SERIES_MATCH_METHODS and imp_series.status == ImportSeriesStatus.MATCHED diff --git a/src/pullbox/services/import_file_match_targets.py b/src/pullbox/services/import_file_match_targets.py index 442641df..fe09beb1 100644 --- a/src/pullbox/services/import_file_match_targets.py +++ b/src/pullbox/services/import_file_match_targets.py @@ -42,6 +42,7 @@ class FileMatchTargetIndex: """Issue lookup maps used while matching imported files.""" cv_id_map: dict[int, FileMatchTargetEntry] = field(default_factory=dict) + exact_number_map: dict[str, FileMatchTargetEntry] = field(default_factory=dict) number_map: dict[float, FileMatchTargetEntry] = field(default_factory=dict) synthetic_issue_types: dict[float, IssueType] = field(default_factory=dict) synthetic_issue_titles: dict[float, str | None] = field(default_factory=dict) @@ -52,7 +53,7 @@ class FileMatchTargetIndex: @property def has_targets(self) -> bool: """Return True when at least one target issue identity is available.""" - return bool(self.cv_id_map or self.number_map) + return bool(self.cv_id_map or self.exact_number_map or self.number_map) @dataclass(frozen=True, slots=True) @@ -68,6 +69,7 @@ async def load_file_match_target_index( duplicate_series: bool, metadata_provider: MetadataProvider | None, files: list[ImportedFile] | None = None, + series_file_count: int | None = None, ) -> FileMatchTargetIndex: """Load issue identity maps for a matched or duplicate imported series.""" target_index = FileMatchTargetIndex() @@ -79,12 +81,20 @@ async def load_file_match_target_index( .outerjoin(LibraryFile, LibraryFile.issue_id == Issue.id) .where(Issue.series_id == imp_series.series_id) ) + ambiguous_issue_numbers: set[float] = set() for issue, library_file_id in issues_result.all(): has_library_file = library_file_id is not None target_index.issue_entries.append((issue, has_library_file)) entry = (issue.id, issue.comicvine_id, has_library_file, issue, issue.title) if issue.comicvine_id is not None: target_index.cv_id_map[issue.comicvine_id] = entry + target_index.exact_number_map[issue.effective_issue_number_text] = entry + if issue.issue_number in ambiguous_issue_numbers: + continue + if issue.issue_number in target_index.number_map: + target_index.number_map.pop(issue.issue_number) + ambiguous_issue_numbers.add(issue.issue_number) + continue target_index.number_map[issue.issue_number] = entry return target_index @@ -120,7 +130,11 @@ async def load_file_match_target_index( f"(cv_id={imp_series.cv_id})." ), ) - placeholder = _provider_zero_issue_placeholder_target(imp_series, files or []) + placeholder = _provider_zero_issue_placeholder_target( + imp_series, + files or [], + series_file_count=series_file_count, + ) if not load_result.summaries and load_result.exhaustive and placeholder is not None: issue_number, issue_type, issue_title = placeholder target_index.number_map[issue_number] = (None, None, False, None, issue_title) @@ -538,9 +552,12 @@ def _should_full_fetch_requested_issue_numbers( def _provider_zero_issue_placeholder_target( imp_series: ImportedSeries, files: list[ImportedFile], + *, + series_file_count: int | None = None, ) -> tuple[float, IssueType, str | None] | None: """Return a synthetic issue target for exact one-shot/special volumes with no issues.""" - if imp_series.cv_issue_count != 0 or len(files) != 1: + effective_file_count = len(files) if series_file_count is None else series_file_count + if imp_series.cv_issue_count != 0 or effective_file_count != 1 or len(files) != 1: return None if not ( imp_series.cv_match_method == "exact_title_year" diff --git a/src/pullbox/services/import_file_matching.py b/src/pullbox/services/import_file_matching.py index 4e89d182..b5283bcd 100644 --- a/src/pullbox/services/import_file_matching.py +++ b/src/pullbox/services/import_file_matching.py @@ -2,13 +2,24 @@ from __future__ import annotations +import json +import tempfile import time from contextlib import suppress +from datetime import UTC, datetime from inspect import isawaitable, iscoroutine -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, NamedTuple +from sqlalchemy import String +from sqlalchemy import and_ as sa_and +from sqlalchemy import case as sa_case +from sqlalchemy import cast as sa_cast +from sqlalchemy import exists as sa_exists from sqlalchemy import func as sa_func +from sqlalchemy import literal as sa_literal +from sqlalchemy import or_ as sa_or from sqlalchemy import select as sa_select +from sqlalchemy.ext.asyncio import async_sessionmaker from pullbox.core.exceptions import ImportProviderDegradedError, JobPausedError from pullbox.models.import_job import ( @@ -19,8 +30,13 @@ ImportJobStatus, ImportSeriesStatus, ) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile +from pullbox.models.series import IssueCatalogState, Series from pullbox.schemas.import_job import ImportProgressEvent +from pullbox.services.import_duplicates import DuplicateMergeProfile from pullbox.services.import_file_conflicts import detect_cross_series_conflicts +from pullbox.services.import_file_issue_signals import candidate_issue_number from pullbox.services.import_file_match_candidates import ( build_file_match_target_context, reset_file_match_state, @@ -34,8 +50,13 @@ from pullbox.services.import_file_match_provider_errors import ( defer_file_matching_for_provider_error, ) -from pullbox.services.import_file_match_results import apply_file_match_series_summary +from pullbox.services.import_file_match_results import ( + FileMatchSeriesSummary, + apply_file_match_series_summary, +) from pullbox.services.import_file_match_targets import ( + FileMatchTargetIndex, + load_file_match_target_index, trusted_source_issue_identity_matches_target, ) from pullbox.services.import_file_matching_progress import ( @@ -47,14 +68,17 @@ ) from pullbox.services.import_progress_runtime import ( ScanReviewFileMatchProfile, - ScanReviewSeriesMatchProfile, + ScanReviewProgressPlan, current_item_payload, estimate_remaining_work_seconds, - scan_review_completed_weight, + scan_review_file_match_weight, + scan_review_file_target_weight, scan_review_progress_pct, - scan_review_progress_plan, ) -from pullbox.services.import_source_metadata import load_archive_entry_issue_hint_for_import_file +from pullbox.services.import_source_metadata import ( + import_file_has_deferred_archive_metadata, + load_archive_entry_issue_hint_for_import_file, +) from pullbox.services.import_workflow_state import ( SCAN_PROGRESS_FILE_MATCH_END, SCAN_PROGRESS_FILE_MATCH_START, @@ -62,20 +86,16 @@ ) if TYPE_CHECKING: - from collections.abc import Awaitable, Callable - from datetime import datetime + from collections.abc import AsyncIterator, Awaitable, Callable from sqlalchemy.ext.asyncio import AsyncSession from pullbox.core.source_metadata import SourceMetadata - from pullbox.models.series import Series from pullbox.providers.base import MetadataProvider - from pullbox.services.import_duplicates import DuplicateMergeProfile from pullbox.services.import_file_match_candidates import FileMatchCandidate from pullbox.services.import_file_match_outcomes import ( DuplicateTargetStateFunc as OutcomeDuplicateTargetStateFunc, ) - from pullbox.services.import_file_match_targets import FileMatchTargetIndex from pullbox.services.semantic_matching import SemanticMatchEngine IsDuplicateSeriesFunc = Callable[[ImportedSeries], bool] @@ -108,6 +128,1101 @@ WarningLogFunc = Callable[..., None] +_SERIES_PAGE_SIZE = 100 +_FILE_PAGE_SIZE = 250 +_PROFILE_PAGE_SIZE = 500 +_TARGET_COHORT_PAGE_SIZE = 100 +_SUMMARY_SAMPLE_LIMIT = 500 +_MAX_IDENTITY_COHORT_SIZE = _FILE_PAGE_SIZE * 2 +_MAX_TARGET_INDEX_ENTRIES = _FILE_PAGE_SIZE * 2 + + +class _ImportFileMatchingCohortLimitError(RuntimeError): + """Stop automatic matching before an unbounded identity cohort is retained.""" + + +class _FileMatchingProfile(NamedTuple): + id: int + file_count: int + issue_count: int | None + completed_file_count: int + target_completed: bool + + +class _FileMatchingPlan(NamedTuple): + series_count: int + total_file_phase_units: int + completed_file_phase_units: int + completed_file_match_weight: float + completed_target_series_ids: frozenset[int] + progress_plan: ScanReviewProgressPlan + + +class _FileTargetCohort(NamedTuple): + kind: str + value: str + first_file_id: int + file_count: int + + +class _CrossConflictCohort(NamedTuple): + target_kind: str + target_value: int + issue_kind: str + issue_value: str + first_file_id: int + file_count: int + + +def _eligible_import_series_filter() -> Any: + return sa_or( + sa_and( + ImportedSeries.status == ImportSeriesStatus.DUPLICATE, + ImportedSeries.series_id.is_not(None), + ), + sa_and( + ImportedSeries.status == ImportSeriesStatus.MATCHED, + sa_or( + ImportedSeries.series_id.is_not(None), + ImportedSeries.cv_id.is_not(None), + ), + ), + ) + + +async def _load_eligible_import_series_page( + session: AsyncSession, + *, + job_id: int, + after_id: int, + page_size: int, + series_ids: list[int] | None, +) -> list[ImportedSeries]: + query = sa_select(ImportedSeries).where( + ImportedSeries.import_job_id == job_id, + _eligible_import_series_filter(), + ImportedSeries.id > after_id, + ) + if series_ids: + query = query.where(ImportedSeries.id.in_(series_ids)) + result = await session.execute(query.order_by(ImportedSeries.id).limit(max(page_size, 1))) + return list(result.scalars()) + + +async def _iter_eligible_import_series( + session: AsyncSession, + *, + job_id: int, + series_ids: list[int] | None, + raise_if_cancelled: RaiseIfCancelledFunc, + page_size: int, + blocked_only_counts: dict[int, int] | None = None, +) -> AsyncIterator[tuple[int, ImportedSeries, bool]]: + after_id = 0 + series_index = 0 + while True: + await raise_if_cancelled(session, job_id) + page = await _load_eligible_import_series_page( + session, + job_id=job_id, + after_id=after_id, + page_size=page_size, + series_ids=series_ids, + ) + if not page: + break + if blocked_only_counts is not None: + blocked_only_counts.clear() + blocked_only_counts.update(await _blocked_only_summary_counts(session, page)) + for page_index, item in enumerate(page): + yield series_index, item, page_index == len(page) - 1 + series_index += 1 + after_id = page[-1].id + + +async def _blocked_only_summary_counts( + session: AsyncSession, series_page: list[ImportedSeries] +) -> dict[int, int]: + """Summarize simple safety-only groups once per bounded series page.""" + ids = [ + item.id + for item in series_page + if item.status == ImportSeriesStatus.MATCHED and item.series_id is None + ] + if not ids: + return {} + needs_detailed_review = sa_or( + ImportedFile.status != ImportedFileStatus.SAFETY_BLOCKED, + ImportedFile.diagnostics["kind"] + .as_string() + .in_(["metadata_conflict", "source_layout_review"]), + ) + rows = await session.execute( + sa_select(ImportedFile.import_series_id, sa_func.count(ImportedFile.id)) + .where(ImportedFile.import_series_id.in_(ids)) + .group_by(ImportedFile.import_series_id) + .having(sa_func.sum(sa_case((needs_detailed_review, 1), else_=0)) == 0) + ) + return {int(series_id): int(count) for series_id, count in rows} + + +def _review_summary_weight(file_count: int) -> float: + return 1.0 + max(file_count, 0) * 0.1 + + +async def _load_file_matching_profile_page( + session: AsyncSession, + *, + job_id: int, + after_id: int, + page_size: int, + series_ids: list[int] | None, +) -> list[_FileMatchingProfile]: + query = sa_select( + ImportedSeries.id, + ImportedSeries.files_total, + ImportedSeries.file_count, + ImportedSeries.cv_issue_count, + ).where( + ImportedSeries.import_job_id == job_id, + _eligible_import_series_filter(), + ImportedSeries.id > after_id, + ) + if series_ids: + query = query.where(ImportedSeries.id.in_(series_ids)) + result = await session.execute(query.order_by(ImportedSeries.id).limit(max(page_size, 1))) + series_rows = result.all() + if not series_rows: + return [] + + page_series_ids = [int(row.id) for row in series_rows] + status_result = await session.execute( + sa_select( + ImportedFile.import_series_id, + ImportedFile.status, + sa_func.count(ImportedFile.id), + ) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.import_series_id.in_(page_series_ids), + ) + .group_by(ImportedFile.import_series_id, ImportedFile.status) + ) + status_counts: dict[int, dict[ImportedFileStatus, int]] = {} + for import_series_id, status, count in status_result: + status_counts.setdefault(int(import_series_id), {})[status] = int(count) + + matching_resolved_statuses = { + ImportedFileStatus.MATCHED, + ImportedFileStatus.DUPLICATE_FILE, + ImportedFileStatus.ALREADY_OWNED, + ImportedFileStatus.CONFLICT, + ImportedFileStatus.NO_MATCH, + ImportedFileStatus.CONFIRMED, + ImportedFileStatus.IMPORTED, + } + return [ + _FileMatchingProfile( + id=int(row.id), + file_count=( + sum(status_counts.get(int(row.id), {}).values()) + or max(int(row.files_total or row.file_count or 0), 0) + ), + issue_count=row.cv_issue_count, + completed_file_count=sum( + count + for status, count in status_counts.get(int(row.id), {}).items() + if status not in {ImportedFileStatus.PENDING, ImportedFileStatus.SAFETY_APPROVED} + ), + target_completed=( + not any( + status_counts.get(int(row.id), {}).get(status, 0) + for status in { + ImportedFileStatus.PENDING, + ImportedFileStatus.SAFETY_APPROVED, + } + ) + or any( + status_counts.get(int(row.id), {}).get(status, 0) + for status in matching_resolved_statuses + ) + ), + ) + for row in series_rows + ] + + +async def _build_file_matching_plan( + session: AsyncSession, + job: ImportJob, + *, + series_ids: list[int] | None, + raise_if_cancelled: RaiseIfCancelledFunc, + page_size: int, +) -> _FileMatchingPlan: + series_count = 0 + total_file_phase_units = 0 + completed_file_phase_units = 0 + file_match_weight = 0.0 + completed_file_match_weight = 0.0 + summary_weight = 0.0 + completed_target_series_ids: set[int] = set() + after_id = 0 + while True: + await raise_if_cancelled(session, job.id) + profiles = await _load_file_matching_profile_page( + session, + job_id=job.id, + after_id=after_id, + page_size=page_size, + series_ids=series_ids, + ) + if not profiles: + break + for profile in profiles: + series_count += 1 + summary_weight += _review_summary_weight(profile.file_count) + total_file_phase_units += 1 + profile.file_count + file_profile = ScanReviewFileMatchProfile( + file_count=profile.file_count, + issue_count=profile.issue_count, + ) + target_weight = scan_review_file_target_weight(file_profile) + aggregate_weight = scan_review_file_match_weight(file_profile) + file_match_weight += aggregate_weight + completed_file_phase_units += profile.completed_file_count + completed_file_match_weight += profile.completed_file_count * ( + (aggregate_weight - target_weight) / max(profile.file_count, 1) + ) + if profile.target_completed: + completed_file_phase_units += 1 + completed_file_match_weight += target_weight + completed_target_series_ids.add(profile.id) + after_id = profiles[-1].id + + # Replan only remaining work at the persisted phase boundary. Resolved files + # need no matching, but their summaries and cross-series review still do. + remaining_weight = max(file_match_weight - completed_file_match_weight, 0.0) + remaining_weight += summary_weight + max(1.0, series_count * 0.1) + progress_start = max( + 35, + min(98, int((job.progress_snapshot or {}).get("progress", SCAN_PROGRESS_FILE_MATCH_START))), + ) + return _FileMatchingPlan( + series_count=series_count, + total_file_phase_units=total_file_phase_units, + completed_file_phase_units=completed_file_phase_units, + completed_file_match_weight=0.0, + completed_target_series_ids=frozenset(completed_target_series_ids), + progress_plan=ScanReviewProgressPlan( + analysis_weights=(), + series_match_weights=(), + file_match_weights=(remaining_weight,), + progress_start=progress_start, + progress_end=98, + ), + ) + + +async def _load_eligible_import_file_page( + session: AsyncSession, + *, + import_series_id: int, + after_id: int, + page_size: int, +) -> list[ImportedFile]: + result = await session.execute( + sa_select(ImportedFile) + .where( + ImportedFile.import_series_id == import_series_id, + ImportedFile.status.in_( + [ImportedFileStatus.PENDING, ImportedFileStatus.SAFETY_APPROVED] + ), + ImportedFile.id > after_id, + ) + .order_by(ImportedFile.id) + .limit(max(page_size, 1)) + ) + return list(result.scalars()) + + +async def _load_import_series_file_page( + session: AsyncSession, + *, + import_series_id: int, + after_id: int, + page_size: int, +) -> list[ImportedFile]: + result = await session.execute( + sa_select(ImportedFile) + .where( + ImportedFile.import_series_id == import_series_id, + ImportedFile.id > after_id, + ) + .order_by(ImportedFile.id) + .limit(max(page_size, 1)) + ) + return list(result.scalars()) + + +_FILE_TARGET_COHORT_KINDS = ("issue_id", "issue_cv_id", "parsed_issue") + + +def _file_target_identity_spec(kind: str) -> tuple[Any, Any]: + if kind == "issue_id": + return ImportedFile.matched_issue_id, ImportedFile.matched_issue_id.is_not(None) + if kind == "issue_cv_id": + return ( + ImportedFile.matched_issue_cv_id, + sa_and( + ImportedFile.matched_issue_id.is_(None), + ImportedFile.matched_issue_cv_id.is_not(None), + ), + ) + if kind == "parsed_issue": + return ( + ImportedFile.parsed_issue_number, + sa_and( + ImportedFile.matched_issue_id.is_(None), + ImportedFile.matched_issue_cv_id.is_(None), + ImportedFile.parsed_issue_number.is_not(None), + ), + ) + raise ValueError(f"Unsupported file target cohort kind: {kind}") + + +def _file_target_cohort_query( + *, + job_id: int, + import_series_id: int, + kind: str, + after_value: str | None, + page_size: int | None, +) -> Any: + identity_column, identity_filter = _file_target_identity_spec(kind) + cursor_filter = None + if after_value is not None: + cursor_value: int | float = ( + int(after_value) if kind != "parsed_issue" else float(after_value) + ) + cursor_filter = identity_column > cursor_value + query = ( + sa_select( + sa_literal(kind).label("kind"), + sa_cast(identity_column, String).label("value"), + sa_func.min(ImportedFile.id).label("first_file_id"), + sa_func.count(ImportedFile.id).label("file_count"), + ) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.import_series_id == import_series_id, + identity_filter, + cursor_filter if cursor_filter is not None else sa_literal(True), + ) + .group_by(identity_column) + .order_by(identity_column) + ) + return query.limit(max(page_size, 1)) if page_size is not None else query + + +async def _load_file_target_cohort_page( + session: AsyncSession, + *, + job_id: int, + import_series_id: int, + kind: str, + after_value: str | None, + page_size: int, +) -> list[_FileTargetCohort]: + result = await session.execute( + _file_target_cohort_query( + job_id=job_id, + import_series_id=import_series_id, + kind=kind, + after_value=after_value, + page_size=page_size, + ) + ) + return [ + _FileTargetCohort( + kind=str(row.kind), + value=str(row.value), + first_file_id=int(row.first_file_id), + file_count=int(row.file_count), + ) + for row in result + ] + + +async def _spool_grouped_cohort_rows( + session: AsyncSession, + *, + job_id: int, + query: Any, + batch_size: int, + scan_name: str, + row_payload: Callable[[Any], list[str | int]], + raise_if_cancelled: RaiseIfCancelledFunc, +) -> Any: + """Execute one grouped scan and spool bounded row batches before writer commits.""" + size = max(batch_size, 1) + spool = tempfile.SpooledTemporaryFile( # noqa: SIM115 - ownership passes to iterator + max_size=1 << 20, + mode="w+t", + encoding="utf-8", + newline="\n", + ) + + def write_rows(rows: Any) -> None: + for row in rows: + spool.write(json.dumps(row_payload(row), separators=(",", ":"))) + spool.write("\n") + + try: + await session.flush() + await session.commit() + await raise_if_cancelled(session, job_id) + bind = session.bind + bind_url = str(bind.sync_engine.url) if bind is not None else "" + execution_options = {"pullbox_cohort_scan": scan_name} + if bind is None or ":memory:" in bind_url: + result = await session.execute(query.execution_options(**execution_options)) + for rows in result.partitions(size): + await raise_if_cancelled(session, job_id) + write_rows(rows) + else: + reader_factory = async_sessionmaker(bind=bind, expire_on_commit=False) + async with reader_factory() as reader_session: + stream_result = await reader_session.stream( + query.execution_options( + yield_per=size, + **execution_options, + ) + ) + try: + while True: + rows = await stream_result.fetchmany(size) + if not rows: + break + await raise_if_cancelled(session, job_id) + write_rows(rows) + finally: + await stream_result.close() + await raise_if_cancelled(session, job_id) + spool.seek(0) + return spool + except BaseException: + spool.close() + raise + + +async def _iter_file_target_cohort_batches( + session: AsyncSession, + *, + job_id: int, + import_series_id: int, + kind: str, + batch_size: int, + raise_if_cancelled: RaiseIfCancelledFunc, +) -> AsyncIterator[list[_FileTargetCohort]]: + """Yield bounded cohort descriptors from one grouped database execution.""" + spool = await _spool_grouped_cohort_rows( + session, + job_id=job_id, + query=_file_target_cohort_query( + job_id=job_id, + import_series_id=import_series_id, + kind=kind, + after_value=None, + page_size=None, + ), + batch_size=batch_size, + scan_name="file_target", + row_payload=lambda row: [ + str(row.kind), + str(row.value), + int(row.first_file_id), + int(row.file_count), + ], + raise_if_cancelled=raise_if_cancelled, + ) + try: + while True: + await raise_if_cancelled(session, job_id) + batch: list[_FileTargetCohort] = [] + while len(batch) < max(batch_size, 1): + line = spool.readline() + if not line: + break + kind_value, value, first_file_id, file_count = json.loads(line) + batch.append( + _FileTargetCohort( + kind=str(kind_value), + value=str(value), + first_file_id=int(first_file_id), + file_count=int(file_count), + ) + ) + if not batch: + break + yield batch + finally: + spool.close() + + +async def _load_file_target_cohort( + session: AsyncSession, + *, + job_id: int, + import_series_id: int, + cohort: _FileTargetCohort, +) -> list[ImportedFile]: + if cohort.file_count > _MAX_IDENTITY_COHORT_SIZE: + raise _ImportFileMatchingCohortLimitError( + "Import file identity cohort exceeds the bounded automatic-matching limit " + f"({_MAX_IDENTITY_COHORT_SIZE} files; kind={cohort.kind})." + ) + if cohort.kind == "issue_id": + identity_filter = ImportedFile.matched_issue_id == int(cohort.value) + elif cohort.kind == "issue_cv_id": + identity_filter = sa_and( + ImportedFile.matched_issue_id.is_(None), + ImportedFile.matched_issue_cv_id == int(cohort.value), + ) + else: + identity_filter = sa_and( + ImportedFile.matched_issue_id.is_(None), + ImportedFile.matched_issue_cv_id.is_(None), + ImportedFile.parsed_issue_number == float(cohort.value), + ) + result = await session.execute( + sa_select(ImportedFile) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.import_series_id == import_series_id, + identity_filter, + ) + .order_by(ImportedFile.id) + .limit(_MAX_IDENTITY_COHORT_SIZE) + ) + return list(result.scalars()) + + +async def _finalize_import_series_file_groups( + session: AsyncSession, + job: ImportJob, + imp_series: ImportedSeries, + *, + duplicate_group_counter: int, + conflict_group_counter: int, + detect_duplicate_copies: DetectDuplicateCopiesFunc, + detect_conflicts: DetectConflictsFunc, + log_event: LogEventFunc, + raise_if_cancelled: RaiseIfCancelledFunc, +) -> tuple[int, int]: + for kind in _FILE_TARGET_COHORT_KINDS: + async for cohorts in _iter_file_target_cohort_batches( + session, + job_id=job.id, + import_series_id=imp_series.id, + kind=kind, + batch_size=_TARGET_COHORT_PAGE_SIZE, + raise_if_cancelled=raise_if_cancelled, + ): + for cohort in cohorts: + await raise_if_cancelled(session, job.id) + if cohort.file_count < 2: + continue + files = await _load_file_target_cohort( + session, + job_id=job.id, + import_series_id=imp_series.id, + cohort=cohort, + ) + if not files: + continue + ( + _duplicate_count, + duplicate_group_counter, + _duplicate_groups, + ) = await detect_duplicate_copies( + session, + job, + imp_series, + files, + duplicate_group_counter, + ) + _conflict_count, conflict_group_counter, conflict_groups = detect_conflicts( + files, + conflict_group_counter, + ) + for group_details in conflict_groups: + await log_event( + session, + job.id, + "DEBUG", + "import_file_conflict_detail", + message=( + f"Conflict group {group_details['conflict_group_id']} in " + f"{imp_series.raw_series_name}" + ), + series=imp_series.raw_series_name, + diagnostics=group_details, + ) + await session.flush() + await session.commit() + return duplicate_group_counter, conflict_group_counter + + +async def _summarize_import_series_files( + session: AsyncSession, + imp_series: ImportedSeries, + *, + duplicate_series: bool, + duplicate_merge_profile: DuplicateMergeProfile | None, + cv_match_threshold: float, + raise_if_cancelled: RaiseIfCancelledFunc, + job_id: int, +) -> FileMatchSeriesSummary: + status_counts: dict[ImportedFileStatus, int] = {} + sample: list[ImportedFile] = [] + representative_by_status: dict[ImportedFileStatus, ImportedFile] = {} + special_representatives: dict[tuple[str, bool, bool], ImportedFile] = {} + source_layout_review_count = 0 + source_layout_review_names: list[str] = [] + metadata_conflict_count = 0 + trusted_identity_conflict_count = 0 + trusted_identity_conflicts: list[dict[str, object]] = [] + trusted_identity_conflict_files: list[dict[str, object]] = [] + after_id = 0 + total_files = 0 + while True: + await raise_if_cancelled(session, job_id) + page = await _load_import_series_file_page( + session, + import_series_id=imp_series.id, + after_id=after_id, + page_size=_FILE_PAGE_SIZE, + ) + if not page: + break + for imp_file in page: + total_files += 1 + status_counts[imp_file.status] = status_counts.get(imp_file.status, 0) + 1 + representative_by_status.setdefault(imp_file.status, imp_file) + diagnostics = dict(imp_file.diagnostics or {}) + kind = str(diagnostics.get("kind") or "") + conflict_type = str(diagnostics.get("conflict_type") or "") + if kind in {"metadata_conflict", "source_layout_review"}: + preserve_series_match = bool(diagnostics.get("preserve_series_match")) + special_representatives.setdefault( + ( + kind, + conflict_type == "trusted_source_identity_conflict", + preserve_series_match, + ), + imp_file, + ) + if kind == "source_layout_review": + source_layout_review_count += 1 + if len(source_layout_review_names) < _SUMMARY_SAMPLE_LIMIT: + source_layout_review_names.append(imp_file.file_name) + if kind == "metadata_conflict": + metadata_conflict_count += 1 + if kind == "metadata_conflict" and conflict_type == ( + "trusted_source_identity_conflict" + ): + trusted_identity_conflict_count += 1 + if len(trusted_identity_conflict_files) < _SUMMARY_SAMPLE_LIMIT: + trusted_identity_conflict_files.append( + { + "file_name": imp_file.file_name, + "rejection_reason": diagnostics.get("rejection_reason"), + } + ) + raw_conflicts = diagnostics.get("identity_conflicts") + if isinstance(raw_conflicts, list): + for conflict_item in raw_conflicts: + if ( + isinstance(conflict_item, dict) + and conflict_item not in trusted_identity_conflicts + and len(trusted_identity_conflicts) < _SUMMARY_SAMPLE_LIMIT + ): + trusted_identity_conflicts.append(dict(conflict_item)) + if len(sample) < _SUMMARY_SAMPLE_LIMIT: + sample.append(imp_file) + after_id = page[-1].id + + if total_files > len(sample): + required = [*representative_by_status.values(), *special_representatives.values()] + required_by_id = {item.id: item for item in required} + sample = [ + *required_by_id.values(), + *(item for item in sample if item.id not in required_by_id), + ][:_SUMMARY_SAMPLE_LIMIT] + + if sample: + summary = apply_file_match_series_summary( + imp_series, + sample, + duplicate_series=duplicate_series, + duplicate_merge_profile=duplicate_merge_profile, + cv_match_threshold=cv_match_threshold, + ) + else: + summary = FileMatchSeriesSummary(0, 0, 0, 0, 0, 0) + + matched = status_counts.get(ImportedFileStatus.MATCHED, 0) + status_counts.get( + ImportedFileStatus.CONFIRMED, + 0, + ) + duplicate = status_counts.get(ImportedFileStatus.DUPLICATE_FILE, 0) + already_owned = status_counts.get(ImportedFileStatus.ALREADY_OWNED, 0) + no_match = status_counts.get(ImportedFileStatus.NO_MATCH, 0) + conflict = status_counts.get(ImportedFileStatus.CONFLICT, 0) + imp_series.files_total = total_files + imp_series.files_matched = matched + imp_series.files_duplicate = duplicate + imp_series.files_already_owned = already_owned + imp_series.files_no_match = no_match + imp_series.files_conflict = conflict + if duplicate_series: + diagnostics = dict(imp_series.diagnostics or {}) + actionable = bool( + (duplicate_merge_profile.actionable if duplicate_merge_profile else False) + or matched + or conflict + ) + diagnostics.update( + { + "actionable_duplicate_merge": actionable, + "has_importable_files": matched > 0, + "importable_files": matched, + "duplicate_files": duplicate, + "already_owned_files": already_owned, + "no_match_files": no_match, + "conflict_files": conflict, + } + ) + imp_series.diagnostics = diagnostics + invalidation_diagnostics = ( + dict(summary.invalidation_diagnostics) + if summary.invalidation_diagnostics is not None + else None + ) + if invalidation_diagnostics is not None: + reason = invalidation_diagnostics.get("reason") + if reason == "selected_layout_no_match": + invalidation_diagnostics.update( + { + "source_layout_review_files": source_layout_review_count, + "unmatched_files": source_layout_review_names, + "unmatched_files_truncated": ( + source_layout_review_count > len(source_layout_review_names) + ), + } + ) + elif reason == "trusted_source_identity_conflict": + invalidation_diagnostics.update( + { + "identity_conflict_files": trusted_identity_conflict_count, + "identity_conflicts": trusted_identity_conflicts, + "conflicting_files": trusted_identity_conflict_files, + "conflicting_files_truncated": ( + trusted_identity_conflict_count > len(trusted_identity_conflict_files) + ), + } + ) + elif reason == "file_metadata_conflict": + invalidation_diagnostics["metadata_conflict_files"] = metadata_conflict_count + imp_series.diagnostics = invalidation_diagnostics + return FileMatchSeriesSummary( + found=total_files, + matched=matched, + duplicate=duplicate, + already_owned=already_owned, + no_match=no_match, + conflict=conflict, + series_invalidated=summary.series_invalidated, + invalidation_diagnostics=invalidation_diagnostics, + ) + + +async def _count_import_series_files( + session: AsyncSession, + *, + import_series_id: int, + eligible_only: bool, +) -> int: + query = sa_select(sa_func.count(ImportedFile.id)).where( + ImportedFile.import_series_id == import_series_id + ) + if eligible_only: + query = query.where( + ImportedFile.status.in_( + [ImportedFileStatus.PENDING, ImportedFileStatus.SAFETY_APPROVED] + ) + ) + count = await session.scalar(query) + return int(count or 0) + + +def _stable_series_file_count( + imp_series: ImportedSeries, + *, + persisted_file_count: int, +) -> int: + """Retain the pre-split series size across cancellation and resume.""" + raw_pre_split_count = (imp_series.diagnostics or {}).get("pre_split_file_count") + try: + pre_split_count = int(raw_pre_split_count or 0) + except (TypeError, ValueError): + pre_split_count = 0 + return max( + persisted_file_count, + int(imp_series.files_total or 0), + pre_split_count, + ) + + +async def _load_existing_series_page_target_index( + session: AsyncSession, + imp_series: ImportedSeries, + files: list[ImportedFile], +) -> FileMatchTargetIndex: + """Load only canonical issues addressable by one bounded imported-file page.""" + target_index = FileMatchTargetIndex() + if imp_series.series_id is None: + return target_index + + target_index.existing_series = await session.get(Series, imp_series.series_id) + requested_cv_ids = { + int(imp_file.comicvine_issue_id) + for imp_file in files + if imp_file.comicvine_issue_id is not None + } + requested_issue_numbers = { + float(issue_number) + for imp_file in files + if imp_file.comicvine_issue_id is None + and (issue_number := candidate_issue_number(imp_file)) is not None + } + identity_filters: list[Any] = [] + if requested_cv_ids: + identity_filters.append(Issue.comicvine_id.in_(sorted(requested_cv_ids))) + if requested_issue_numbers: + identity_filters.append(Issue.issue_number.in_(sorted(requested_issue_numbers))) + if not identity_filters: + return target_index + + has_library_file = sa_exists(sa_select(LibraryFile.id).where(LibraryFile.issue_id == Issue.id)) + result = await session.execute( + sa_select(Issue, has_library_file.label("has_library_file")) + .where( + Issue.series_id == imp_series.series_id, + sa_or(*identity_filters), + ) + .order_by(Issue.id) + .limit(_MAX_TARGET_INDEX_ENTRIES + 1) + ) + target_rows = result.all() + if len(target_rows) > _MAX_TARGET_INDEX_ENTRIES: + raise _ImportFileMatchingCohortLimitError( + "Issue target index exceeds the bounded automatic-matching limit " + f"({_MAX_TARGET_INDEX_ENTRIES} entries)." + ) + + ambiguous_issue_numbers: set[float] = set() + for issue, owned in target_rows: + entry = (issue.id, issue.comicvine_id, bool(owned), issue, issue.title) + if issue.comicvine_id is not None: + target_index.cv_id_map[issue.comicvine_id] = entry + target_index.exact_number_map[issue.effective_issue_number_text] = entry + if issue.issue_number in ambiguous_issue_numbers: + continue + if issue.issue_number in target_index.number_map: + target_index.number_map.pop(issue.issue_number) + ambiguous_issue_numbers.add(issue.issue_number) + continue + target_index.number_map[issue.issue_number] = entry + return target_index + + +def _ensure_bounded_target_index(target_index: FileMatchTargetIndex) -> None: + unique_target_keys: set[tuple[int | None, int | None, float | None]] = { + ( + entry[0], + entry[1], + issue_number if entry[0] is None and entry[1] is None else None, + ) + for issue_number, entry in target_index.number_map.items() + } + unique_target_keys.update( + (entry[0], entry[1], None) for entry in target_index.cv_id_map.values() + ) + unique_target_keys.update( + (entry[0], entry[1], None) for entry in target_index.exact_number_map.values() + ) + if len(unique_target_keys) > _MAX_TARGET_INDEX_ENTRIES: + raise _ImportFileMatchingCohortLimitError( + "Issue target index exceeds the bounded automatic-matching limit " + f"({_MAX_TARGET_INDEX_ENTRIES} entries)." + ) + + +async def _load_file_page_target_index( + session: AsyncSession, + job: ImportJob, + imp_series: ImportedSeries, + files: list[ImportedFile], + *, + series_file_count: int, + duplicate_series: bool, + metadata_provider: MetadataProvider | None, + series_idx: int, + total_series: int, + completed_units: int, + progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None, + emit_file_matching_progress: Callable[..., Awaitable[None]], + raise_if_cancelled: RaiseIfCancelledFunc, +) -> FileMatchTargetIndex: + if imp_series.series_id is not None: + if progress_callback is not None: + await emit_file_matching_progress( + imp_series, + completed_units, + message=( + f"Loading issue targets for {imp_series.raw_series_name} " + f"(series {series_idx + 1}/{total_series})..." + ), + current_item_stage="file_matching", + current_item_progress_pct=5, + current_work_unit_progress_pct=0, + ) + target_index = await _load_existing_series_page_target_index( + session, + imp_series, + files, + ) + else: + target_index = await load_file_match_target_index_with_progress( + session=session, + job_id=job.id, + item=imp_series, + files_to_match=files, + series_file_count=series_file_count, + duplicate_series=duplicate_series, + metadata_provider=metadata_provider, + series_idx=series_idx, + total_series=total_series, + completed_units=completed_units, + progress_callback=progress_callback, + emit_file_matching_progress=emit_file_matching_progress, + raise_if_cancelled=raise_if_cancelled, + ) + _ensure_bounded_target_index(target_index) + return target_index + + +async def _reload_file_page_target_index_after_deferred_identity( + session: AsyncSession, + imp_series: ImportedSeries, + files: list[ImportedFile], + *, + series_file_count: int, + duplicate_series: bool, + metadata_provider: MetadataProvider | None, +) -> FileMatchTargetIndex: + """Reload a bounded page index after ComicInfo reveals a new exact issue ID.""" + if imp_series.series_id is not None: + target_index = await _load_existing_series_page_target_index( + session, + imp_series, + files, + ) + else: + target_index = await load_file_match_target_index( + session, + imp_series, + duplicate_series=duplicate_series, + metadata_provider=metadata_provider, + files=files, + series_file_count=series_file_count, + ) + _ensure_bounded_target_index(target_index) + return target_index + + +async def _build_bounded_duplicate_merge_profile( + session: AsyncSession, + imp_series: ImportedSeries, + *, + incoming_file_count: int, + build_duplicate_merge_profile: BuildDuplicateMergeProfileFunc, +) -> DuplicateMergeProfile | None: + """Build the exact duplicate summary without retaining every canonical issue.""" + if imp_series.series_id is None: + return None + existing_series = await session.get(Series, imp_series.series_id) + owned_issue = sa_case((LibraryFile.id.is_not(None), Issue.id), else_=None) + result = await session.execute( + sa_select( + sa_func.count(sa_func.distinct(Issue.id)), + sa_func.count(sa_func.distinct(owned_issue)), + ) + .select_from(Issue) + .outerjoin(LibraryFile, LibraryFile.issue_id == Issue.id) + .where(Issue.series_id == imp_series.series_id) + ) + existing_issue_count, owned_issue_count = result.one() + existing_count = int(existing_issue_count or 0) + owned_count = int(owned_issue_count or 0) + + if existing_count <= 1: + issue_entries: list[tuple[Issue, bool]] = [] + if existing_count: + single_result = await session.execute( + sa_select( + Issue, + sa_exists( + sa_select(LibraryFile.id).where(LibraryFile.issue_id == Issue.id) + ).label("has_library_file"), + ) + .where(Issue.series_id == imp_series.series_id) + .limit(1) + ) + issue, owned = single_result.one() + issue_entries.append((issue, bool(owned))) + return build_duplicate_merge_profile( + existing_series, + issue_entries, + incoming_file_count=incoming_file_count, + ) + + catalog_state = ( + existing_series.issue_catalog_state + if existing_series is not None + else IssueCatalogState.COMPLETE + ) + if catalog_state is None: + catalog_state = IssueCatalogState.COMPLETE + elif not isinstance(catalog_state, IssueCatalogState): + catalog_state = IssueCatalogState(str(catalog_state).lower()) + expected_issue_count = int(existing_series.issue_count or 0) if existing_series else 0 + catalog_proves_full_ownership = not ( + catalog_state != IssueCatalogState.COMPLETE and expected_issue_count > 0 + ) and (expected_issue_count <= 0 or existing_count >= expected_issue_count) + fully_owned = bool( + existing_count > 0 and owned_count == existing_count and catalog_proves_full_ownership + ) + return DuplicateMergeProfile( + actionable=owned_count < existing_count, + fully_owned=fully_owned, + existing_issue_count=existing_count, + owned_issue_count=owned_count, + ) + + async def run_import_file_matching( session: AsyncSession, job: ImportJob, @@ -134,28 +1249,19 @@ async def run_import_file_matching( log_warning: WarningLogFunc, series_ids: list[int] | None = None, progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None = None, + rebuild_cross_conflicts: bool = True, ) -> None: """Match individual imported files to issue targets for review.""" started_at = time.monotonic() - persist_batch_size = 10 + work_started_at = datetime.now(UTC) last_checkpoint_at = time.monotonic() - series_query = sa_select(ImportedSeries).where( - ImportedSeries.import_job_id == job.id, - ImportedSeries.status.in_([ImportSeriesStatus.MATCHED, ImportSeriesStatus.DUPLICATE]), + matching_plan = await _build_file_matching_plan( + session, + job, + series_ids=series_ids, + raise_if_cancelled=raise_if_cancelled, + page_size=_PROFILE_PAGE_SIZE, ) - if series_ids: - series_query = series_query.where(ImportedSeries.id.in_(series_ids)) - items_result = await session.execute(series_query) - candidate_series = list(items_result.scalars().all()) - matched_series_list = [ - series_item - for series_item in candidate_series - if is_duplicate_series(series_item) - or ( - series_item.status == ImportSeriesStatus.MATCHED - and (series_item.series_id is not None or series_item.cv_id is not None) - ) - ] total_found = 0 total_matched = 0 @@ -174,43 +1280,27 @@ async def run_import_file_matching( deferred_metadata_loads = 0 series_processed = 0 files_processed = 0 - if series_ids: - counter_result = await session.execute( - sa_select( - sa_func.max(ImportedFile.conflict_group_id), - sa_func.max(ImportedFile.duplicate_group_id), - ).where(ImportedFile.import_job_id == job.id) - ) - max_conflict_group_id, max_duplicate_group_id = counter_result.one() - conflict_group_counter = int(max_conflict_group_id or 0) - duplicate_group_counter = int(max_duplicate_group_id or 0) - total_file_phase_units = len(matched_series_list) + sum( - max(int(series_item.files_total or series_item.file_count or 0), 0) - for series_item in matched_series_list - ) - series_match_profile_count = max( - int(job.series_matched or 0) + int(job.series_no_match or 0), - len(matched_series_list), - ) - progress_plan = scan_review_progress_plan( - analysis_series_count=max(int(job.series_found or 0), series_match_profile_count), - series_match_profiles=[ - ScanReviewSeriesMatchProfile(direct_match=False) - for _idx in range(series_match_profile_count) - ], - file_match_profiles=[ - ScanReviewFileMatchProfile( - file_count=max(int(series_item.files_total or series_item.file_count or 0), 0), - issue_count=series_item.cv_issue_count, - ) - for series_item in matched_series_list - ], + counter_result = await session.execute( + sa_select( + sa_func.max(ImportedFile.conflict_group_id), + sa_func.max(ImportedFile.duplicate_group_id), + ).where(ImportedFile.import_job_id == job.id) ) - completed_file_phase_units = 0 - total_series = max(len(matched_series_list), 1) + max_conflict_group_id, max_duplicate_group_id = counter_result.one() + conflict_group_counter = int(max_conflict_group_id or 0) + duplicate_group_counter = int(max_duplicate_group_id or 0) + total_file_phase_units = matching_plan.total_file_phase_units + progress_plan = matching_plan.progress_plan + completed_file_phase_units = matching_plan.completed_file_phase_units + completed_file_match_weight = matching_plan.completed_file_match_weight + active_target_weight = 0.0 + file_match_unit_weight = scan_review_file_match_weight( + ScanReviewFileMatchProfile(file_count=1, issue_count=0) + ) - scan_review_file_target_weight(ScanReviewFileMatchProfile(file_count=1, issue_count=0)) + total_series = max(matching_plan.series_count, 1) runtime_revision_state: dict[str, int] = {"value": int(job.progress_revision or 0)} - emit_file_matching_progress = build_file_matching_progress_emitter( + emit_weighted_file_matching_progress = build_file_matching_progress_emitter( session=session, job=job, progress_callback=progress_callback, @@ -225,81 +1315,48 @@ async def run_import_file_matching( scan_review_plan=progress_plan, phase_start=SCAN_PROGRESS_FILE_MATCH_START, phase_end=SCAN_PROGRESS_FILE_MATCH_END, + work_started_at=work_started_at, ) - for series_idx, imp_series in enumerate(matched_series_list): - await raise_if_cancelled(session, job.id) - duplicate_series = is_duplicate_series(imp_series) - - files_result = await session.execute( - sa_select(ImportedFile).where( - ImportedFile.import_series_id == imp_series.id, - ImportedFile.status.in_( - [ImportedFileStatus.PENDING, ImportedFileStatus.SAFETY_APPROVED] - ), - ) + async def emit_file_matching_progress( + item: ImportedSeries, + _completed_units: int, + **kwargs: Any, + ) -> None: + current_progress = int(kwargs.get("current_work_unit_progress_pct") or 0) + effective_file_weight = completed_file_match_weight + ( + active_target_weight * max(min(current_progress, 100), 0) / 100 ) - files = list(files_result.scalars().all()) - if not files: - continue + overall_file_progress = ( + effective_file_weight / max(progress_plan.file_match_weight, 1.0) + ) * 100 + kwargs["current_work_unit_progress_pct"] = overall_file_progress + await emit_weighted_file_matching_progress(item, 0, **kwargs) - target_index_started_at = time.monotonic() - try: - target_index = await load_file_match_target_index_with_progress( - session=session, - job_id=job.id, - item=imp_series, - files_to_match=files, - duplicate_series=duplicate_series, - metadata_provider=metadata_provider, - series_idx=series_idx, - total_series=total_series, - completed_units=completed_file_phase_units, - progress_callback=progress_callback, - emit_file_matching_progress=emit_file_matching_progress, - raise_if_cancelled=raise_if_cancelled, - ) - target_index_duration_ms += (time.monotonic() - target_index_started_at) * 1000 - except ImportProviderDegradedError as exc: - target_index_duration_ms += (time.monotonic() - target_index_started_at) * 1000 - deferred_count += 1 - deferred_titles.append(imp_series.raw_series_name) - await defer_file_matching_for_provider_error( - session=session, - job_id=job.id, - imp_series=imp_series, - exc=exc, - log_event=log_event, - ) - continue - except Exception: - if imp_series.cv_id is not None: - log_warning( - "file_matching_cv_fetch_failed", - series=imp_series.raw_series_name, - cv_id=imp_series.cv_id, - ) - continue - raise + async def process_file_page( + imp_series: ImportedSeries, + files: list[ImportedFile], + *, + target_index: FileMatchTargetIndex, + duplicate_series: bool, + duplicate_merge_profile: Any, + series_high_confidence: bool, + file_index_offset: int, + total_file_count: int, + ) -> tuple[list[ImportedFile], list[int]]: + nonlocal completed_file_phase_units + nonlocal completed_file_match_weight + nonlocal deferred_metadata_duration_ms + nonlocal deferred_metadata_loads + nonlocal file_evaluation_duration_ms + nonlocal files_processed - completed_file_phase_units += 1 - if not target_index.has_targets: - if not can_mark_missing_issue_targets(imp_series, metadata_provider): - continue + if not target_index.has_targets and can_mark_missing_issue_targets( + imp_series, + metadata_provider, + ): mark_files_missing_provider_targets(imp_series, files) - series_high_confidence = ( - imp_series.cv_match_score is not None and imp_series.cv_match_score >= 0.90 - ) - duplicate_merge_profile = ( - build_duplicate_merge_profile( - target_index.existing_series, - target_index.issue_entries, - incoming_file_count=len(files), - ) - if duplicate_series - else None - ) files, created_split_series_ids = await _split_explicit_issue_series_mismatches( session, job, @@ -310,25 +1367,52 @@ async def run_import_file_matching( target_series=target_index.existing_series, log_event=log_event, ) - split_series_ids.update(created_split_series_ids) if not files: - continue + return files, created_split_series_ids await emit_file_matching_progress( imp_series, completed_file_phase_units, message=( f"Matching files to issues for {imp_series.raw_series_name} " - f"({len(files)} file{'s' if len(files) != 1 else ''})..." + f"({total_file_count} file{'s' if total_file_count != 1 else ''})..." ), current_item_progress_pct=50, current_work_unit_progress_pct=0, ) - series_processed += 1 - for file_idx, imp_file in enumerate(files, start=1): + for page_file_index, imp_file in enumerate(files, start=1): + if page_file_index > 1 and page_file_index % 25 == 0: + await raise_if_cancelled(session, job.id) files_processed += 1 reset_file_match_state(imp_file) file_metadata = source_metadata_for_import_file(imp_series, imp_file) + if ( + imp_series.cv_match_method == "mylar3_cv_id" + and load_deferred_source_metadata_for_import_file is not None + and _file_has_deferred_archive_metadata(imp_file) + ): + deferred_metadata_loads += 1 + deferred_metadata_started_at = time.monotonic() + file_metadata = await load_deferred_source_metadata_for_import_file( + imp_series, + imp_file, + ) + deferred_metadata_duration_ms += ( + time.monotonic() - deferred_metadata_started_at + ) * 1000 + _persist_deferred_source_evidence(imp_file, file_metadata) + if ( + imp_file.comicvine_issue_id is not None + and imp_file.comicvine_issue_id not in target_index.cv_id_map + ): + target_index = await _reload_file_page_target_index_after_deferred_identity( + session, + imp_series, + files, + series_file_count=stable_series_file_count, + duplicate_series=duplicate_series, + metadata_provider=metadata_provider, + ) file_evaluation_started_at = time.monotonic() match_candidate, metadata_conflict = _evaluate_file_match_candidate( imp_series=imp_series, @@ -367,102 +1451,423 @@ async def run_import_file_matching( deferred_metadata_started_at = time.monotonic() file_metadata = await load_deferred_source_metadata_for_import_file( imp_series, - imp_file, + imp_file, + ) + deferred_metadata_duration_ms += ( + time.monotonic() - deferred_metadata_started_at + ) * 1000 + _persist_deferred_source_evidence(imp_file, file_metadata) + if ( + imp_file.comicvine_issue_id is not None + and imp_file.comicvine_issue_id not in target_index.cv_id_map + ): + target_index = await _reload_file_page_target_index_after_deferred_identity( + session, + imp_series, + files, + series_file_count=stable_series_file_count, + duplicate_series=duplicate_series, + metadata_provider=metadata_provider, + ) + file_evaluation_started_at = time.monotonic() + match_candidate, metadata_conflict = _evaluate_file_match_candidate( + imp_series=imp_series, + imp_file=imp_file, + target_index=target_index, + target_series=target_index.existing_series, + file_metadata=file_metadata, + semantic_match_engine=semantic_match_engine, + build_import_metadata_conflict=build_import_metadata_conflict, + series_high_confidence=series_high_confidence, + ) + if match_candidate is not None: + enriched_metadata = await load_archive_entry_issue_hint_for_import_file( + imp_file, + file_metadata, + ) + if enriched_metadata.diagnostics != file_metadata.diagnostics: + file_metadata = enriched_metadata + match_candidate, metadata_conflict = _evaluate_file_match_candidate( + imp_series=imp_series, + imp_file=imp_file, + target_index=target_index, + target_series=target_index.existing_series, + file_metadata=file_metadata, + semantic_match_engine=semantic_match_engine, + build_import_metadata_conflict=build_import_metadata_conflict, + series_high_confidence=series_high_confidence, + ) + file_evaluation_duration_ms += ( + time.monotonic() - file_evaluation_started_at + ) * 1000 + + await apply_and_log_file_match_outcome( + session=session, + job_id=job.id, + imp_file=imp_file, + imp_series=imp_series, + match_candidate=match_candidate, + duplicate_series=duplicate_series, + duplicate_target_state=duplicate_target_state, + duplicate_merge_profile=duplicate_merge_profile, + metadata_conflict=metadata_conflict, + log_event=log_event, + ) + completed_file_phase_units += 1 + completed_file_match_weight += file_match_unit_weight + file_index = file_index_offset + page_file_index + await emit_file_matching_progress( + imp_series, + completed_file_phase_units, + message=f"Matched file {file_index}/{total_file_count} for " + f"{imp_series.raw_series_name}", + current_item_stage="file_matching", + current_item_progress_pct=( + 50 + round((file_index / max(total_file_count, 1)) * 50) + ), + current_work_unit_progress_pct=0, + live_only=True, + ) + + return files, created_split_series_ids + + async def process_split_series_batch(batch_ids: list[int]) -> None: + nonlocal conflict_group_counter + nonlocal duplicate_group_counter + + if not batch_ids: + return + await session.flush() + await session.commit() + await run_import_file_matching( + session, + job, + metadata_provider=metadata_provider, + semantic_match_engine=semantic_match_engine, + is_duplicate_series=is_duplicate_series, + build_duplicate_merge_profile=build_duplicate_merge_profile, + duplicate_target_state=duplicate_target_state, + source_metadata_for_import_file=source_metadata_for_import_file, + load_deferred_source_metadata_for_import_file=( + load_deferred_source_metadata_for_import_file + ), + build_import_metadata_conflict=build_import_metadata_conflict, + raise_if_cancelled=raise_if_cancelled, + detect_duplicate_copies=detect_duplicate_copies, + detect_conflicts=detect_conflicts, + recompute_file_counters=recompute_file_counters, + recompute_series_counters=recompute_series_counters, + log_event=log_event, + emit_progress=emit_progress, + phase_progress=phase_progress, + estimate_remaining_seconds=estimate_remaining_seconds, + job_stats=job_stats, + maybe_slow_item_delay=maybe_slow_item_delay, + log_warning=log_warning, + series_ids=batch_ids, + progress_callback=None, + rebuild_cross_conflicts=False, + ) + counters = await session.execute( + sa_select( + sa_func.max(ImportedFile.conflict_group_id), + sa_func.max(ImportedFile.duplicate_group_id), + ).where(ImportedFile.import_job_id == job.id) + ) + max_conflict_group_id, max_duplicate_group_id = counters.one() + conflict_group_counter = max( + conflict_group_counter, + int(max_conflict_group_id or 0), + ) + duplicate_group_counter = max( + duplicate_group_counter, + int(max_duplicate_group_id or 0), + ) + + async def emit_review_progress(message: str, *, complete: bool = False) -> None: + if progress_callback is None: + return + await emit_progress( + session, + job, + ImportProgressEvent( + job_id=job.id, + status=ImportJobStatus.FILE_MATCHING, + phase="file_matching", + progress=( + 99 + if complete + else scan_review_progress_pct( + progress_plan, completed_weight=completed_file_match_weight + ) + ), + message=message, + estimated_seconds_remaining=( + None + if complete + else estimate_remaining_work_seconds( + work_started_at, + completed_units=completed_file_match_weight, + total_units=progress_plan.total_weight, + ) + ), + **current_item_payload( + kind="scan", stage="file_matching", progress_pct=100 if complete else None + ), + **job_stats(job), + ), + progress_callback, + ) + + async def checkpoint_summary( + item: ImportedSeries, + summary: FileMatchSeriesSummary, + *, + series_idx: int, + page_end: bool, + ) -> None: + nonlocal total_found, total_matched, total_duplicate, total_already_owned + nonlocal total_no_match, total_conflict, last_checkpoint_at, completed_file_match_weight + completed_file_match_weight += _review_summary_weight(summary.found) + total_found += summary.found + total_matched += summary.matched + total_duplicate += summary.duplicate + total_already_owned += summary.already_owned + total_no_match += summary.no_match + total_conflict += summary.conflict + # Inventory is already known; partial summary counts must not shrink it. + job.total_files_found = max(int(job.total_files_found or 0), total_found) + job.total_files_matched = total_matched + job.total_files_duplicate = total_duplicate + job.total_files_already_owned = total_already_owned + job.total_files_no_match = total_no_match + job.total_files_conflict = total_conflict + should_checkpoint = ( + series_idx == 0 + or page_end + or series_idx == matching_plan.series_count - 1 + or time.monotonic() - last_checkpoint_at >= 0.5 + ) + if not should_checkpoint: + return + await raise_if_cancelled(session, job.id) + await session.flush() + await session.commit() + last_checkpoint_at = time.monotonic() + await emit_file_matching_progress( + item, + completed_file_phase_units, + message=f"Prepared file review for {item.raw_series_name}", + current_item_progress_pct=100, + current_work_unit_progress_pct=0, + ) + if progress_callback: + await maybe_slow_item_delay() + + await emit_review_progress("Preparing file matching and review summaries...") + blocked_only_counts: dict[int, int] = {} + blocked_only_series = 0 + async for series_idx, imp_series, series_page_end in _iter_eligible_import_series( + session, + job_id=job.id, + series_ids=series_ids, + raise_if_cancelled=raise_if_cancelled, + page_size=_SERIES_PAGE_SIZE, + blocked_only_counts=blocked_only_counts, + ): + if imp_series.id in blocked_only_counts: + count = blocked_only_counts[imp_series.id] + blocked_only_series += 1 + imp_series.files_total = count + imp_series.files_matched = 0 + imp_series.files_duplicate = 0 + imp_series.files_already_owned = 0 + imp_series.files_no_match = 0 + imp_series.files_conflict = 0 + await checkpoint_summary( + imp_series, + FileMatchSeriesSummary(count, 0, 0, 0, 0, 0), + series_idx=series_idx, + page_end=series_page_end, + ) + continue + await raise_if_cancelled(session, job.id) + duplicate_series = is_duplicate_series(imp_series) + + eligible_file_count = await _count_import_series_files( + session, + import_series_id=imp_series.id, + eligible_only=True, + ) + total_series_file_count = await _count_import_series_files( + session, + import_series_id=imp_series.id, + eligible_only=False, + ) + stable_series_file_count = _stable_series_file_count( + imp_series, + persisted_file_count=total_series_file_count, + ) + target_progress_completed = imp_series.id in matching_plan.completed_target_series_ids + active_target_weight = ( + 0.0 + if target_progress_completed + else scan_review_file_target_weight( + ScanReviewFileMatchProfile( + file_count=eligible_file_count, + issue_count=imp_series.cv_issue_count, + ) + ) + ) + target_load_failed = False + series_high_confidence = ( + imp_series.cv_match_score is not None and imp_series.cv_match_score >= 0.90 + ) + duplicate_merge_profile = ( + await _build_bounded_duplicate_merge_profile( + session, + imp_series, + incoming_file_count=total_series_file_count, + build_duplicate_merge_profile=build_duplicate_merge_profile, + ) + if duplicate_series + else None + ) + original_series_status = imp_series.status + original_series_diagnostics = dict(imp_series.diagnostics or {}) + retained_file_count = 0 + processed_file_count = 0 + series_had_processed_files = False + file_after_id = 0 + while eligible_file_count: + await raise_if_cancelled(session, job.id) + file_page = await _load_eligible_import_file_page( + session, + import_series_id=imp_series.id, + after_id=file_after_id, + page_size=_FILE_PAGE_SIZE, + ) + if not file_page: + break + page_last_id = file_page[-1].id + # End the page-read transaction before any provider/archive-backed target work. + await session.commit() + target_index_started_at = time.monotonic() + try: + target_index = await _load_file_page_target_index( + session, + job, + imp_series, + file_page, + series_file_count=stable_series_file_count, + duplicate_series=duplicate_series, + metadata_provider=metadata_provider, + series_idx=series_idx, + total_series=total_series, + completed_units=completed_file_phase_units, + progress_callback=progress_callback, + emit_file_matching_progress=emit_file_matching_progress, + raise_if_cancelled=raise_if_cancelled, ) - deferred_metadata_duration_ms += ( - time.monotonic() - deferred_metadata_started_at - ) * 1000 - file_evaluation_started_at = time.monotonic() - match_candidate, metadata_conflict = _evaluate_file_match_candidate( + target_index_duration_ms += (time.monotonic() - target_index_started_at) * 1000 + if ( + imp_series.series_id is None + and not target_index.has_targets + and not can_mark_missing_issue_targets(imp_series, metadata_provider) + ): + target_load_failed = True + break + except ImportProviderDegradedError as exc: + active_target_weight = 0.0 + target_index_duration_ms += (time.monotonic() - target_index_started_at) * 1000 + deferred_count += 1 + if len(deferred_titles) < 10: + deferred_titles.append(imp_series.raw_series_name) + await defer_file_matching_for_provider_error( + session=session, + job_id=job.id, imp_series=imp_series, - imp_file=imp_file, - target_index=target_index, - target_series=target_index.existing_series, - file_metadata=file_metadata, - semantic_match_engine=semantic_match_engine, - build_import_metadata_conflict=build_import_metadata_conflict, - series_high_confidence=series_high_confidence, + exc=exc, + log_event=log_event, ) - if match_candidate is not None: - enriched_metadata = await load_archive_entry_issue_hint_for_import_file( - imp_file, - file_metadata, + target_load_failed = True + break + except _ImportFileMatchingCohortLimitError: + raise + except Exception: + active_target_weight = 0.0 + if imp_series.cv_id is not None: + log_warning( + "file_matching_cv_fetch_failed", + series=imp_series.raw_series_name, + cv_id=imp_series.cv_id, ) - if enriched_metadata.diagnostics != file_metadata.diagnostics: - file_metadata = enriched_metadata - match_candidate, metadata_conflict = _evaluate_file_match_candidate( - imp_series=imp_series, - imp_file=imp_file, - target_index=target_index, - target_series=target_index.existing_series, - file_metadata=file_metadata, - semantic_match_engine=semantic_match_engine, - build_import_metadata_conflict=build_import_metadata_conflict, - series_high_confidence=series_high_confidence, - ) - file_evaluation_duration_ms += ( - time.monotonic() - file_evaluation_started_at - ) * 1000 + target_load_failed = True + break + raise - await apply_and_log_file_match_outcome( - session=session, - job_id=job.id, - imp_file=imp_file, - imp_series=imp_series, - match_candidate=match_candidate, + if not target_progress_completed: + completed_file_phase_units += 1 + completed_file_match_weight += active_target_weight + active_target_weight = 0.0 + target_progress_completed = True + # File metadata/archive evaluation starts outside the target-read transaction. + await session.commit() + matched_page, created_split_series_ids = await process_file_page( + imp_series, + file_page, + target_index=target_index, duplicate_series=duplicate_series, - duplicate_target_state=duplicate_target_state, duplicate_merge_profile=duplicate_merge_profile, - metadata_conflict=metadata_conflict, - log_event=log_event, - ) - completed_file_phase_units += 1 - await emit_file_matching_progress( - imp_series, - completed_file_phase_units, - message=(f"Matched file {file_idx}/{len(files)} for {imp_series.raw_series_name}"), - current_item_stage="file_matching", - current_item_progress_pct=50 + round((file_idx / max(len(files), 1)) * 50), - current_work_unit_progress_pct=0, - live_only=True, + series_high_confidence=series_high_confidence, + file_index_offset=processed_file_count, + total_file_count=eligible_file_count, ) + retained_file_count += len(matched_page) + processed_file_count += len(file_page) + series_had_processed_files = series_had_processed_files or bool(matched_page) + for split_id in created_split_series_ids: + blocked_only_counts.pop(split_id, None) + split_series_ids.update(created_split_series_ids) + while len(split_series_ids) >= _SERIES_PAGE_SIZE: + split_batch = sorted(split_series_ids)[:_SERIES_PAGE_SIZE] + split_series_ids.difference_update(split_batch) + await process_split_series_batch(split_batch) + file_after_id = page_last_id + await session.flush() + await session.commit() - ( - _duplicate_count, - duplicate_group_counter, - _duplicate_groups, - ) = await detect_duplicate_copies( + if series_had_processed_files: + series_processed += 1 + + if target_load_failed: + await session.flush() + await session.commit() + continue + + if retained_file_count and imp_series.status == ImportSeriesStatus.SKIPPED: + imp_series.status = original_series_status + imp_series.diagnostics = original_series_diagnostics + + duplicate_group_counter, conflict_group_counter = await _finalize_import_series_file_groups( session, job, imp_series, - files, - duplicate_group_counter, - ) - _conflict_count, conflict_group_counter, conflict_groups = detect_conflicts( - files, - conflict_group_counter, + duplicate_group_counter=duplicate_group_counter, + conflict_group_counter=conflict_group_counter, + detect_duplicate_copies=detect_duplicate_copies, + detect_conflicts=detect_conflicts, + log_event=log_event, + raise_if_cancelled=raise_if_cancelled, ) - - for group_details in conflict_groups: - await log_event( - session, - job.id, - "DEBUG", - "import_file_conflict_detail", - message=( - f"Conflict group {group_details['conflict_group_id']} in " - f"{imp_series.raw_series_name}" - ), - series=imp_series.raw_series_name, - diagnostics=group_details, - ) - - series_summary = apply_file_match_series_summary( + series_summary = await _summarize_import_series_files( + session, imp_series, - files, duplicate_series=duplicate_series, duplicate_merge_profile=duplicate_merge_profile, cv_match_threshold=job.cv_match_threshold, + raise_if_cancelled=raise_if_cancelled, + job_id=job.id, ) if series_summary.series_invalidated: invalidation_reason = (series_summary.invalidation_diagnostics or {}).get("reason") @@ -480,106 +1885,24 @@ async def run_import_file_matching( diagnostics=series_summary.invalidation_diagnostics, ) - total_found += series_summary.found - total_matched += series_summary.matched - total_duplicate += series_summary.duplicate - total_already_owned += series_summary.already_owned - total_no_match += series_summary.no_match - total_conflict += series_summary.conflict - job.total_files_found = total_found - job.total_files_matched = total_matched - job.total_files_duplicate = total_duplicate - job.total_files_already_owned = total_already_owned - job.total_files_no_match = total_no_match - job.total_files_conflict = total_conflict - - should_checkpoint = ( - (series_idx == 0) - or ((series_idx + 1) % persist_batch_size == 0) - or series_idx == len(matched_series_list) - 1 - or (time.monotonic() - last_checkpoint_at) >= 0.5 + await checkpoint_summary( + imp_series, series_summary, series_idx=series_idx, page_end=series_page_end ) - if should_checkpoint: - await session.flush() - await session.commit() - last_checkpoint_at = time.monotonic() - - if progress_callback and should_checkpoint: - completed_weight = scan_review_completed_weight( - progress_plan, - phase="file_matching", - completed_items=completed_file_phase_units, - ) - progress = scan_review_progress_pct( - progress_plan, - completed_weight=completed_weight, - ) - await emit_progress( - session, - job, - ImportProgressEvent( - job_id=job.id, - status=ImportJobStatus.FILE_MATCHING, - phase="file_matching", - progress=progress, - message=f"Matched files in {imp_series.raw_series_name}", - current_series=imp_series.raw_series_name, - estimated_seconds_remaining=estimate_remaining_work_seconds( - job.scan_completed_at or job.match_completed_at or job.scan_started_at, - completed_units=completed_weight, - total_units=progress_plan.total_weight, - ), - **current_item_payload( - kind="series", - stage="file_matching", - name=imp_series.raw_series_name, - progress_pct=100, - ), - **job_stats(job), - ), - progress_callback, - ) - await maybe_slow_item_delay() if split_series_ids: - await session.flush() - await session.commit() - await run_import_file_matching( + await process_split_series_batch(sorted(split_series_ids)) + + await emit_review_progress("Finalizing review summaries and cross-series conflicts...") + if rebuild_cross_conflicts: + conflict_group_counter = await _rebuild_cross_series_conflicts( session, job, - metadata_provider=metadata_provider, - semantic_match_engine=semantic_match_engine, + conflict_group_counter=conflict_group_counter, is_duplicate_series=is_duplicate_series, - build_duplicate_merge_profile=build_duplicate_merge_profile, - duplicate_target_state=duplicate_target_state, - source_metadata_for_import_file=source_metadata_for_import_file, - load_deferred_source_metadata_for_import_file=load_deferred_source_metadata_for_import_file, - build_import_metadata_conflict=build_import_metadata_conflict, - raise_if_cancelled=raise_if_cancelled, - detect_duplicate_copies=detect_duplicate_copies, - detect_conflicts=detect_conflicts, - recompute_file_counters=recompute_file_counters, - recompute_series_counters=recompute_series_counters, log_event=log_event, - emit_progress=emit_progress, - phase_progress=phase_progress, - estimate_remaining_seconds=estimate_remaining_seconds, - job_stats=job_stats, - maybe_slow_item_delay=maybe_slow_item_delay, - log_warning=log_warning, - series_ids=sorted(split_series_ids), - progress_callback=None, + raise_if_cancelled=raise_if_cancelled, ) - conflict_group_counter = await _rebuild_cross_series_conflicts( - session, - job, - conflict_group_counter=conflict_group_counter, - is_duplicate_series=is_duplicate_series, - cv_match_threshold=job.cv_match_threshold, - log_event=log_event, - ) - if deferred_count: job.error_message = ( f"ComicVine issue targets were unavailable for {deferred_count} matched series. " @@ -626,6 +1949,7 @@ async def run_import_file_matching( files_already_owned=job.total_files_already_owned, files_no_match=job.total_files_no_match, files_conflict=job.total_files_conflict, + blocked_only_series=blocked_only_series, duration_ms=round((time.monotonic() - started_at) * 1000), series_processed=series_processed, files_processed=files_processed, @@ -635,15 +1959,38 @@ async def run_import_file_matching( deferred_metadata_loads=deferred_metadata_loads, provider_cache_metrics=_provider_cache_metrics(metadata_provider), ) + await emit_review_progress("File review summaries ready", complete=True) def _file_has_deferred_archive_metadata(imp_file: ImportedFile) -> bool: + return import_file_has_deferred_archive_metadata(imp_file) + + +def _persist_deferred_source_evidence( + imp_file: ImportedFile, + metadata: SourceMetadata, +) -> None: + """Persist local archive evidence without replacing Mylar row authority.""" diagnostics = dict(imp_file.diagnostics or {}) - source_metadata = diagnostics.get("source_metadata") - return bool( - isinstance(source_metadata, dict) - and source_metadata.get("archive_metadata_deferred") is True - ) + diagnostics["source_metadata"] = dict(metadata.diagnostics) + diagnostics["metadata_signals"] = { + key: signal.value for key, signal in metadata.signals.items() + } + diagnostics["source_issue_type"] = metadata.issue_type.value + if metadata.comicvine_series_id is not None: + diagnostics["comicvine_series_id"] = metadata.comicvine_series_id + imp_file.diagnostics = diagnostics + imp_file.has_comicinfo = bool(metadata.diagnostics.get("has_comicinfo")) + issue_identity_reconciliation = metadata.diagnostics.get("mylar3_issue_identity_reconciliation") + if metadata.comicvine_issue_id is not None and ( + imp_file.comicvine_issue_id is None + or ( + isinstance(issue_identity_reconciliation, dict) + and issue_identity_reconciliation.get("embedded_comicvine_issue_id") + == metadata.comicvine_issue_id + ) + ): + imp_file.comicvine_issue_id = metadata.comicvine_issue_id def _provider_cache_metrics(metadata_provider: MetadataProvider | None) -> dict[str, Any]: @@ -722,14 +2069,6 @@ def _evaluate_file_match_candidate( return match_candidate, metadata_conflict -def _resolved_target_series_key(imp_series: ImportedSeries) -> tuple[str, int] | None: - if imp_series.series_id is not None: - return ("series", int(imp_series.series_id)) - if imp_series.cv_id is not None: - return ("cv", int(imp_series.cv_id)) - return None - - def _is_cross_series_conflict(imp_file: ImportedFile) -> bool: diagnostics = dict(imp_file.diagnostics or {}) return ( @@ -739,52 +2078,84 @@ def _is_cross_series_conflict(imp_file: ImportedFile) -> bool: ) +async def _apply_cross_conflict_counter_deltas( + session: AsyncSession, + counter_deltas: dict[int, tuple[int, int]], + *, + is_duplicate_series: IsDuplicateSeriesFunc, +) -> dict[int, str]: + if not counter_deltas: + return {} + if len(counter_deltas) > _MAX_IDENTITY_COHORT_SIZE: + raise _ImportFileMatchingCohortLimitError( + "Cross-series summary cohort exceeds the bounded automatic-matching limit." + ) + result = await session.execute( + sa_select(ImportedSeries) + .where(ImportedSeries.id.in_(sorted(counter_deltas))) + .order_by(ImportedSeries.id) + ) + series_labels: dict[int, str] = {} + for imp_series in result.scalars(): + series_labels[imp_series.id] = imp_series.raw_series_name + matched_delta, conflict_delta = counter_deltas.get(imp_series.id, (0, 0)) + imp_series.files_matched = max(int(imp_series.files_matched or 0) + matched_delta, 0) + imp_series.files_conflict = max( + int(imp_series.files_conflict or 0) + conflict_delta, + 0, + ) + if is_duplicate_series(imp_series): + diagnostics = dict(imp_series.diagnostics or {}) + actionable = bool( + diagnostics.get("actionable_duplicate_merge") + or imp_series.files_matched + or imp_series.files_conflict + ) + diagnostics.update( + { + "actionable_duplicate_merge": actionable, + "has_importable_files": imp_series.files_matched > 0, + "importable_files": imp_series.files_matched, + "conflict_files": imp_series.files_conflict, + } + ) + imp_series.diagnostics = diagnostics + return series_labels + + async def _rebuild_cross_series_conflicts( session: AsyncSession, job: ImportJob, *, conflict_group_counter: int, is_duplicate_series: IsDuplicateSeriesFunc, - cv_match_threshold: float, log_event: LogEventFunc, + raise_if_cancelled: RaiseIfCancelledFunc, ) -> int: - series_result = await session.execute( - sa_select(ImportedSeries).where( - ImportedSeries.import_job_id == job.id, - ImportedSeries.status.in_([ImportSeriesStatus.MATCHED, ImportSeriesStatus.DUPLICATE]), - ) - ) - series_list = list(series_result.scalars().all()) - target_series_key_by_series_id = { - imp_series.id: key - for imp_series in series_list - if (key := _resolved_target_series_key(imp_series)) is not None - } - if not target_series_key_by_series_id: - return conflict_group_counter - - file_result = await session.execute( - sa_select(ImportedFile).where( - ImportedFile.import_job_id == job.id, - ImportedFile.import_series_id.in_(list(target_series_key_by_series_id)), - ImportedFile.status.in_([ImportedFileStatus.MATCHED, ImportedFileStatus.CONFLICT]), + reset_after_id = 0 + while True: + await raise_if_cancelled(session, job.id) + reset_result = await session.execute( + sa_select(ImportedFile) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .where( + ImportedFile.import_job_id == job.id, + ImportedFile.status == ImportedFileStatus.CONFLICT, + ImportedSeries.status.in_( + [ImportSeriesStatus.MATCHED, ImportSeriesStatus.DUPLICATE] + ), + ImportedFile.id > reset_after_id, + ) + .order_by(ImportedFile.id) + .limit(_FILE_PAGE_SIZE) ) - ) - files = list(file_result.scalars().all()) - if not files: - return conflict_group_counter - - affected_series_ids: set[int] = set() - target_series_key_by_file_id: dict[int, tuple[str, int]] = {} - series_by_id = {imp_series.id: imp_series for imp_series in series_list} - for imp_file in files: - if imp_file.id is None: - continue - target_series_key = target_series_key_by_series_id.get(imp_file.import_series_id) - if target_series_key is None: - continue - target_series_key_by_file_id[imp_file.id] = target_series_key - if _is_cross_series_conflict(imp_file): + reset_page = list(reset_result.scalars()) + if not reset_page: + break + reset_counter_deltas: dict[int, tuple[int, int]] = {} + for imp_file in reset_page: + if not _is_cross_series_conflict(imp_file): + continue imp_file.status = ImportedFileStatus.MATCHED imp_file.conflict_group_id = None imp_file.is_preferred = False @@ -793,67 +2164,317 @@ async def _rebuild_cross_series_conflicts( imp_file.diagnostics = ( dict(previous_diagnostics) if isinstance(previous_diagnostics, dict) else {} ) - affected_series_ids.add(imp_file.import_series_id) + matched_delta, conflict_delta = reset_counter_deltas.get( + imp_file.import_series_id, + (0, 0), + ) + reset_counter_deltas[imp_file.import_series_id] = ( + matched_delta + 1, + conflict_delta - 1, + ) + reset_after_id = reset_page[-1].id + await _apply_cross_conflict_counter_deltas( + session, + reset_counter_deltas, + is_duplicate_series=is_duplicate_series, + ) + await session.flush() + await session.commit() - ( - _cross_conflict_count, - conflict_group_counter, - cross_conflict_groups, - ) = detect_cross_series_conflicts( - files, - conflict_group_counter, - target_series_key_by_file_id=target_series_key_by_file_id, - ) + for target_kind in _CROSS_TARGET_COHORT_KINDS: + for issue_kind in _FILE_TARGET_COHORT_KINDS: + async for cohorts in _iter_cross_conflict_cohort_batches( + session, + job_id=job.id, + target_kind=target_kind, + issue_kind=issue_kind, + batch_size=_TARGET_COHORT_PAGE_SIZE, + raise_if_cancelled=raise_if_cancelled, + ): + for cohort in cohorts: + await raise_if_cancelled(session, job.id) + files = await _load_cross_conflict_cohort( + session, + job_id=job.id, + cohort=cohort, + ) + if not files: + continue + target_key = (cohort.target_kind, cohort.target_value) + target_series_key_by_file_id = { + imp_file.id: target_key for imp_file in files if imp_file.id is not None + } + ( + _cross_conflict_count, + conflict_group_counter, + cross_conflict_groups, + ) = detect_cross_series_conflicts( + files, + conflict_group_counter, + target_series_key_by_file_id=target_series_key_by_file_id, + ) + cohort_counter_deltas: dict[int, tuple[int, int]] = {} + group_series_ids_by_group: list[tuple[dict[str, Any], set[int]]] = [] + for group_details in cross_conflict_groups: + group_file_ids = { + file_info["file_id"] for file_info in group_details.get("files", []) + } + group_series_ids = { + imp_file.import_series_id + for imp_file in files + if imp_file.id in group_file_ids + } + group_series_ids_by_group.append((group_details, group_series_ids)) + for imp_file in files: + if imp_file.id not in group_file_ids: + continue + matched_delta, conflict_delta = cohort_counter_deltas.get( + imp_file.import_series_id, + (0, 0), + ) + cohort_counter_deltas[imp_file.import_series_id] = ( + matched_delta - 1, + conflict_delta + 1, + ) + series_labels = await _apply_cross_conflict_counter_deltas( + session, + cohort_counter_deltas, + is_duplicate_series=is_duplicate_series, + ) + for group_details, group_series_ids in group_series_ids_by_group: + group_series_labels = sorted( + series_labels[series_id] + for series_id in group_series_ids + if series_id in series_labels + ) + await log_event( + session, + job.id, + "DEBUG", + "import_file_conflict_detail", + message=( + f"Cross-series conflict group " + f"{group_details['conflict_group_id']} across " + f"{', '.join(group_series_labels)}" + ), + series=", ".join(group_series_labels), + diagnostics=group_details, + ) + await session.flush() + await session.commit() - for group_details in cross_conflict_groups: - group_file_ids = {file_info["file_id"] for file_info in group_details.get("files", [])} - group_series_labels = sorted( - { - series_item.raw_series_name - for imp_file in files - if imp_file.id in group_file_ids - for series_item in [series_by_id.get(imp_file.import_series_id)] - if series_item is not None - } - ) - for imp_file in files: - if imp_file.id in group_file_ids: - affected_series_ids.add(imp_file.import_series_id) - await log_event( - session, - job.id, - "DEBUG", - "import_file_conflict_detail", - message=( - f"Cross-series conflict group {group_details['conflict_group_id']} across " - f"{', '.join(group_series_labels)}" + return conflict_group_counter + + +_CROSS_TARGET_COHORT_KINDS = ("series", "cv") + + +def _cross_target_identity_spec(kind: str) -> tuple[Any, Any]: + if kind == "series": + return ImportedSeries.series_id, ImportedSeries.series_id.is_not(None) + if kind == "cv": + return ( + ImportedSeries.cv_id, + sa_and( + ImportedSeries.series_id.is_(None), + ImportedSeries.cv_id.is_not(None), ), - series=", ".join(group_series_labels), - diagnostics=group_details, ) + raise ValueError(f"Unsupported cross-series target cohort kind: {kind}") - if not affected_series_ids: - return conflict_group_counter - affected_files_result = await session.execute( - sa_select(ImportedFile).where( - ImportedFile.import_series_id.in_(sorted(affected_series_ids)) +def _cross_conflict_cohort_query( + *, + job_id: int, + target_kind: str, + issue_kind: str, + after_target_value: int | None, + after_issue_value: str | None, + page_size: int | None, +) -> Any: + target_column, target_filter = _cross_target_identity_spec(target_kind) + issue_column, issue_filter = _file_target_identity_spec(issue_kind) + cursor_filter = None + if after_target_value is not None: + if after_issue_value is None: + raise ValueError("Cross-series cohort cursor requires an issue identity") + issue_cursor: int | float = ( + int(after_issue_value) if issue_kind != "parsed_issue" else float(after_issue_value) + ) + cursor_filter = sa_or( + target_column > after_target_value, + sa_and( + target_column == after_target_value, + issue_column > issue_cursor, + ), + ) + query = ( + sa_select( + sa_literal(target_kind).label("target_kind"), + target_column.label("target_value"), + sa_literal(issue_kind).label("issue_kind"), + sa_cast(issue_column, String).label("issue_value"), + sa_func.min(ImportedFile.id).label("first_file_id"), + sa_func.count(ImportedFile.id).label("file_count"), + ) + .select_from(ImportedFile) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status.in_([ImportedFileStatus.MATCHED, ImportedFileStatus.CONFLICT]), + ImportedSeries.status.in_([ImportSeriesStatus.MATCHED, ImportSeriesStatus.DUPLICATE]), + target_filter, + issue_filter, + cursor_filter if cursor_filter is not None else sa_literal(True), ) + .group_by(target_column, issue_column) + .having(sa_func.count(sa_func.distinct(ImportedFile.import_series_id)) > 1) + .order_by(target_column, issue_column) ) - files_by_series_id: dict[int, list[ImportedFile]] = {} - for imp_file in affected_files_result.scalars().all(): - files_by_series_id.setdefault(imp_file.import_series_id, []).append(imp_file) + return query.limit(max(page_size, 1)) if page_size is not None else query - for series_id in sorted(affected_series_ids): - imp_series = series_by_id.get(series_id) - if imp_series is None: - continue - apply_file_match_series_summary( - imp_series, - files_by_series_id.get(series_id, []), - duplicate_series=is_duplicate_series(imp_series), - duplicate_merge_profile=None, - cv_match_threshold=cv_match_threshold, + +async def _load_cross_conflict_cohort_page( + session: AsyncSession, + *, + job_id: int, + target_kind: str, + issue_kind: str, + after_target_value: int | None, + after_issue_value: str | None, + page_size: int, +) -> list[_CrossConflictCohort]: + result = await session.execute( + _cross_conflict_cohort_query( + job_id=job_id, + target_kind=target_kind, + issue_kind=issue_kind, + after_target_value=after_target_value, + after_issue_value=after_issue_value, + page_size=page_size, + ) + ) + return [ + _CrossConflictCohort( + target_kind=str(row.target_kind), + target_value=int(row.target_value), + issue_kind=str(row.issue_kind), + issue_value=str(row.issue_value), + first_file_id=int(row.first_file_id), + file_count=int(row.file_count), ) + for row in result + ] - return conflict_group_counter + +async def _iter_cross_conflict_cohort_batches( + session: AsyncSession, + *, + job_id: int, + target_kind: str, + issue_kind: str, + batch_size: int, + raise_if_cancelled: RaiseIfCancelledFunc, +) -> AsyncIterator[list[_CrossConflictCohort]]: + """Yield bounded cross-conflict descriptors from one grouped execution.""" + spool = await _spool_grouped_cohort_rows( + session, + job_id=job_id, + query=_cross_conflict_cohort_query( + job_id=job_id, + target_kind=target_kind, + issue_kind=issue_kind, + after_target_value=None, + after_issue_value=None, + page_size=None, + ), + batch_size=batch_size, + scan_name="cross_conflict", + row_payload=lambda row: [ + str(row.target_kind), + int(row.target_value), + str(row.issue_kind), + str(row.issue_value), + int(row.first_file_id), + int(row.file_count), + ], + raise_if_cancelled=raise_if_cancelled, + ) + try: + while True: + await raise_if_cancelled(session, job_id) + batch: list[_CrossConflictCohort] = [] + while len(batch) < max(batch_size, 1): + line = spool.readline() + if not line: + break + ( + target_kind_value, + target_value, + issue_kind_value, + issue_value, + first_file_id, + file_count, + ) = json.loads(line) + batch.append( + _CrossConflictCohort( + target_kind=str(target_kind_value), + target_value=int(target_value), + issue_kind=str(issue_kind_value), + issue_value=str(issue_value), + first_file_id=int(first_file_id), + file_count=int(file_count), + ) + ) + if not batch: + break + yield batch + finally: + spool.close() + + +async def _load_cross_conflict_cohort( + session: AsyncSession, + *, + job_id: int, + cohort: _CrossConflictCohort, +) -> list[ImportedFile]: + if cohort.file_count > _MAX_IDENTITY_COHORT_SIZE: + raise _ImportFileMatchingCohortLimitError( + "Cross-series issue cohort exceeds the bounded automatic-matching limit " + f"({_MAX_IDENTITY_COHORT_SIZE} files; issue kind={cohort.issue_kind})." + ) + if cohort.target_kind == "series": + target_filter = ImportedSeries.series_id == cohort.target_value + else: + target_filter = sa_and( + ImportedSeries.series_id.is_(None), + ImportedSeries.cv_id == cohort.target_value, + ) + if cohort.issue_kind == "issue_id": + issue_filter = ImportedFile.matched_issue_id == int(cohort.issue_value) + elif cohort.issue_kind == "issue_cv_id": + issue_filter = sa_and( + ImportedFile.matched_issue_id.is_(None), + ImportedFile.matched_issue_cv_id == int(cohort.issue_value), + ) + else: + issue_filter = sa_and( + ImportedFile.matched_issue_id.is_(None), + ImportedFile.matched_issue_cv_id.is_(None), + ImportedFile.parsed_issue_number == float(cohort.issue_value), + ) + result = await session.execute( + sa_select(ImportedFile) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status.in_([ImportedFileStatus.MATCHED, ImportedFileStatus.CONFLICT]), + ImportedSeries.status.in_([ImportSeriesStatus.MATCHED, ImportSeriesStatus.DUPLICATE]), + target_filter, + issue_filter, + ) + .order_by(ImportedFile.id) + .limit(_MAX_IDENTITY_COHORT_SIZE) + ) + return list(result.scalars()) diff --git a/src/pullbox/services/import_file_matching_progress.py b/src/pullbox/services/import_file_matching_progress.py index 1a846677..02ef8e05 100644 --- a/src/pullbox/services/import_file_matching_progress.py +++ b/src/pullbox/services/import_file_matching_progress.py @@ -78,6 +78,7 @@ def build_file_matching_progress_emitter( scan_review_plan: ScanReviewProgressPlan | None = None, phase_start: int = 80, phase_end: int = 99, + work_started_at: datetime | None = None, ) -> Callable[..., Awaitable[None]]: """Build the Step 2 file-matching progress emitter.""" @@ -88,7 +89,7 @@ async def emit_file_matching_progress( message: str, current_item_stage: str = "file_matching", current_item_progress_pct: int | None = None, - current_work_unit_progress_pct: int | None = None, + current_work_unit_progress_pct: int | float | None = None, live_only: bool = False, ) -> None: if progress_callback is None: @@ -121,7 +122,10 @@ async def emit_file_matching_progress( ) estimated_seconds_remaining = ( estimate_remaining_work_seconds( - job.scan_completed_at or job.match_completed_at or job.scan_started_at, + work_started_at + or job.scan_completed_at + or job.match_completed_at + or job.scan_started_at, completed_units=( completed_weight if completed_weight is not None else completed_units ), @@ -183,6 +187,7 @@ async def load_file_match_target_index_with_progress( job_id: int, item: ImportedSeries, files_to_match: list[ImportedFile], + series_file_count: int | None = None, duplicate_series: bool, metadata_provider: MetadataProvider | None, series_idx: int, @@ -202,6 +207,7 @@ async def load_file_match_target_index_with_progress( duplicate_series=duplicate_series, metadata_provider=metadata_provider, files=files_to_match, + series_file_count=series_file_count, ) await emit_file_matching_progress( @@ -223,6 +229,7 @@ async def load_file_match_target_index_with_progress( duplicate_series=duplicate_series, metadata_provider=metadata_provider, files=files_to_match, + series_file_count=series_file_count, ) heartbeat_interval = ( @@ -238,6 +245,7 @@ async def _load_targets() -> FileMatchTargetIndex: duplicate_series=duplicate_series, metadata_provider=metadata_provider, files=files_to_match, + series_file_count=series_file_count, ) task: asyncio.Task[FileMatchTargetIndex] = asyncio.create_task(_load_targets()) diff --git a/src/pullbox/services/import_file_preparation.py b/src/pullbox/services/import_file_preparation.py index 790e771a..1ff58c8e 100644 --- a/src/pullbox/services/import_file_preparation.py +++ b/src/pullbox/services/import_file_preparation.py @@ -13,6 +13,7 @@ from pullbox.core.archive import inspect_archive_page_count as inspect_archive_page_count from pullbox.core.exceptions import NotFoundError, ValidationError from pullbox.core.file_safety import is_resource_safety_exception_allowed +from pullbox.core.issue_numbers import format_issue_number from pullbox.models.publisher import Publisher from pullbox.models.series import Series from pullbox.utilities.comicinfo import embed_comicinfo_in_cbz @@ -132,9 +133,7 @@ def format_comicinfo_issue_number(issue_number: float | None) -> str | None: """Render an issue number for ComicInfo.xml output.""" if issue_number is None: return None - if float(issue_number).is_integer(): - return str(int(issue_number)) - return f"{issue_number:g}" + return format_issue_number(issue_number) async def build_comicinfo_payload_for_issue( @@ -168,7 +167,7 @@ async def build_comicinfo_payload_for_issue( payload: dict[str, Any] = { "Series": series.title, - "Number": format_comicinfo_issue_number(issue.issue_number), + "Number": issue.effective_issue_number_text, "Title": issue.title, "Summary": issue.description, "Publisher": publisher_name, diff --git a/src/pullbox/services/import_file_registration_adapters.py b/src/pullbox/services/import_file_registration_adapters.py index b517cda2..36ca12b0 100644 --- a/src/pullbox/services/import_file_registration_adapters.py +++ b/src/pullbox/services/import_file_registration_adapters.py @@ -2,10 +2,16 @@ from __future__ import annotations +import hashlib +import os +import threading import time +from contextlib import suppress from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast +from pullbox.core.exceptions import ImportDestinationValidationError + if TYPE_CHECKING: from collections.abc import Awaitable, Callable from pathlib import Path @@ -21,9 +27,13 @@ class ImportLibraryFileAdapters: comicinfo_embedder: Any artifact_transfer: Any comicinfo_materializer: Any + placement_temp_paths: Any operation_timings: list[dict[str, Any]] +_PUBLISH_LOCK = threading.Lock() + + def build_import_library_file_adapters( *, session: Any, @@ -37,6 +47,23 @@ def build_import_library_file_adapters( """Build interruptible register_library_file adapters for import execution.""" operation_timings: list[dict[str, Any]] = [] + def placement_temp_paths(artifact_source: Path, artifact_target: Path) -> tuple[Path, Path]: + identity = "\0".join( + ( + str(getattr(job, "id", "unknown")), + str(artifact_source.expanduser().resolve(strict=False)), + str(artifact_target.expanduser().resolve(strict=False)), + ) + ) + digest = hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16] + stage_path = artifact_target.with_name( + f".pullbox-import-{getattr(job, 'id', 'unknown')}-{digest}{artifact_target.suffix}" + ) + return ( + stage_path, + stage_path.with_name(f"{stage_path.name}.pullbox-write.tmp"), + ) + async def convert_import_file( convert_source: Path, target_format: str, @@ -95,19 +122,31 @@ async def transfer_import_artifact( transfer_progress_callback = transfer_kwargs.get("transfer_progress_callback") source_size = artifact_source.stat().st_size if artifact_source.exists() else None started_at = clock() - result = cast( - "Path", + stage_path, _materializer_temp_path = placement_temp_paths( + artifact_source, + artifact_target, + ) + _require_unused_stage(stage_path) + try: await transfer_artifact_interruptible( session, job, artifact_source, - artifact_target, + stage_path, transfer_method, transfer_progress_callback=transfer_progress_callback if callable(transfer_progress_callback) else None, - ), - ) + ) + _publish_stage_without_overwrite(stage_path, artifact_target) + except BaseException: + _restore_source_or_cleanup_stage( + stage_path, + artifact_source, + transfer_method=transfer_method, + ) + raise + result = artifact_target operation_timings.append( { "kind": "transfer", @@ -132,17 +171,33 @@ async def materialize_import_cbz_with_comicinfo( ) -> bool: source_size = artifact_source.stat().st_size if artifact_source.exists() else None started_at = clock() - result = bool( - await materialize_cbz_with_comicinfo_interruptible( - session, - job, + stage_path, materializer_temp_path = placement_temp_paths( + artifact_source, + artifact_target, + ) + _require_unused_stage(stage_path) + _require_unused_stage(materializer_temp_path) + try: + result = bool( + await materialize_cbz_with_comicinfo_interruptible( + session, + job, + artifact_source, + stage_path, + payload, + transfer_method=transfer_method, + temp_path=materializer_temp_path, + progress_callback=progress_callback, + ) + ) + _publish_stage_without_overwrite(stage_path, artifact_target) + except BaseException: + _restore_source_or_cleanup_stage( + stage_path, artifact_source, - artifact_target, - payload, transfer_method=transfer_method, - progress_callback=progress_callback, ) - ) + raise operation_timings.append( { "kind": "cbz_comicinfo_materialize", @@ -165,5 +220,76 @@ async def materialize_import_cbz_with_comicinfo( comicinfo_embedder=embed_import_comicinfo, artifact_transfer=transfer_import_artifact, comicinfo_materializer=materialize_import_cbz_with_comicinfo, + placement_temp_paths=placement_temp_paths, operation_timings=operation_timings, ) + + +def _require_unused_stage(stage_path: Path) -> None: + if os.path.lexists(stage_path): + raise ImportDestinationValidationError( + "staging_path_exists", + f"Import staging path already exists and was preserved for review: {stage_path}", + ) + + +def _publish_stage_without_overwrite(stage_path: Path, target_path: Path) -> None: + """Publish a same-directory stage with an atomic no-overwrite filesystem claim.""" + if not os.path.lexists(stage_path): + raise FileNotFoundError(f"Import staging artifact is missing: {stage_path}") + target_path.parent.mkdir(parents=True, exist_ok=True) + with _PUBLISH_LOCK: + collision = _casefold_destination_collision(target_path) + if collision is not None: + raise ImportDestinationValidationError( + "destination_appeared", + f"Managed import destination appeared during import and was preserved: {collision}", + ) + try: + if stage_path.is_symlink(): + os.symlink(os.readlink(stage_path), target_path) + else: + os.link(stage_path, target_path, follow_symlinks=False) + except FileExistsError as exc: + raise ImportDestinationValidationError( + "destination_appeared", + "Managed import destination appeared during import and was preserved: " + f"{target_path}", + ) from exc + stage_path.unlink() + + +def _casefold_destination_collision(target_path: Path) -> Path | None: + key = target_path.name.casefold() + try: + with os.scandir(target_path.parent) as entries: + for entry in entries: + if entry.name.casefold() == key: + return target_path.parent / entry.name + except FileNotFoundError: + return None + return None + + +def _restore_source_or_cleanup_stage( + stage_path: Path, + source_path: Path, + *, + transfer_method: str, +) -> None: + if not os.path.lexists(stage_path): + return + if transfer_method == "move": + if os.path.lexists(source_path): + # A source reappeared after the move. Its identity is unknown, so + # preserve both it and the journaled stage for explicit review. + return + source_path.parent.mkdir(parents=True, exist_ok=True) + try: + _publish_stage_without_overwrite(stage_path, source_path) + except (ImportDestinationValidationError, OSError): + # Both paths are preserved when restoration cannot be proven safe. + return + return + with suppress(FileNotFoundError): + stage_path.unlink() diff --git a/src/pullbox/services/import_file_resolution.py b/src/pullbox/services/import_file_resolution.py index 36de7b17..498f982e 100644 --- a/src/pullbox/services/import_file_resolution.py +++ b/src/pullbox/services/import_file_resolution.py @@ -10,6 +10,7 @@ from pullbox.models.import_job import ImportedFile, ImportedFileStatus, ImportedSeries from pullbox.models.issue import Issue from pullbox.models.series import Series +from pullbox.services.import_file_issue_signals import candidate_issue_number_text if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -32,26 +33,16 @@ async def load_importable_files( files_result = await session.execute(sa_select(ImportedFile).where(*file_filters)) importable_files = list(files_result.scalars().all()) - if not duplicate_mode: - conflict_result = await session.execute( - sa_select(ImportedFile).where( - ImportedFile.import_series_id == item.id, - ImportedFile.status == ImportedFileStatus.CONFLICT, - ImportedFile.is_preferred.is_(True), - ) - ) - importable_files.extend(conflict_result.scalars().all()) - return importable_files async def load_issue_lookup_for_series( session: AsyncSession, series_id: int | None, -) -> tuple[dict[int, Issue], dict[float, Issue]]: - """Build ComicVine-ID and issue-number lookup maps for a library series.""" +) -> tuple[dict[int, Issue], dict[str, Issue], dict[float, Issue]]: + """Build ComicVine, exact-number, and unambiguous numeric lookup maps.""" if series_id is None: - return {}, {} + return {}, {}, {} issues_result = await session.execute( sa_select(Issue) @@ -62,12 +53,19 @@ async def load_issue_lookup_for_series( issues = issues_result.scalars().all() cv_id_to_issue: dict[int, Issue] = {} - number_to_issue: dict[float, Issue] = {} + exact_number_to_issue: dict[str, Issue] = {} + issues_by_number: dict[float, list[Issue]] = {} for issue in issues: if issue.comicvine_id is not None: cv_id_to_issue[issue.comicvine_id] = issue - number_to_issue[issue.issue_number] = issue - return cv_id_to_issue, number_to_issue + exact_number_to_issue[issue.effective_issue_number_text] = issue + issues_by_number.setdefault(issue.issue_number, []).append(issue) + number_to_issue = { + issue_number: candidates[0] + for issue_number, candidates in issues_by_number.items() + if len(candidates) == 1 + } + return cv_id_to_issue, exact_number_to_issue, number_to_issue async def resolve_import_file_issue( @@ -75,6 +73,7 @@ async def resolve_import_file_issue( imp_file: ImportedFile, *, cv_id_to_issue: dict[int, Issue], + exact_number_to_issue: dict[str, Issue], number_to_issue: dict[float, Issue], ) -> Issue | None: """Resolve a pre-import file match to a persisted library issue.""" @@ -93,6 +92,10 @@ async def resolve_import_file_issue( if resolved_issue is not None: return resolved_issue + exact_issue_number = candidate_issue_number_text(imp_file) + if exact_issue_number is not None: + return exact_number_to_issue.get(exact_issue_number) + if imp_file.parsed_issue_number is not None: return number_to_issue.get(imp_file.parsed_issue_number) diff --git a/src/pullbox/services/import_file_review.py b/src/pullbox/services/import_file_review.py index ba1b3e00..da0b5525 100644 --- a/src/pullbox/services/import_file_review.py +++ b/src/pullbox/services/import_file_review.py @@ -15,6 +15,7 @@ ImportedFile, ImportedFileStatus, ImportedSeries, + ImportFileHandlingMode, ImportJob, ImportJobStatus, ) @@ -87,6 +88,8 @@ async def apply_manual_file_match( confidence: str = "high", ) -> tuple[str, str | None]: """Apply a manual issue assignment, respecting duplicate-series merge rules.""" + if imp_file.status == ImportedFileStatus.SAFETY_BLOCKED: + raise ValidationError("Resolve this file's safety review before assigning an issue.") imp_series = await session.get(ImportedSeries, imp_file.import_series_id) duplicate_series = is_duplicate_series(imp_series) has_library_file = False @@ -157,6 +160,9 @@ async def override_file_match( if imp_file is None or imp_file.import_job_id != job_id: raise NotFoundError("ImportedFile", file_id) + if imp_file.status == ImportedFileStatus.SAFETY_BLOCKED: + raise ValidationError("Resolve this file's safety review before assigning an issue.") + issue = await session.get(Issue, issue_id) if issue is None: raise NotFoundError("Issue", issue_id) @@ -212,11 +218,19 @@ async def repair_file_metadata( raise NotFoundError("ImportJob", job_id) if job.status != ImportJobStatus.REVIEW: raise ValidationError("Job must be in REVIEW state to repair file metadata") + if job.file_handling_mode == ImportFileHandlingMode.IN_PLACE: + raise ValidationError( + "In-place import files cannot have embedded metadata rewritten. " + "Choose a managed copy if Pullbox should repair ComicInfo metadata." + ) imp_file = await session.get(ImportedFile, file_id) if imp_file is None or imp_file.import_job_id != job_id: raise NotFoundError("ImportedFile", file_id) + if imp_file.status == ImportedFileStatus.SAFETY_BLOCKED: + raise ValidationError("Resolve this file's safety review before repairing metadata.") + target_issue_id = issue_id or imp_file.matched_issue_id if target_issue_id is None: raise ValidationError( diff --git a/src/pullbox/services/import_file_split_series.py b/src/pullbox/services/import_file_split_series.py index 34cac70c..ea49efd1 100644 --- a/src/pullbox/services/import_file_split_series.py +++ b/src/pullbox/services/import_file_split_series.py @@ -70,6 +70,15 @@ async def split_explicit_issue_series_mismatches( remaining_files.append(imp_file) continue + series_signal = metadata.signals.get("comicvine_series_id") + if ( + metadata.comicvine_series_id == current_target_cv_id + and series_signal in {MetadataSignal.COMICINFO, MetadataSignal.SIDECAR} + and not metadata.diagnostics.get("identity_conflicts") + ): + remaining_files.append(imp_file) + continue + try: issue_meta = await metadata_provider.get_issue(str(metadata.comicvine_issue_id)) except Exception as exc: @@ -134,20 +143,31 @@ async def split_explicit_issue_series_mismatches( imp_series.sample_paths = [ path for path in list(imp_series.sample_paths or []) if path not in moved_file_paths ] - imp_series.file_count = max(int(imp_series.file_count or 0) - len(moved_file_ids), 0) - if not remaining_files: + pre_move_file_count = int(imp_series.file_count or 0) + diagnostics = dict(imp_series.diagnostics or {}) + diagnostics.setdefault( + "pre_split_file_count", + max(pre_move_file_count, int(imp_series.files_total or 0)), + ) + imp_series.file_count = max(pre_move_file_count - len(moved_file_ids), 0) + accumulated_file_ids = list(diagnostics.get("moved_file_ids") or []) + accumulated_split_ids = list(diagnostics.get("split_series_ids") or []) + diagnostics["moved_file_ids"] = list( + dict.fromkeys([*accumulated_file_ids, *moved_file_ids]) + ) + diagnostics["split_series_ids"] = list( + dict.fromkeys([*accumulated_split_ids, *created_split_series_ids]) + ) + if int(imp_series.file_count or 0) == 0: imp_series.status = ImportSeriesStatus.SKIPPED imp_series.selected_for_import = False - diagnostics = dict(imp_series.diagnostics or {}) diagnostics.update( { "kind": "rebucketed_series", "reason": "all_files_moved_to_split_series", - "moved_file_ids": moved_file_ids, - "split_series_ids": created_split_series_ids, } ) - imp_series.diagnostics = diagnostics + imp_series.diagnostics = diagnostics return remaining_files, created_split_series_ids diff --git a/src/pullbox/services/import_folder_story_arc_evidence.py b/src/pullbox/services/import_folder_story_arc_evidence.py new file mode 100644 index 00000000..4618a187 --- /dev/null +++ b/src/pullbox/services/import_folder_story_arc_evidence.py @@ -0,0 +1,79 @@ +"""Build provider-free folder story-arc evidence from staged import rows.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING + +from pullbox.core.issue_numbers import format_issue_number +from pullbox.core.story_arc_ordering import extract_story_arc_order_prefix +from pullbox.services.import_story_arc_detection import ( + FolderArcDetection, + FolderArcFileEvidence, + detect_folder_story_arc, +) + +if TYPE_CHECKING: + from pullbox.models.import_job import ImportedFile + + +def detect_imported_folder_story_arc( + *, + folder_label: str, + files: Sequence[ImportedFile], + confirmed_order_pattern: bool = False, +) -> FolderArcDetection: + """Classify one complete staged folder cohort without I/O or providers.""" + evidence = tuple(_evidence_from_imported_file(item) for item in files) + return detect_folder_story_arc( + folder_label=folder_label, + files=evidence, + confirmed_order_pattern=confirmed_order_pattern, + ) + + +def _evidence_from_imported_file(imp_file: ImportedFile) -> FolderArcFileEvidence: + diagnostics = _mapping(imp_file.diagnostics) + source_metadata = _mapping(diagnostics.get("source_metadata")) or diagnostics + archive_evidence = _mapping(source_metadata.get("archive_member_evidence")) or _mapping( + diagnostics.get("archive_member_evidence") + ) + comicinfo = _mapping(archive_evidence.get("comicinfo")) or _mapping( + source_metadata.get("comicinfo") + ) + + story_arc = _optional_text(comicinfo.get("story_arc")) + story_arc_number = _optional_text(comicinfo.get("story_arc_number")) + story_arc_number_source = "comicinfo" if story_arc_number is not None else None + if story_arc_number is None: + prefix = extract_story_arc_order_prefix(str(imp_file.file_name or "")) + if prefix is not None: + story_arc_number = prefix.reading_order_raw + story_arc_number_source = "filename_prefix" + + issue_number = _optional_text(comicinfo.get("number")) or _optional_text( + imp_file.issue_number_raw + ) + if issue_number is None and imp_file.parsed_issue_number is not None: + issue_number = format_issue_number(imp_file.parsed_issue_number) + + return FolderArcFileEvidence( + relative_path=str(imp_file.file_name or ""), + series=_optional_text(comicinfo.get("series")) or _optional_text(imp_file.parsed_series), + issue_number=issue_number, + story_arc=story_arc, + story_arc_number=story_arc_number, + story_arc_number_source=story_arc_number_source, + evidence_complete=not isinstance(diagnostics.get("safety_block"), Mapping), + ) + + +def _mapping(value: object) -> Mapping[str, object]: + return value if isinstance(value, Mapping) else {} + + +def _optional_text(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None diff --git a/src/pullbox/services/import_job_actions.py b/src/pullbox/services/import_job_actions.py index 7df3059d..def26bfd 100644 --- a/src/pullbox/services/import_job_actions.py +++ b/src/pullbox/services/import_job_actions.py @@ -3,29 +3,146 @@ from __future__ import annotations import contextlib +import os import shutil +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path -from typing import TYPE_CHECKING, Any, Protocol +from typing import TYPE_CHECKING, Any, Protocol, TypedDict from sqlalchemy import func as sa_func +from sqlalchemy import insert as sa_insert +from sqlalchemy import or_ from sqlalchemy import select as sa_select +from sqlalchemy import update as sa_update -from pullbox.core.exceptions import NotFoundError +from pullbox.core.exceptions import ConfigurationError, NotFoundError +from pullbox.core.library_file_ownership import ( + build_file_identity_signature, + build_managed_placement_signature, +) +from pullbox.models.blocklist import BlocklistEntry +from pullbox.models.direct_acquisition import DirectAcquisitionAttempt +from pullbox.models.download import DownloadHistory from pullbox.models.import_job import ( ImportedFile, + ImportedFileStatus, + ImportedSeries, ImportJob, ImportJobAction, ImportJobActionStatus, ) -from pullbox.models.library import LibraryFile +from pullbox.models.issue import Issue, IssueStatus +from pullbox.models.library import ( + FileFormat, + LibraryFile, + LibraryFileStorageMode, + LibraryRoot, + MatchConfidence, +) +from pullbox.models.pending_match import PendingMatch +from pullbox.models.reader import IssueReaderState +from pullbox.models.search_log import SearchLog from pullbox.models.series import Series +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcExternalIdentity, + StoryArcPlacement, + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.models.story_arc_sync import StoryArcSyncWork, StoryArcSyncWorkState from pullbox.utilities.settings import restore_file_from_utility_trash if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + from sqlalchemy.ext.asyncio import AsyncSession +_ACTION_SEQUENCE_CACHE_KEY = "pullbox.import_action_last_sequence" +_ACTION_INSERT_BATCH_SIZE = 200 +_STORY_ARC_MANAGED_PLACEMENT_ACTION = "story_arc_managed_placement_requested" +_STORY_ARC_PLACEMENT_PHASE = "story_arc_placements" +_STORY_ARC_MANAGED_PLACEMENT_PAYLOAD_KEYS = frozenset( + { + "schema_version", + "sync_work_id", + "membership_id", + "desired_generation", + "imported_story_arc_id", + "imported_story_arc_entry_id", + "source_import_job_id", + } +) +_CANCELLABLE_STORY_ARC_WORK_STATES = frozenset( + { + StoryArcSyncWorkState.QUEUED, + StoryArcSyncWorkState.RETRY_WAIT, + StoryArcSyncWorkState.FAILED, + } +) + + +class _ManagedPlacementRollbackPayload(TypedDict): + sync_work_id: int + membership_id: int + desired_generation: str + imported_story_arc_id: int + imported_story_arc_entry_id: int + source_import_job_id: int + + +@dataclass(frozen=True, slots=True) +class _AdoptedReferenceRollback: + source_imported_file: ImportedFile + source_library_file_id: int + file_path: str + file_name: str + file_size: int + file_format: FileFormat + file_hash: str | None + file_modified_at: datetime + match_confidence: MatchConfidence + parsed_series: str | None + parsed_issue_number: float | None + parsed_year: int | None + parsed_publisher: str | None + has_comicinfo: bool + naming_snapshot: dict[str, object] + source_signature: dict[str, object] + issue_id: int + library_root_id: int + series: Series + previous_series_path: str | None + previous_series_library_root_id: int | None + previous_series_preferred_library_root_id: int | None + + +class StoryArcManagedPlacementRollbackDeferredError(RuntimeError): + """A running placement must acknowledge cooperative cancellation first.""" + + def __init__(self, work_id: int) -> None: + self.work_id = work_id + super().__init__( + "Story Arc placement rollback is waiting for running work " + f"{work_id} to acknowledge cancellation" + ) + + +@dataclass(frozen=True, slots=True) +class ImportJobActionSpec: + """One ordered rollback-journal action awaiting sequence allocation.""" + + phase: str + action_type: str + payload: dict[str, Any] + + class DeleteSeriesForRollback(Protocol): """Callable used to remove a series created by an import action.""" @@ -39,6 +156,182 @@ async def __call__( ) -> None: ... +async def build_series_created_action_payload( + session: AsyncSession, + *, + series_id: int, + import_series_id: int, +) -> dict[str, Any]: + """Capture the user-owned series state required for safe rollback. + + Metadata hydration may legitimately update provider-owned fields after the + series is created, so this seal intentionally covers only user-owned + choices. Related files and activity are checked independently at rollback + time. Older journal rows without this seal are preserved for manual review. + """ + await session.flush() + series = await session.get(Series, series_id) + if series is None: + raise NotFoundError("Series", series_id) + imported_series = await session.get(ImportedSeries, import_series_id) + if imported_series is None: + raise NotFoundError("ImportedSeries", import_series_id) + issue_rows = ( + await session.execute( + sa_select(Issue.id, Issue.status, Issue.manual_skip) + .where(Issue.series_id == series_id) + .order_by(Issue.id) + ) + ).all() + payload: dict[str, Any] = { + "series_id": series_id, + "import_series_id": import_series_id, + "series_ownership_snapshot": _series_user_owned_snapshot(series), + "issue_ownership_snapshot": { + str(issue_id): { + "status": status.value, + "manual_skip": bool(manual_skip), + } + for issue_id, status, manual_skip in issue_rows + }, + } + cover_cache_ownership = _validated_cover_cache_ownership( + (imported_series.diagnostics or {}).get("cover_cache_ownership") + ) + if cover_cache_ownership is not None: + payload["cover_cache_ownership"] = cover_cache_ownership + series_folder_ownership = _validated_series_folder_ownership( + (imported_series.diagnostics or {}).get("series_folder_ownership") + ) + if series_folder_ownership is not None and _series_path_matches_folder_ownership( + series, + series_folder_ownership, + ): + payload["series_folder_ownership"] = _folder_ownership_with_installed_state( + series_folder_ownership, + series, + ) + return payload + + +async def build_series_cover_cache_action_payload( + session: AsyncSession, + *, + series_id: int, + import_series_id: int, + previous_cover_path: str | None, +) -> dict[str, Any] | None: + """Build a rollback action only for a cover artifact this import created.""" + await session.flush() + series = await session.get(Series, series_id) + imported_series = await session.get(ImportedSeries, import_series_id) + if series is None: + raise NotFoundError("Series", series_id) + if imported_series is None: + raise NotFoundError("ImportedSeries", import_series_id) + ownership = _validated_cover_cache_ownership( + (imported_series.diagnostics or {}).get("cover_cache_ownership") + ) + if ownership is None: + return None + installed_cover_path = series.cover_path + if not isinstance(installed_cover_path, str) or not installed_cover_path: + return None + return { + "series_id": series_id, + "import_series_id": import_series_id, + "previous_cover_path": previous_cover_path, + "installed_cover_path": installed_cover_path, + "cover_cache_ownership": ownership, + } + + +async def build_series_cover_path_updated_action_payload( + session: AsyncSession, + *, + series_id: int, + import_series_id: int, + previous_cover_path: str | None, +) -> dict[str, Any] | None: + """Journal a DB-only cover-path mutation without claiming its artifact.""" + await session.flush() + series = await session.get(Series, series_id) + imported_series = await session.get(ImportedSeries, import_series_id) + if series is None: + raise NotFoundError("Series", series_id) + if imported_series is None: + raise NotFoundError("ImportedSeries", import_series_id) + if series.cover_path == previous_cover_path: + return None + return { + "series_id": series_id, + "import_series_id": import_series_id, + "previous_cover_path": previous_cover_path, + "installed_cover_path": series.cover_path, + } + + +async def build_series_monitoring_updated_action_payload( + session: AsyncSession, + *, + series_id: int, + import_series_id: int, + previous_monitored: bool, +) -> dict[str, Any] | None: + """Journal an existing Series monitoring mutation made by search-on-add.""" + await session.flush() + series = await session.get(Series, series_id) + imported_series = await session.get(ImportedSeries, import_series_id) + if series is None: + raise NotFoundError("Series", series_id) + if imported_series is None: + raise NotFoundError("ImportedSeries", import_series_id) + installed_monitored = bool(series.monitored) + if installed_monitored == previous_monitored: + return None + return { + "series_id": series_id, + "import_series_id": import_series_id, + "previous_monitored": previous_monitored, + "installed_monitored": installed_monitored, + } + + +async def build_series_folder_created_action_payload( + session: AsyncSession, + *, + series_id: int, + import_series_id: int, + previous_series_path: str | None, + previous_library_root_id: int | None, + previous_preferred_library_root_id: int | None, +) -> dict[str, Any] | None: + """Build the replay-safe state restoration for an existing pathless Series.""" + await session.flush() + series = await session.get(Series, series_id) + imported_series = await session.get(ImportedSeries, import_series_id) + if series is None: + raise NotFoundError("Series", series_id) + if imported_series is None: + raise NotFoundError("ImportedSeries", import_series_id) + ownership = _validated_series_folder_ownership( + (imported_series.diagnostics or {}).get("series_folder_ownership") + ) + if ownership is None or not _series_path_matches_folder_ownership(series, ownership): + return None + return { + "series_id": series_id, + "import_series_id": import_series_id, + "previous_series_path": previous_series_path, + "previous_library_root_id": previous_library_root_id, + "previous_preferred_library_root_id": previous_preferred_library_root_id, + "series_folder_ownership": _folder_ownership_with_installed_state( + ownership, + series, + ), + } + + async def next_action_sequence(session: AsyncSession, job_id: int) -> int: """Return the next durable action sequence number for a job.""" max_seq = await session.scalar( @@ -57,10 +350,22 @@ async def record_action( action_type: str, payload: dict[str, Any], ) -> ImportJobAction: - """Persist a durable rollback journal action.""" + """Persist a durable rollback journal action. + + ``ImportRunner`` permits one active import execution at a time. Within that + single-writer boundary, cache the last sequence per session/job so a large + journal does not issue ``SELECT max(...)`` for every action. Rollback may + leave legal sequence gaps; ordering, not contiguity, is the contract. + """ + sequence_cache = _action_sequence_cache(session) + last_sequence = sequence_cache.get(job.id) + if last_sequence is None: + sequence_no = await next_action_sequence(session, job.id) + else: + sequence_no = last_sequence + 1 action = ImportJobAction( import_job_id=job.id, - sequence_no=await next_action_sequence(session, job.id), + sequence_no=sequence_no, phase=phase, action_type=action_type, status=ImportJobActionStatus.COMPLETED, @@ -68,9 +373,396 @@ async def record_action( ) session.add(action) await session.flush() + sequence_cache[job.id] = sequence_no return action +def _action_insert_rows( + *, + job_id: int, + first_sequence: int, + specs: Sequence[ImportJobActionSpec], +) -> list[dict[str, Any]]: + """Render an ordered sequence block as Core insert mappings.""" + return [ + { + "import_job_id": job_id, + "sequence_no": first_sequence + offset, + "phase": spec.phase, + "action_type": spec.action_type, + "status": ImportJobActionStatus.COMPLETED, + "payload": dict(spec.payload), + } + for offset, spec in enumerate(specs) + ] + + +def _action_insert_statement( + *, + returning: bool, +) -> Any: + """Build one portable executemany INSERT, optionally returning ORM rows.""" + statement = sa_insert(ImportJobAction) + return statement.returning(ImportJobAction) if returning else statement + + +def _supports_multirow_insert_returning(session: AsyncSession) -> bool: + dialect = session.get_bind().dialect + return bool( + getattr(dialect, "insert_returning", False) + and getattr(dialect, "supports_multivalues_insert", False) + ) + + +async def record_actions( + session: AsyncSession, + job: ImportJob, + specs: Sequence[ImportJobActionSpec], +) -> list[ImportJobAction]: + """Persist an ordered action batch inside the caller-owned transaction. + + One sequence range is allocated up front. Inserts are page-bounded and use + multi-row ``INSERT ... RETURNING`` on SQLite/PostgreSQL; other dialects use + bounded multi-row inserts followed by one range readback. The session cache + advances only after the complete batch succeeds, and this helper never + commits the surrounding transaction. + """ + ordered_specs = tuple(specs) + if not ordered_specs: + return [] + + sequence_cache = _action_sequence_cache(session) + last_sequence = sequence_cache.get(job.id) + first_sequence = ( + await next_action_sequence(session, job.id) if last_sequence is None else last_sequence + 1 + ) + rows = _action_insert_rows( + job_id=job.id, + first_sequence=first_sequence, + specs=ordered_specs, + ) + use_returning = _supports_multirow_insert_returning(session) + actions: list[ImportJobAction] = [] + for offset in range(0, len(rows), _ACTION_INSERT_BATCH_SIZE): + page = rows[offset : offset + _ACTION_INSERT_BATCH_SIZE] + if use_returning: + statement = _action_insert_statement(returning=True) + actions.extend((await session.scalars(statement, page)).all()) + else: + await session.execute(sa_insert(ImportJobAction), page) + + last_allocated_sequence = first_sequence + len(rows) - 1 + if not use_returning: + actions = list( + ( + await session.scalars( + sa_select(ImportJobAction) + .where( + ImportJobAction.import_job_id == job.id, + ImportJobAction.sequence_no >= first_sequence, + ImportJobAction.sequence_no <= last_allocated_sequence, + ) + .order_by(ImportJobAction.sequence_no.asc()) + ) + ).all() + ) + else: + actions.sort(key=lambda action: action.sequence_no) + + expected_sequences = list(range(first_sequence, last_allocated_sequence + 1)) + if [action.sequence_no for action in actions] != expected_sequences: + raise RuntimeError("Import action batch did not return its exact allocated sequence block") + sequence_cache[job.id] = last_allocated_sequence + return actions + + +def seed_action_sequence_cache( + session: AsyncSession, + job_id: int, + *, + last_sequence: int, +) -> None: + """Seed a lock-serialized worker session without another max query.""" + if isinstance(last_sequence, bool) or last_sequence < 0: + raise ValueError("Import action sequence seed must be non-negative") + cache = _action_sequence_cache(session) + cache[job_id] = max(cache.get(job_id, 0), last_sequence) + + +def _action_sequence_cache(session: AsyncSession) -> dict[int, int]: + cached = session.info.setdefault(_ACTION_SEQUENCE_CACHE_KEY, {}) + if not isinstance(cached, dict): + raise ValueError("Import action sequence cache has an invalid value") + return cached + + +def _series_rollback_lock_statement(series_id: int) -> Any: + """Lock the candidate Series against concurrent user-owned changes.""" + return ( + sa_select(Series) + .where(Series.id == series_id) + .execution_options(populate_existing=True) + .with_for_update() + ) + + +def _series_issue_rollback_lock_statement(series_id: int) -> Any: + """Lock every Issue before checking activity and deleting its Series.""" + return ( + sa_select(Issue.id).where(Issue.series_id == series_id).order_by(Issue.id).with_for_update() + ) + + +def _adopted_reference_string( + snapshot: dict[str, object], + key: str, + *, + optional: bool = False, +) -> str | None: + value = snapshot.get(key) + if value is None and optional: + return None + if not isinstance(value, str) or (not value and not optional): + raise ValueError(f"Invalid {key} in clean-library rollback action") + return value + + +def _adopted_reference_optional_int( + snapshot: dict[str, object], + key: str, +) -> int | None: + value = snapshot.get(key) + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"Invalid {key} in clean-library rollback action") + return value + + +def _adopted_reference_optional_float( + snapshot: dict[str, object], + key: str, +) -> float | None: + value = snapshot.get(key) + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int | float): + raise ValueError(f"Invalid {key} in clean-library rollback action") + return float(value) + + +async def _prepare_adopted_reference_rollback( + session: AsyncSession, + *, + payload: dict[str, Any], + managed_library_file: LibraryFile | None, +) -> tuple[_AdoptedReferenceRollback | None, str | None]: + raw_snapshot = payload.get("adopted_reference") + if raw_snapshot is None: + return None, None + if not isinstance(raw_snapshot, dict) or raw_snapshot.get("schema_version") != 1: + raise ValueError("Invalid adopted_reference in clean-library rollback action") + snapshot: dict[str, object] = dict(raw_snapshot) + source_imported_file_id = _positive_int( + snapshot.get("source_imported_file_id"), + "source_imported_file_id", + ) + source_library_file_id = _positive_int( + snapshot.get("source_library_file_id"), + "source_library_file_id", + ) + issue_id = _positive_int(snapshot.get("issue_id"), "issue_id") + library_root_id = _positive_int(snapshot.get("library_root_id"), "library_root_id") + source_series_id = _positive_int(snapshot.get("source_series_id"), "source_series_id") + file_path = _adopted_reference_string(snapshot, "file_path") + file_name = _adopted_reference_string(snapshot, "file_name") + file_modified_at_raw = _adopted_reference_string(snapshot, "file_modified_at") + assert file_path is not None + assert file_name is not None + assert file_modified_at_raw is not None + file_size = _non_negative_int(snapshot.get("file_size"), "file_size") + try: + file_format = FileFormat(str(snapshot.get("file_format") or "")) + match_confidence = MatchConfidence(str(snapshot.get("match_confidence") or "")) + file_modified_at = datetime.fromisoformat(file_modified_at_raw) + except ValueError as exc: + raise ValueError("Invalid file metadata in clean-library rollback action") from exc + if file_modified_at.tzinfo is None: + raise ValueError("Invalid file_modified_at in clean-library rollback action") + file_hash = _adopted_reference_string(snapshot, "file_hash", optional=True) + parsed_series = _adopted_reference_string(snapshot, "parsed_series", optional=True) + parsed_publisher = _adopted_reference_string(snapshot, "parsed_publisher", optional=True) + parsed_issue_number = _adopted_reference_optional_float(snapshot, "parsed_issue_number") + parsed_year = _adopted_reference_optional_int(snapshot, "parsed_year") + previous_series_path = _adopted_reference_string( + snapshot, + "previous_series_path", + optional=True, + ) + previous_series_library_root_id = _adopted_reference_optional_int( + snapshot, + "previous_series_library_root_id", + ) + previous_series_preferred_library_root_id = _adopted_reference_optional_int( + snapshot, + "previous_series_preferred_library_root_id", + ) + installed_series_path = _adopted_reference_string(snapshot, "installed_series_path") + installed_series_library_root_id = _positive_int( + snapshot.get("installed_series_library_root_id"), + "installed_series_library_root_id", + ) + installed_series_preferred_library_root_id = _adopted_reference_optional_int( + snapshot, + "installed_series_preferred_library_root_id", + ) + assert installed_series_path is not None + has_comicinfo = snapshot.get("has_comicinfo") + naming_snapshot = snapshot.get("naming_snapshot") + source_signature = snapshot.get("source_signature") + if ( + snapshot.get("storage_mode") != LibraryFileStorageMode.REFERENCED.value + or not isinstance(has_comicinfo, bool) + or not isinstance(naming_snapshot, dict) + or not isinstance(source_signature, dict) + or not source_signature + ): + raise ValueError("Invalid reference metadata in clean-library rollback action") + + source_imported_file = await session.get(ImportedFile, source_imported_file_id) + source_root = await session.get(LibraryRoot, library_root_id) + issue = await session.get(Issue, issue_id) + series = await session.get(Series, source_series_id) + installed_series_path_matches = False + if managed_library_file is not None: + try: + installed_series_path_matches = Path(installed_series_path).resolve( + strict=False + ) == Path(managed_library_file.file_path).parent.resolve(strict=False) + except (OSError, RuntimeError): + installed_series_path_matches = False + occupied_source = await session.scalar( + sa_select(LibraryFile.id).where(LibraryFile.file_path == file_path).limit(1) + ) + occupied_id = await session.get(LibraryFile, source_library_file_id) + source_row_reused = ( + managed_library_file is not None and managed_library_file.id == source_library_file_id + ) + if ( + managed_library_file is None + or managed_library_file.issue_id != issue_id + or source_imported_file is None + or source_imported_file.status is not ImportedFileStatus.IMPORTED + or source_imported_file.matched_issue_id != issue_id + or source_imported_file.file_path != file_path + or source_imported_file.library_file_id not in {None, source_library_file_id} + or source_root is None + or issue is None + or issue.series_id != source_series_id + or series is None + or managed_library_file.library_root_id != installed_series_library_root_id + or not installed_series_path_matches + or occupied_source is not None + or (occupied_id is not None and not source_row_reused) + ): + return None, ( + "The original Mylar reference no longer matches its clean-library rollback " + "record. Pullbox preserved the managed file for review." + ) + installed_series_state = _series_matches_folder_state( + series, + path=installed_series_path, + library_root_id=installed_series_library_root_id, + preferred_library_root_id=installed_series_preferred_library_root_id, + ) + previous_series_state = _series_matches_folder_state( + series, + path=previous_series_path, + library_root_id=previous_series_library_root_id, + preferred_library_root_id=previous_series_preferred_library_root_id, + ) + if not installed_series_state and not previous_series_state: + return None, ( + "The series storage location changed after clean-library adoption. Pullbox " + "preserved the managed file for review." + ) + try: + current_signature = build_file_identity_signature(Path(file_path)) + except (ConfigurationError, OSError, RuntimeError, ValueError): + return None, ( + "The original Mylar source is unavailable. Pullbox preserved the managed file " + "for review." + ) + if current_signature != source_signature: + return None, ( + "The original Mylar source changed after adoption. Pullbox preserved the " + "managed file for review." + ) + return ( + _AdoptedReferenceRollback( + source_imported_file=source_imported_file, + source_library_file_id=source_library_file_id, + file_path=file_path, + file_name=file_name, + file_size=file_size, + file_format=file_format, + file_hash=file_hash, + file_modified_at=file_modified_at, + match_confidence=match_confidence, + parsed_series=parsed_series, + parsed_issue_number=parsed_issue_number, + parsed_year=parsed_year, + parsed_publisher=parsed_publisher, + has_comicinfo=has_comicinfo, + naming_snapshot=dict(naming_snapshot), + source_signature=dict(source_signature), + issue_id=issue_id, + library_root_id=library_root_id, + series=series, + previous_series_path=previous_series_path, + previous_series_library_root_id=previous_series_library_root_id, + previous_series_preferred_library_root_id=(previous_series_preferred_library_root_id), + ), + None, + ) + + +async def _restore_adopted_reference( + session: AsyncSession, + adopted: _AdoptedReferenceRollback, + *, + reusable_library_file: LibraryFile | None = None, +) -> None: + restored = reusable_library_file or LibraryFile(id=adopted.source_library_file_id) + restored.file_path = adopted.file_path + restored.file_name = adopted.file_name + restored.file_size = adopted.file_size + restored.file_format = adopted.file_format + restored.file_hash = adopted.file_hash + restored.file_modified_at = adopted.file_modified_at + restored.match_confidence = adopted.match_confidence + restored.parsed_series = adopted.parsed_series + restored.parsed_issue_number = adopted.parsed_issue_number + restored.parsed_year = adopted.parsed_year + restored.parsed_publisher = adopted.parsed_publisher + restored.has_comicinfo = adopted.has_comicinfo + restored.naming_snapshot = adopted.naming_snapshot + restored.storage_mode = LibraryFileStorageMode.REFERENCED + restored.source_signature = adopted.source_signature + restored.issue_id = adopted.issue_id + restored.library_root_id = adopted.library_root_id + if reusable_library_file is None: + session.add(restored) + await session.flush() + adopted.source_imported_file.library_file_id = restored.id + issue = await session.get(Issue, adopted.issue_id) + if issue is not None: + issue.status = IssueStatus.OWNED + adopted.series.path = adopted.previous_series_path + adopted.series.library_root_id = adopted.previous_series_library_root_id + adopted.series.preferred_library_root_id = adopted.previous_series_preferred_library_root_id + + async def rollback_action( session: AsyncSession, *, @@ -80,53 +772,130 @@ async def rollback_action( delete_series: DeleteSeriesForRollback, ) -> None: """Reverse a recorded import action in reverse execution order.""" - if action_type == "library_file_registered": + action = await session.get(ImportJobAction, action_id) + if action is None: + return + if action.action_type != action_type or dict(action.payload or {}) != payload: + raise ValueError("Import action changed after it was selected for rollback") + + if action_type == _STORY_ARC_MANAGED_PLACEMENT_ACTION: + await _rollback_import_managed_story_arc_placement(session, action, payload) + elif action_type == "story_arc_referenced_placement_attached": + await _rollback_attached_story_arc_reference(session, action, payload) + elif action_type == "story_arc_membership_created": + await _rollback_created_story_arc_membership(session, action, payload) + elif action_type == "story_arc_membership_updated": + await _rollback_updated_story_arc_membership(session, action, payload) + elif action_type == "story_arc_external_identity_created": + await _rollback_created_story_arc_external_identity(session, action, payload) + elif action_type == "story_arc_policy_updated": + await _rollback_story_arc_policy_update(session, action, payload) + elif action_type == "story_arc_created": + await _rollback_created_story_arc(session, action, payload) + elif action_type == "library_file_registered": library_file_id = int(payload.get("library_file_id") or 0) destination_path = Path(str(payload.get("destination_path") or "")) original_source_path = Path(str(payload.get("original_source_path") or "")) transfer_method = str(payload.get("transfer_method") or "move") + storage_mode = str(payload.get("storage_mode") or "") + referenced_file = storage_mode == "referenced" or transfer_method == "leave_in_place" original_trash_path = str(payload.get("original_trash_path") or "") - created_series_folder = bool(payload.get("created_series_folder")) - created_series_folder_path_raw = str(payload.get("created_series_folder_path") or "") permission_restores = list(payload.get("permission_restores") or []) library_file = await session.get(LibraryFile, library_file_id) - if library_file is not None: + adopted_reference, adoption_block_reason = await _prepare_adopted_reference_rollback( + session, + payload=payload, + managed_library_file=library_file, + ) + if adoption_block_reason is not None: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = adoption_block_reason + action.rolled_back_at = None + await session.flush() + return + source_reappeared = ( + not referenced_file + and transfer_method == "move" + and os.path.lexists(original_source_path) + and ( + ( + os.path.lexists(destination_path) + and destination_path.resolve(strict=False) + != original_source_path.resolve(strict=False) + ) + or (bool(original_trash_path) and os.path.lexists(original_trash_path)) + ) + ) + if source_reappeared: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The original move source reappeared after import. Pullbox preserved both " + "the source and managed destination for review." + ) + action.rolled_back_at = None + await session.flush() + return + if ( + not referenced_file + and os.path.lexists(destination_path) + and not _managed_destination_matches_rollback_action( + destination_path, + library_file=library_file, + expected_signature=payload.get("destination_signature"), + ) + ): + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "Managed library file changed after import or no longer matches its " + "rollback ownership record. Pullbox preserved the file." + ) + action.rolled_back_at = None + await session.flush() + return + reusable_adopted_file = ( + library_file + if adopted_reference is not None + and library_file is not None + and library_file.id == adopted_reference.source_library_file_id + else None + ) + if library_file is not None and reusable_adopted_file is None: await session.delete(library_file) - if transfer_method in {"move", "leave_in_place"}: - if original_trash_path: - restore_file_from_utility_trash(Path(original_trash_path), original_source_path) - if destination_path.exists(): - destination_path.unlink(missing_ok=True) - elif destination_path.exists(): - original_source_path.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(destination_path), str(original_source_path)) - elif destination_path.exists(): - destination_path.unlink(missing_ok=True) - - for entry in permission_restores: - restore_path = Path(str(entry.get("path") or "")) - restore_mode = entry.get("mode") - if not restore_path or restore_mode is None or not restore_path.exists(): - continue - try: - restore_path.chmod(int(restore_mode)) - except OSError: - continue - - if created_series_folder and created_series_folder_path_raw: - created_series_folder_path = Path(created_series_folder_path_raw) - else: - created_series_folder_path = None + if not referenced_file: + if transfer_method == "move": + if original_trash_path: + restore_file_from_utility_trash(Path(original_trash_path), original_source_path) + if os.path.lexists(destination_path): + destination_path.unlink(missing_ok=True) + elif os.path.lexists(destination_path): + original_source_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(destination_path), str(original_source_path)) + elif os.path.lexists(destination_path): + destination_path.unlink(missing_ok=True) - if created_series_folder_path is not None and created_series_folder_path.exists(): - try: - next(created_series_folder_path.iterdir()) - except StopIteration: - created_series_folder_path.rmdir() - except OSError: - pass + for entry in permission_restores: + restore_path = Path(str(entry.get("path") or "")) + restore_mode = entry.get("mode") + if not restore_path or restore_mode is None or not restore_path.exists(): + continue + try: + restore_path.chmod(int(restore_mode)) + except OSError: + continue + + _cleanup_import_created_directories( + payload, + destination_parent=destination_path.parent, + ) + if adopted_reference is not None: + await session.flush() + await _restore_adopted_reference( + session, + adopted_reference, + reusable_library_file=reusable_adopted_file, + ) elif action_type == "library_file_placement_started": destination_path_raw = str(payload.get("destination_path") or "") @@ -140,13 +909,17 @@ async def rollback_action( Path(artifact_source_path_raw) if artifact_source_path_raw else None ) transfer_method = str(payload.get("transfer_method") or "move") - created_series_folder = bool(payload.get("created_series_folder")) - created_series_folder_path_raw = str(payload.get("created_series_folder_path") or "") temp_paths = [Path(str(path)) for path in payload.get("temp_paths") or [] if str(path)] - - for temp_path in temp_paths: - if temp_path.exists() and temp_path.is_file(): - temp_path.unlink(missing_ok=True) + surviving_temp_paths = [path for path in temp_paths if os.path.lexists(path)] + if surviving_temp_paths: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "Import staging artifacts have no completed ownership signature. " + "Pullbox preserved them for review." + ) + action.rolled_back_at = None + await session.flush() + return destination_is_original_source = ( partial_destination_path is not None @@ -156,46 +929,360 @@ async def rollback_action( ) if ( partial_destination_path is not None - and partial_destination_path.exists() + and os.path.lexists(partial_destination_path) and not destination_is_original_source ): + placement_completed = payload.get("placement_completed") is True + destination_signature = payload.get("destination_signature") + if not placement_completed or not _path_matches_signature( + partial_destination_path, + destination_signature, + ): + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "Import destination is incomplete, changed, or lacks durable ownership " + "evidence. Pullbox preserved it for review." + ) + action.rolled_back_at = None + await session.flush() + return can_restore_move = ( transfer_method == "move" and partial_original_source_path is not None and partial_artifact_source_path is not None and partial_artifact_source_path == partial_original_source_path - and not partial_original_source_path.exists() + and not os.path.lexists(partial_original_source_path) ) if can_restore_move: assert partial_original_source_path is not None partial_original_source_path.parent.mkdir(parents=True, exist_ok=True) shutil.move(str(partial_destination_path), str(partial_original_source_path)) + elif ( + transfer_method == "move" + and partial_original_source_path is not None + and partial_artifact_source_path == partial_original_source_path + and os.path.lexists(partial_original_source_path) + ): + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The original move source reappeared after import. Pullbox preserved both " + "the source and managed destination for review." + ) + action.rolled_back_at = None + await session.flush() + return elif partial_destination_path.is_file() or partial_destination_path.is_symlink(): partial_destination_path.unlink(missing_ok=True) - if created_series_folder and created_series_folder_path_raw: - created_series_folder_path = Path(created_series_folder_path_raw) - if created_series_folder_path.exists(): - try: - next(created_series_folder_path.iterdir()) - except StopIteration: - created_series_folder_path.rmdir() - except OSError: - pass + if partial_destination_path is not None: + _cleanup_import_created_directories( + payload, + destination_parent=partial_destination_path.parent, + ) + + elif action_type == "series_preferred_root_updated": + series_id = _positive_int(payload.get("series_id"), "series_id") + old_root_id = _optional_positive_int( + payload.get("old_preferred_library_root_id"), + "old_preferred_library_root_id", + ) + new_root_id = _positive_int( + payload.get("new_preferred_library_root_id"), + "new_preferred_library_root_id", + ) + series = await session.get(Series, series_id) + if series is not None: + if series.preferred_library_root_id != new_root_id: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The series preferred destination changed after import. " + "Pullbox preserved the current choice for review." + ) + action.rolled_back_at = None + await session.flush() + return + series.preferred_library_root_id = old_root_id + + elif action_type == "series_monitoring_updated": + series_id = _positive_int(payload.get("series_id"), "series_id") + import_series_id = _positive_int( + payload.get("import_series_id"), + "import_series_id", + ) + previous_monitored = payload.get("previous_monitored") + installed_monitored = payload.get("installed_monitored") + if not isinstance(previous_monitored, bool) or not isinstance( + installed_monitored, + bool, + ): + raise ValueError("Monitoring rollback states must be booleans") + if previous_monitored == installed_monitored: + raise ValueError("Monitoring rollback states must differ") + + series = await session.get(Series, series_id) + if series is not None: + imported_series = await session.get(ImportedSeries, import_series_id) + if ( + imported_series is None + or imported_series.import_job_id != action.import_job_id + or imported_series.series_id not in {None, series_id} + ): + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The Series monitoring change is no longer owned by this import. " + "Pullbox preserved the current setting." + ) + action.rolled_back_at = None + await session.flush() + return + expected_monitoring_state = ( + previous_monitored + if action.status == ImportJobActionStatus.ROLLED_BACK + else installed_monitored + ) + if bool(series.monitored) != expected_monitoring_state: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "Series monitoring changed after import. Pullbox preserved the current setting." + ) + action.rolled_back_at = None + await session.flush() + return + series.monitored = previous_monitored + + elif action_type == "series_folder_created": + series_id = _positive_int(payload.get("series_id"), "series_id") + import_series_id = _positive_int( + payload.get("import_series_id"), + "import_series_id", + ) + previous_series_path = payload.get("previous_series_path") + if previous_series_path is not None and not isinstance(previous_series_path, str): + raise ValueError("previous_series_path must be a string or null") + previous_library_root_id = _optional_positive_int( + payload.get("previous_library_root_id"), + "previous_library_root_id", + ) + previous_preferred_root_id = _optional_positive_int( + payload.get("previous_preferred_library_root_id"), + "previous_preferred_library_root_id", + ) + ownership = _validated_series_folder_action_ownership( + payload.get("series_folder_ownership") + ) + if ownership is None: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The import journal does not contain valid series-folder ownership evidence. " + "Pullbox preserved the folder for manual recovery." + ) + action.rolled_back_at = None + await session.flush() + return + + series = await session.get(Series, series_id) + if series is not None: + imported_series = await session.get(ImportedSeries, import_series_id) + if ( + imported_series is None + or imported_series.import_job_id != action.import_job_id + or imported_series.series_id not in {None, series_id} + ): + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The imported series folder is no longer owned by this import. " + "Pullbox preserved it for manual recovery." + ) + action.rolled_back_at = None + await session.flush() + return + installed_state = _series_matches_installed_folder_state(series, ownership) + restored_state = _series_matches_folder_state( + series, + path=previous_series_path, + library_root_id=previous_library_root_id, + preferred_library_root_id=previous_preferred_root_id, + ) + if not installed_state and not restored_state: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The series folder or storage selection changed after import. " + "Pullbox preserved the current state." + ) + action.rolled_back_at = None + await session.flush() + return + + folder_error = _series_folder_rollback_block_reason( + payload, + series=series, + require_installed_state=False, + ) + if folder_error is not None: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = folder_error + action.rolled_back_at = None + await session.flush() + return + + _cleanup_owned_series_folder_directories(payload) + if series is not None: + series.path = previous_series_path + series.library_root_id = previous_library_root_id + series.preferred_library_root_id = previous_preferred_root_id + + elif action_type == "series_cover_path_updated": + series_id = _positive_int(payload.get("series_id"), "series_id") + import_series_id = _positive_int( + payload.get("import_series_id"), + "import_series_id", + ) + previous_cover_path = payload.get("previous_cover_path") + installed_cover_path = payload.get("installed_cover_path") + if previous_cover_path is not None and not isinstance(previous_cover_path, str): + raise ValueError("previous_cover_path must be a string or null") + if installed_cover_path is not None and not isinstance(installed_cover_path, str): + raise ValueError("installed_cover_path must be a string or null") + if previous_cover_path == installed_cover_path: + raise ValueError("Cover-path rollback states must differ") + + series = await session.get(Series, series_id) + if series is not None: + imported_series = await session.get(ImportedSeries, import_series_id) + if ( + imported_series is None + or imported_series.import_job_id != action.import_job_id + or imported_series.series_id not in {None, series_id} + ): + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The Series cover-path change is no longer owned by this import. " + "Pullbox preserved the current value." + ) + action.rolled_back_at = None + await session.flush() + return + expected_cover_path = ( + previous_cover_path + if action.status == ImportJobActionStatus.ROLLED_BACK + else installed_cover_path + ) + if series.cover_path != expected_cover_path: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The Series cover path changed after import. Pullbox preserved the current " + "value." + ) + action.rolled_back_at = None + await session.flush() + return + series.cover_path = previous_cover_path + + elif action_type == "series_cover_cache_created": + series_id = _positive_int(payload.get("series_id"), "series_id") + import_series_id = _positive_int( + payload.get("import_series_id"), + "import_series_id", + ) + installed_cover_path = payload.get("installed_cover_path") + previous_cover_path = payload.get("previous_cover_path") + if not isinstance(installed_cover_path, str) or not installed_cover_path: + raise ValueError("installed_cover_path must be a non-empty string") + if previous_cover_path is not None and not isinstance(previous_cover_path, str): + raise ValueError("previous_cover_path must be a string or null") + + cover_error = _cover_cache_rollback_block_reason(payload) + if cover_error is not None: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = cover_error + action.rolled_back_at = None + await session.flush() + return + + series = await session.get(Series, series_id) + if series is not None: + imported_series = await session.get(ImportedSeries, import_series_id) + if ( + imported_series is None + or imported_series.import_job_id != action.import_job_id + or imported_series.series_id not in {None, series_id} + ): + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The imported cover is no longer owned by this import. " + "Pullbox preserved it for manual recovery." + ) + action.rolled_back_at = None + await session.flush() + return + artifact_missing = not os.path.lexists( + Path(payload["cover_cache_ownership"]["artifact_path"]) + ) + already_restored = series.cover_path == previous_cover_path and artifact_missing + if series.cover_path != installed_cover_path and not already_restored: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = ( + "The series cover changed after import. Pullbox preserved the current cover." + ) + action.rolled_back_at = None + await session.flush() + return + + _remove_owned_cover_cache_artifact(payload) + if series is not None: + series.cover_path = previous_cover_path elif action_type == "series_created": series_id = int(payload.get("series_id") or 0) if series_id: - with contextlib.suppress(NotFoundError): - await delete_series( + series = await session.scalar(_series_rollback_lock_statement(series_id)) + folder_error = _series_folder_rollback_block_reason(payload, series=series) + if folder_error is not None: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = folder_error + action.rolled_back_at = None + await session.flush() + return + cover_error = _cover_cache_rollback_block_reason(payload) + if cover_error is not None: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = cover_error + action.rolled_back_at = None + await session.flush() + return + if series is not None: + unsafe_reason = await _created_series_rollback_block_reason( session, - series_id, - delete_files=False, - delete_folder=True, + action=action, + series=series, + payload=payload, ) - # A series-created action can be replayed after a partial rollback or - # after multiple import rows converged on the same real series. Missing - # here means the rollback objective is already satisfied. + if unsafe_reason is not None: + action.status = ImportJobActionStatus.ROLLBACK_FAILED + action.error_message = unsafe_reason + action.rolled_back_at = None + await session.flush() + return + + with contextlib.suppress(NotFoundError): + # SeriesService.delete has broader user-facing behavior: it can + # cancel downloads and recursively delete a folder. The guards + # above prove there are no related files/activity. Folder + # ownership is not durably proven, so even an empty directory is + # preserved rather than being inferred safe to remove. + await delete_series( + session, + series_id, + delete_files=False, + delete_folder=False, + ) + # A series-created action can be replayed after a partial rollback. + # Missing here means the rollback objective is already satisfied. + # This cleanup is intentionally replayable. A prior rollback may have + # deleted the Series and its cover leaf before interruption, while the + # import-owned empty cache ancestors still need removal. + _remove_owned_cover_cache_artifact(payload) + _cleanup_owned_series_folder_directories(payload) elif action_type == "series_folder_renamed": series_id = int(payload.get("series_id") or 0) @@ -224,14 +1311,1641 @@ async def rollback_action( int(old_library_root_id_raw) if old_library_root_id_raw is not None else None ) - action = await session.get(ImportJobAction, action_id) - if action is None: + elif action_type == "library_root_policy_applied": + from pullbox.services.import_root_policy_activation import ( + rollback_future_root_policy, + ) + + action = await session.get(ImportJobAction, action_id) + if action is None: + return + job = await session.get(ImportJob, action.import_job_id) + if job is None: + raise NotFoundError("ImportJob", action.import_job_id) + await rollback_future_root_policy(session, job=job, action=action) return + action.status = ImportJobActionStatus.ROLLED_BACK action.rolled_back_at = datetime.now(UTC) await session.flush() +def _validated_series_folder_ownership(value: object) -> dict[str, Any] | None: + if not isinstance(value, dict) or value.get("schema_version") != 1: + return None + folder_path_raw = value.get("folder_path") + boundary_path_raw = value.get("ownership_boundary_path") + directory_paths_raw = value.get("created_directory_paths") + if not isinstance(folder_path_raw, str) or not folder_path_raw: + return None + if not isinstance(boundary_path_raw, str) or not boundary_path_raw: + return None + if not isinstance(directory_paths_raw, list): + return None + + try: + folder_resolved = Path(folder_path_raw).resolve(strict=False) + boundary_resolved = Path(boundary_path_raw).resolve(strict=False) + folder_relative = folder_resolved.relative_to(boundary_resolved) + except (OSError, RuntimeError, ValueError): + return None + if not folder_relative.parts: + return None + + validated_paths: list[str] = [] + for raw_path in directory_paths_raw: + if not isinstance(raw_path, str) or not raw_path: + return None + try: + path_resolved = Path(raw_path).resolve(strict=False) + relative = path_resolved.relative_to(boundary_resolved) + folder_resolved.relative_to(path_resolved) + except (OSError, RuntimeError, ValueError): + return None + if not relative.parts: + return None + if raw_path not in validated_paths: + validated_paths.append(raw_path) + if validated_paths: + try: + last_resolved = Path(validated_paths[-1]).resolve(strict=False) + except (OSError, RuntimeError): + return None + if last_resolved != folder_resolved: + return None + + return { + "schema_version": 1, + "folder_path": folder_path_raw, + "ownership_boundary_path": boundary_path_raw, + "created_directory_paths": validated_paths, + } + + +def _series_path_matches_folder_ownership( + series: Series, + ownership: dict[str, Any], +) -> bool: + if not series.path: + return False + try: + return Path(series.path).resolve(strict=False) == Path(ownership["folder_path"]).resolve( + strict=False + ) + except (OSError, RuntimeError): + return False + + +def _folder_ownership_with_installed_state( + ownership: dict[str, Any], + series: Series, +) -> dict[str, Any]: + return { + **ownership, + "installed_library_root_id": series.library_root_id, + "installed_preferred_library_root_id": series.preferred_library_root_id, + } + + +def _validated_series_folder_action_ownership(value: object) -> dict[str, Any] | None: + ownership = _validated_series_folder_ownership(value) + if ownership is None or not isinstance(value, dict): + return None + if ( + "installed_library_root_id" not in value + or "installed_preferred_library_root_id" not in value + ): + return None + installed_root_id = value.get("installed_library_root_id") + installed_preferred_root_id = value.get("installed_preferred_library_root_id") + if installed_root_id is not None and ( + not isinstance(installed_root_id, int) + or isinstance(installed_root_id, bool) + or installed_root_id <= 0 + ): + return None + if installed_preferred_root_id is not None and ( + not isinstance(installed_preferred_root_id, int) + or isinstance(installed_preferred_root_id, bool) + or installed_preferred_root_id <= 0 + ): + return None + return { + **ownership, + "installed_library_root_id": installed_root_id, + "installed_preferred_library_root_id": installed_preferred_root_id, + } + + +def _series_matches_folder_state( + series: Series, + *, + path: str | None, + library_root_id: int | None, + preferred_library_root_id: int | None, +) -> bool: + if series.library_root_id != library_root_id: + return False + if series.preferred_library_root_id != preferred_library_root_id: + return False + if series.path is None or path is None: + return series.path is path + try: + return Path(series.path).resolve(strict=False) == Path(path).resolve(strict=False) + except (OSError, RuntimeError): + return False + + +def _series_matches_installed_folder_state( + series: Series, + ownership: dict[str, Any], +) -> bool: + return _series_matches_folder_state( + series, + path=ownership["folder_path"], + library_root_id=ownership["installed_library_root_id"], + preferred_library_root_id=ownership["installed_preferred_library_root_id"], + ) + + +def _series_folder_rollback_block_reason( + payload: dict[str, Any], + *, + series: Series | None, + require_installed_state: bool = True, +) -> str | None: + raw_ownership = payload.get("series_folder_ownership") + if raw_ownership is None: + return None + ownership = _validated_series_folder_action_ownership(raw_ownership) + if ownership is None: + return ( + "The import journal does not contain valid series-folder ownership evidence. " + "Pullbox preserved the folder for manual recovery." + ) + if ( + require_installed_state + and series is not None + and not _series_matches_installed_folder_state(series, ownership) + ): + return ( + "The import-created series folder or storage selection changed after import. " + "Pullbox preserved the current state." + ) + if not ownership["created_directory_paths"]: + return None + + folder_path = Path(ownership["folder_path"]) + if not os.path.lexists(folder_path): + return None + if folder_path.is_symlink() or not folder_path.is_dir(): + return ( + "The import-created series folder changed after import. Pullbox preserved the " + "current path for manual recovery." + ) + try: + next(folder_path.iterdir()) + except StopIteration: + return None + except OSError: + return ( + "The import-created series folder could not be verified as empty. Pullbox " + "preserved it for manual recovery." + ) + return ( + "The import-created series folder contains later files. Pullbox preserved the series " + "and folder for manual recovery." + ) + + +def _cleanup_owned_series_folder_directories(payload: dict[str, Any]) -> None: + ownership = _validated_series_folder_action_ownership(payload.get("series_folder_ownership")) + if ownership is None: + return + _remove_empty_directories(Path(path) for path in ownership["created_directory_paths"]) + + +def _validated_cover_cache_ownership(value: object) -> dict[str, Any] | None: + if not isinstance(value, dict) or value.get("schema_version") != 1: + return None + base_path_raw = value.get("base_path") + boundary_path_raw = value.get("ownership_boundary_path") + directory_paths_raw = value.get("created_directory_paths") + artifact_path_raw = value.get("artifact_path") + artifact_signature = value.get("artifact_signature") + if not isinstance(base_path_raw, str) or not base_path_raw: + return None + if not isinstance(boundary_path_raw, str) or not boundary_path_raw: + return None + if not isinstance(directory_paths_raw, list): + return None + if not isinstance(artifact_path_raw, str) or not artifact_path_raw: + return None + if not isinstance(artifact_signature, dict) or not artifact_signature: + return None + + base_path = Path(base_path_raw) + boundary_path = Path(boundary_path_raw) + try: + base_resolved = base_path.resolve(strict=False) + boundary_resolved = boundary_path.resolve(strict=False) + base_resolved.relative_to(boundary_resolved) + artifact_resolved = Path(artifact_path_raw).resolve(strict=False) + artifact_relative = artifact_resolved.relative_to(base_resolved) + except (OSError, RuntimeError, ValueError): + return None + if not artifact_relative.parts: + return None + artifact_parent_resolved = artifact_resolved.parent + validated_paths: list[str] = [] + for raw_path in directory_paths_raw: + if not isinstance(raw_path, str) or not raw_path: + return None + path = Path(raw_path) + try: + path_resolved = path.resolve(strict=False) + relative_path = path_resolved.relative_to(boundary_resolved) + except (OSError, RuntimeError, ValueError): + return None + if not relative_path.parts: + return None + try: + artifact_parent_resolved.relative_to(path_resolved) + except ValueError: + return None + if raw_path not in validated_paths: + validated_paths.append(raw_path) + return { + "schema_version": 1, + "base_path": base_path_raw, + "ownership_boundary_path": boundary_path_raw, + "created_directory_paths": validated_paths, + "artifact_path": artifact_path_raw, + "artifact_signature": dict(artifact_signature), + } + + +def _cover_cache_rollback_block_reason(payload: dict[str, Any]) -> str | None: + raw_ownership = payload.get("cover_cache_ownership") + if raw_ownership is None: + return None + ownership = _validated_cover_cache_ownership(raw_ownership) + if ownership is None: + return ( + "The import journal does not contain valid cover ownership evidence. " + "Pullbox preserved the cover for manual recovery." + ) + artifact_path = Path(ownership["artifact_path"]) + if not os.path.lexists(artifact_path): + return None + if not artifact_path.is_file() or not _path_matches_signature( + artifact_path, + ownership["artifact_signature"], + ): + return ( + "The imported cover changed after import. Pullbox preserved the current cover " + "for manual recovery." + ) + return None + + +def _remove_owned_cover_cache_artifact(payload: dict[str, Any]) -> None: + ownership = _validated_cover_cache_ownership(payload.get("cover_cache_ownership")) + if ownership is None: + return + artifact_path = Path(ownership["artifact_path"]) + if os.path.lexists(artifact_path) and artifact_path.is_file(): + artifact_path.unlink() + _remove_empty_directories(Path(path) for path in ownership["created_directory_paths"]) + + +def _cleanup_import_created_directories( + payload: dict[str, Any], + *, + destination_parent: Path, +) -> None: + raw_paths = payload.get("created_directory_paths") + if raw_paths is None: + if not bool(payload.get("created_series_folder")): + return + legacy_path = str(payload.get("created_series_folder_path") or "") + if not legacy_path: + return + try: + if Path(legacy_path).resolve(strict=False) != destination_parent.resolve(strict=False): + return + except (OSError, RuntimeError): + return + _remove_empty_directories((Path(legacy_path),)) + return + if not isinstance(raw_paths, list): + return + + boundary_raw = payload.get("directory_ownership_boundary_path") + if not isinstance(boundary_raw, str) or not boundary_raw: + return + try: + boundary_resolved = Path(boundary_raw).resolve(strict=False) + destination_parent_resolved = destination_parent.resolve(strict=False) + destination_relative = destination_parent_resolved.relative_to(boundary_resolved) + except (OSError, RuntimeError, ValueError): + return + if not destination_relative.parts: + return + + owned_paths: list[Path] = [] + for raw_path in raw_paths: + if not isinstance(raw_path, str) or not raw_path: + continue + path = Path(raw_path) + try: + path_resolved = path.resolve(strict=False) + relative = path_resolved.relative_to(boundary_resolved) + destination_parent_resolved.relative_to(path_resolved) + except (OSError, RuntimeError, ValueError): + continue + if not relative.parts: + continue + owned_paths.append(path) + _remove_empty_directories(owned_paths) + + +def _remove_empty_directories(paths: Iterable[Path]) -> None: + unique_paths = {Path(path) for path in paths} + for directory in sorted(unique_paths, key=lambda path: len(path.parts), reverse=True): + try: + directory.rmdir() + except (FileNotFoundError, OSError): + continue + + +def _series_user_owned_snapshot(series: Series) -> dict[str, Any]: + status_override = series.status_override + return { + "schema_version": 1, + "comicvine_id": series.comicvine_id, + "monitored": bool(series.monitored), + "status_override": status_override.value if status_override is not None else None, + "alternate_names": list(series.alternate_names or []), + "parent_series_id": series.parent_series_id, + "preferred_library_root_id": series.preferred_library_root_id, + } + + +async def _created_series_rollback_block_reason( + session: AsyncSession, + *, + action: ImportJobAction, + series: Series, + payload: dict[str, Any], +) -> str | None: + """Return why a job-created series is no longer safe to remove.""" + # The locked Series row prevents concurrent Series changes and new rows + # that reference it. Locking its Issues also prevents concurrent issue + # state changes and new dependent activity until deletion completes. + await session.execute(_series_issue_rollback_lock_statement(series.id)) + + import_series_id = _optional_positive_int(payload.get("import_series_id"), "import_series_id") + expected_snapshot = payload.get("series_ownership_snapshot") + expected_issue_snapshot = payload.get("issue_ownership_snapshot") + if ( + import_series_id is None + or not isinstance(expected_snapshot, dict) + or not isinstance(expected_issue_snapshot, dict) + ): + return ( + "The import journal does not contain enough series ownership evidence. " + "Pullbox preserved the series for manual recovery." + ) + + imported_series = await session.get(ImportedSeries, import_series_id) + if ( + imported_series is None + or imported_series.import_job_id != action.import_job_id + or imported_series.series_id not in {None, series.id} + ): + return ( + "The import-created series is no longer owned exclusively by this import. " + "Pullbox preserved it for manual recovery." + ) + if expected_snapshot != _series_user_owned_snapshot(series): + return ( + "The import-created series changed after import. Pullbox preserved the current " + "series and its data for manual recovery." + ) + + issue_state_changed = await _created_series_issue_state_changed( + session, + action=action, + series=series, + import_series_id=import_series_id, + expected_snapshot=expected_issue_snapshot, + monitored=bool(expected_snapshot.get("monitored")), + ) + if issue_state_changed: + return ( + "An issue in the import-created series changed after import. Pullbox preserved " + "the series and its issue state for manual recovery." + ) + + other_import_reference = await session.scalar( + sa_select(ImportedSeries.id) + .where( + ImportedSeries.series_id == series.id, + ImportedSeries.import_job_id != action.import_job_id, + ) + .limit(1) + ) + child_series = await session.scalar( + sa_select(Series.id).where(Series.parent_series_id == series.id).limit(1) + ) + issue_ids = sa_select(Issue.id).where(Issue.series_id == series.id) + related_activity_checks = ( + sa_select(LibraryFile.id).where(LibraryFile.issue_id.in_(issue_ids)).limit(1), + sa_select(DownloadHistory.id).where(DownloadHistory.issue_id.in_(issue_ids)).limit(1), + sa_select(IssueReaderState.id).where(IssueReaderState.issue_id.in_(issue_ids)).limit(1), + sa_select(PendingMatch.id).where(PendingMatch.issue_id.in_(issue_ids)).limit(1), + sa_select(SearchLog.id).where(SearchLog.issue_id.in_(issue_ids)).limit(1), + sa_select(DirectAcquisitionAttempt.id) + .where(DirectAcquisitionAttempt.issue_id.in_(issue_ids)) + .limit(1), + sa_select(IssueStoryArc.id).where(IssueStoryArc.issue_id.in_(issue_ids)).limit(1), + sa_select(BlocklistEntry.id) + .where( + or_( + BlocklistEntry.series_id == series.id, + BlocklistEntry.issue_id.in_(issue_ids), + ) + ) + .limit(1), + sa_select(Issue.id) + .where(Issue.series_id == series.id, Issue.manual_skip.is_(True)) + .limit(1), + ) + has_related_activity = False + for statement in related_activity_checks: + if await session.scalar(statement) is not None: + has_related_activity = True + break + if other_import_reference is not None or child_series is not None or has_related_activity: + return ( + "The import-created series has later files or activity. Pullbox preserved the " + "series and related data for manual recovery." + ) + return None + + +async def _created_series_issue_state_changed( + session: AsyncSession, + *, + action: ImportJobAction, + series: Series, + import_series_id: int, + expected_snapshot: dict[str, Any], + monitored: bool, +) -> bool: + """Detect user-owned issue-state changes while allowing import-owned OWNED state.""" + imported_issue_ids = set( + ( + await session.scalars( + sa_select(ImportedFile.matched_issue_id).where( + ImportedFile.import_job_id == action.import_job_id, + ImportedFile.import_series_id == import_series_id, + ImportedFile.status == ImportedFileStatus.IMPORTED, + ImportedFile.matched_issue_id.is_not(None), + ) + ) + ).all() + ) + current_rows = ( + await session.execute( + sa_select(Issue.id, Issue.status, Issue.manual_skip) + .where(Issue.series_id == series.id) + .order_by(Issue.id) + ) + ).all() + current_ids = {str(issue_id) for issue_id, _status, _manual_skip in current_rows} + if not set(expected_snapshot).issubset(current_ids): + return True + + default_status = IssueStatus.WANTED if monitored else IssueStatus.SKIPPED + for issue_id, status, manual_skip in current_rows: + expected = expected_snapshot.get(str(issue_id)) + if expected is None: + if issue_id in imported_issue_ids: + if bool(manual_skip) or status != IssueStatus.OWNED: + return True + elif bool(manual_skip) or status != default_status: + return True + continue + if not isinstance(expected, dict): + return True + expected_manual_skip = expected.get("manual_skip") + expected_status = expected.get("status") + if not isinstance(expected_manual_skip, bool) or not isinstance(expected_status, str): + return True + if bool(manual_skip) != expected_manual_skip: + return True + if issue_id in imported_issue_ids: + if status != IssueStatus.OWNED: + return True + elif status.value != expected_status: + return True + return False + + +def _managed_destination_matches_rollback_action( + destination_path: Path, + *, + library_file: LibraryFile | None, + expected_signature: object, +) -> bool: + """Require path, ownership, and creation signature before managed removal.""" + if library_file is None or library_file.storage_mode != LibraryFileStorageMode.MANAGED: + return False + if Path(library_file.file_path).resolve(strict=False) != destination_path.resolve(strict=False): + return False + if not isinstance(expected_signature, dict) or not expected_signature: + return False + try: + current_signature = build_managed_placement_signature(destination_path) + except (ConfigurationError, OSError, RuntimeError, ValueError): + return False + return expected_signature == current_signature + + +def _path_matches_signature(path: Path, expected_signature: object) -> bool: + if not isinstance(expected_signature, dict) or not expected_signature: + return False + try: + return build_managed_placement_signature(path) == expected_signature + except (ConfigurationError, OSError, RuntimeError, ValueError): + return False + + +async def _rollback_import_managed_story_arc_placement( + session: AsyncSession, + action: ImportJobAction, + payload: dict[str, Any], +) -> None: + """Cancel or remove exactly one placement owned by an import action. + + The durable sync-work row is the rollback checkpoint. Running work is + asked to cancel and deliberately stops the reverse action walk until its + worker reaches a terminal state. Filesystem removal is delegated to the + ownership- and fingerprint-aware Story Arc placement lifecycle. + """ + if action.status is ImportJobActionStatus.ROLLED_BACK: + return + if action.status is not ImportJobActionStatus.COMPLETED: + raise ValueError("Import Story Arc placement action is not rollback-eligible") + parsed = _managed_placement_rollback_payload(action, payload) + work = await session.get(StoryArcSyncWork, parsed["sync_work_id"]) + if work is None: + raise ValueError("Import Story Arc placement work is missing; rollback refused") + action_id = int(action.id) + _require_import_story_arc_work_identity(action_id, work, parsed) + + membership = await session.get(IssueStoryArc, parsed["membership_id"]) + if membership is None: + raise ValueError("Import Story Arc placement membership is missing; rollback refused") + library_file = await session.get(LibraryFile, work.library_file_id) + if ( + library_file is None + or membership.issue_id is None + or library_file.issue_id != membership.issue_id + ): + raise ValueError("Import Story Arc canonical binding changed; rollback refused") + membership_id = int(membership.id) + story_arc_id = int(membership.story_arc_id) + await _require_import_story_arc_staged_binding( + session, + action=action, + story_arc_id=story_arc_id, + membership_id=membership_id, + imported_story_arc_id=parsed["imported_story_arc_id"], + imported_story_arc_entry_id=parsed["imported_story_arc_entry_id"], + ) + + pre_fence_work_state = work.state + work = await _fence_import_story_arc_work_for_rollback( + session, + action=action, + work=work, + payload=parsed, + ) + current_action = await session.get(ImportJobAction, action_id) + if current_action is None: + raise ValueError("Import Story Arc placement action disappeared during rollback") + placements = list( + ( + await session.scalars( + sa_select(StoryArcPlacement) + .where(StoryArcPlacement.creating_action_id == current_action.id) + .order_by(StoryArcPlacement.id.asc()) + .limit(2) + ) + ).all() + ) + if len(placements) > 1: + raise ValueError("Import Story Arc placement action owns multiple rows; rollback refused") + placement = placements[0] if placements else None + + if work.state not in { + StoryArcSyncWorkState.COMPLETED, + StoryArcSyncWorkState.CANCELLED, + }: + raise ValueError("Import Story Arc placement work is not terminal; rollback refused") + if placement is None: + if _completed_story_arc_removal_checkpoint_matches( + work, + action=current_action, + membership_id=membership_id, + ): + return + prepared_removal = _prepared_story_arc_removal_checkpoint( + work, + action=current_action, + membership_id=membership_id, + ) + if prepared_removal is not None: + placement_id, placement_ownership = prepared_removal + recovered_status = _recovered_story_arc_removal_status( + membership, + placement_id=placement_id, + placement_ownership=placement_ownership, + ) + await _persist_story_arc_work_rollback_checkpoint( + session, + work=work, + action=current_action, + membership_id=membership_id, + status=recovered_status, + placement_id=placement_id, + placement_ownership=placement_ownership.value, + ) + return + if work.state is StoryArcSyncWorkState.CANCELLED: + await _persist_story_arc_work_rollback_checkpoint( + session, + work=work, + action=current_action, + membership_id=membership_id, + status="cancelled_before_publish", + ) + return + raise ValueError("Import Story Arc placement evidence is missing; rollback refused") + + _require_action_owned_placement_identity( + placement, + action=current_action, + work=work, + membership_id=membership_id, + ) + abandoned_published_operation_token = _terminal_published_operation_token( + placement, + work=work, + pre_fence_work_state=pre_fence_work_state, + ) + removal_status = ( + "referenced_placement_detached" + if placement.ownership is StoryArcPlacementOwnership.REFERENCED + else "managed_placement_removed" + ) + placement_id = int(placement.id) + placement_ownership = placement.ownership + await _persist_story_arc_work_rollback_checkpoint( + session, + work=work, + action=current_action, + membership_id=membership_id, + status="placement_removal_prepared", + placement_id=placement_id, + placement_ownership=placement_ownership.value, + ) + + # Local import avoids the sync queue -> import action helper cycle. + from pullbox.services.story_arc_placement_integration import ( + StoryArcPlacementSyncService, + ) + + await StoryArcPlacementSyncService().remove_placement( + session, + story_arc_id, + placement_id, + confirm_managed_artifact_removal=( + placement_ownership is StoryArcPlacementOwnership.MANAGED + ), + abandoned_published_operation_token=abandoned_published_operation_token, + ) + session.expire_all() + current_work = await session.get(StoryArcSyncWork, int(parsed["sync_work_id"])) + if current_work is None: + raise ValueError("Import Story Arc placement work disappeared during rollback") + _require_import_story_arc_work_identity(action_id, current_work, parsed) + reloaded_action = await session.get(ImportJobAction, action_id) + if reloaded_action is None: + raise ValueError("Import Story Arc placement action disappeared during rollback") + await _persist_story_arc_work_rollback_checkpoint( + session, + work=current_work, + action=reloaded_action, + membership_id=membership_id, + status=removal_status, + placement_id=placement_id, + placement_ownership=placement_ownership.value, + ) + + +def _managed_placement_rollback_payload( + action: ImportJobAction, + payload: dict[str, Any], +) -> _ManagedPlacementRollbackPayload: + if action.phase != _STORY_ARC_PLACEMENT_PHASE: + raise ValueError("Import Story Arc placement action phase changed; rollback refused") + if set(payload) != _STORY_ARC_MANAGED_PLACEMENT_PAYLOAD_KEYS: + raise ValueError("Import Story Arc placement payload shape changed; rollback refused") + schema_version = _non_negative_int(payload.get("schema_version"), "schema_version") + if schema_version != 1: + raise ValueError("Import Story Arc placement payload version changed; rollback refused") + desired_generation = _required_string(payload, "desired_generation") + if len(desired_generation) != 64: + raise ValueError("Import Story Arc placement generation changed; rollback refused") + source_import_job_id = _positive_int( + payload.get("source_import_job_id"), + "source_import_job_id", + ) + if source_import_job_id != action.import_job_id: + raise ValueError("Import Story Arc placement job changed; rollback refused") + return { + "sync_work_id": _positive_int(payload.get("sync_work_id"), "sync_work_id"), + "membership_id": _positive_int(payload.get("membership_id"), "membership_id"), + "desired_generation": desired_generation, + "imported_story_arc_id": _positive_int( + payload.get("imported_story_arc_id"), + "imported_story_arc_id", + ), + "imported_story_arc_entry_id": _positive_int( + payload.get("imported_story_arc_entry_id"), + "imported_story_arc_entry_id", + ), + "source_import_job_id": source_import_job_id, + } + + +def _require_import_story_arc_work_identity( + action_id: int, + work: StoryArcSyncWork, + payload: _ManagedPlacementRollbackPayload, +) -> None: + if ( + work.origin_import_action_id != action_id + or work.origin_import_job_id != payload["source_import_job_id"] + or work.origin_imported_story_arc_id != payload["imported_story_arc_id"] + or work.origin_imported_story_arc_entry_id != payload["imported_story_arc_entry_id"] + or work.issue_story_arc_id != payload["membership_id"] + ): + raise ValueError("Import Story Arc placement work ownership changed; rollback refused") + if work.desired_generation != payload["desired_generation"]: + raise ValueError("Import Story Arc placement generation changed; rollback refused") + + +async def _require_import_story_arc_staged_binding( + session: AsyncSession, + *, + action: ImportJobAction, + story_arc_id: int, + membership_id: int, + imported_story_arc_id: int, + imported_story_arc_entry_id: int, +) -> None: + staged_entry_id = await session.scalar( + sa_select(ImportedStoryArcEntry.id) + .join( + ImportedStoryArc, + ImportedStoryArc.id == ImportedStoryArcEntry.imported_story_arc_id, + ) + .where( + ImportedStoryArc.id == imported_story_arc_id, + ImportedStoryArc.import_job_id == action.import_job_id, + ImportedStoryArc.materialized_story_arc_id == story_arc_id, + ImportedStoryArcEntry.id == imported_story_arc_entry_id, + ImportedStoryArcEntry.materialized_membership_id == membership_id, + ) + ) + if staged_entry_id is None: + raise ValueError("Import Story Arc staged provenance changed; rollback refused") + + +async def _fence_import_story_arc_work_for_rollback( + session: AsyncSession, + *, + action: ImportJobAction, + work: StoryArcSyncWork, + payload: _ManagedPlacementRollbackPayload, +) -> StoryArcSyncWork: + """Atomically stop claimable work or durably request running cancellation.""" + action_id = int(action.id) + work_id = int(work.id) + for _attempt in range(3): + state = work.state + now = datetime.now(UTC) + if state is not StoryArcSyncWorkState.RUNNING and work.claim_token is not None: + raise StoryArcManagedPlacementRollbackDeferredError(work_id) + if state is StoryArcSyncWorkState.RUNNING: + result = await session.execute( + sa_update(StoryArcSyncWork) + .where( + StoryArcSyncWork.id == work.id, + StoryArcSyncWork.origin_import_action_id == action_id, + StoryArcSyncWork.origin_import_job_id == payload["source_import_job_id"], + StoryArcSyncWork.origin_imported_story_arc_id + == payload["imported_story_arc_id"], + StoryArcSyncWork.origin_imported_story_arc_entry_id + == payload["imported_story_arc_entry_id"], + StoryArcSyncWork.issue_story_arc_id == payload["membership_id"], + StoryArcSyncWork.desired_generation == payload["desired_generation"], + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + ) + .values(cancel_requested_at=now) + ) + await session.commit() + if result.rowcount == 1: # type: ignore[attr-defined] + raise StoryArcManagedPlacementRollbackDeferredError(work_id) + elif state in _CANCELLABLE_STORY_ARC_WORK_STATES: + checkpoint = _story_arc_work_rollback_result( + work, + action=action, + membership_id=int(payload["membership_id"]), + status="work_fenced_for_rollback", + ) + result = await session.execute( + sa_update(StoryArcSyncWork) + .where( + StoryArcSyncWork.id == work.id, + StoryArcSyncWork.origin_import_action_id == action_id, + StoryArcSyncWork.origin_import_job_id == payload["source_import_job_id"], + StoryArcSyncWork.origin_imported_story_arc_id + == payload["imported_story_arc_id"], + StoryArcSyncWork.origin_imported_story_arc_entry_id + == payload["imported_story_arc_entry_id"], + StoryArcSyncWork.issue_story_arc_id == payload["membership_id"], + StoryArcSyncWork.desired_generation == payload["desired_generation"], + StoryArcSyncWork.state == state, + ) + .values( + state=StoryArcSyncWorkState.CANCELLED, + claim_token=None, + claimed_at=None, + next_attempt_at=None, + cancel_requested_at=now, + last_error_code="import_rollback_requested", + last_error_category="cancelled", + last_error_detail="Import rollback fenced Story Arc placement work.", + last_result=checkpoint, + ) + ) + await session.commit() + if result.rowcount == 1: # type: ignore[attr-defined] + session.expire_all() + cancelled = await session.get(StoryArcSyncWork, work_id) + if cancelled is None: + raise ValueError("Import Story Arc placement work disappeared during rollback") + return cancelled + else: + return work + session.expire_all() + reloaded = await session.get(StoryArcSyncWork, work_id) + if reloaded is None: + raise ValueError("Import Story Arc placement work disappeared during rollback") + _require_import_story_arc_work_identity(action_id, reloaded, payload) + current_action = await session.get(ImportJobAction, action_id) + if current_action is None: + raise ValueError("Import Story Arc placement action disappeared during rollback") + action = current_action + work = reloaded + raise ValueError("Import Story Arc placement work changed concurrently; rollback refused") + + +def _require_action_owned_placement_identity( + placement: StoryArcPlacement, + *, + action: ImportJobAction, + work: StoryArcSyncWork, + membership_id: int, +) -> None: + if ( + placement.issue_story_arc_id != membership_id + or placement.library_file_id != work.library_file_id + or placement.source_import_job_id != action.import_job_id + or placement.creating_action_id != action.id + or placement.source_kind is not StoryArcSourceKind.PULLBOX + ): + raise ValueError("Import Story Arc placement provenance changed; rollback refused") + if placement.ownership is StoryArcPlacementOwnership.MANAGED: + if placement.mode is StoryArcPlacementMode.REFERENCE_ONLY: + raise ValueError("Managed Story Arc placement mode changed; rollback refused") + elif ( + placement.ownership is not StoryArcPlacementOwnership.REFERENCED + or placement.mode is not StoryArcPlacementMode.REFERENCE_ONLY + ): + raise ValueError("Referenced Story Arc placement mode changed; rollback refused") + + +def _terminal_published_operation_token( + placement: StoryArcPlacement, + *, + work: StoryArcSyncWork, + pre_fence_work_state: StoryArcSyncWorkState, +) -> str | None: + """Return only a terminal, unclaimed, exactly checkpointed publish token.""" + if ( + work.state + not in { + StoryArcSyncWorkState.COMPLETED, + StoryArcSyncWorkState.CANCELLED, + } + or work.claim_token is not None + ): + raise StoryArcManagedPlacementRollbackDeferredError(int(work.id)) + observed_token = placement.operation_token + if observed_token is None: + return None + if pre_fence_work_state not in { + StoryArcSyncWorkState.COMPLETED, + StoryArcSyncWorkState.CANCELLED, + StoryArcSyncWorkState.FAILED, + }: + raise ValueError( + "Import Story Arc published checkpoint did not belong to terminal work; " + "rollback refused" + ) + last_result = dict(placement.last_result or {}) + target_fingerprint = last_result.get("target_fingerprint") + if ( + not observed_token + or last_result.get("schema_version") != 1 + or last_result.get("status") != "published_pending_reconcile" + or last_result.get("operation_token") != observed_token + or not isinstance(target_fingerprint, dict) + or not target_fingerprint + ): + raise ValueError( + "Import Story Arc placement token is not an exact published checkpoint; " + "rollback refused" + ) + return observed_token + + +async def _persist_story_arc_work_rollback_checkpoint( + session: AsyncSession, + *, + work: StoryArcSyncWork, + action: ImportJobAction, + membership_id: int, + status: str, + placement_id: int | None = None, + placement_ownership: str | None = None, +) -> None: + work.cancel_requested_at = work.cancel_requested_at or datetime.now(UTC) + work.last_result = _story_arc_work_rollback_result( + work, + action=action, + membership_id=membership_id, + status=status, + placement_id=placement_id, + placement_ownership=placement_ownership, + ) + await session.commit() + + +def _story_arc_work_rollback_result( + work: StoryArcSyncWork, + *, + action: ImportJobAction, + membership_id: int, + status: str, + placement_id: int | None = None, + placement_ownership: str | None = None, +) -> dict[str, object]: + marker: dict[str, object] = { + "schema_version": 1, + "status": status, + "import_job_id": action.import_job_id, + "import_action_id": action.id, + "sync_work_id": work.id, + "membership_id": membership_id, + "desired_generation": work.desired_generation, + } + if placement_id is not None: + marker["placement_id"] = placement_id + if placement_ownership is not None: + marker["placement_ownership"] = placement_ownership + return {**dict(work.last_result or {}), "rollback": marker} + + +def _completed_story_arc_removal_checkpoint_matches( + work: StoryArcSyncWork, + *, + action: ImportJobAction, + membership_id: int, +) -> bool: + marker = dict(work.last_result or {}).get("rollback") + if not isinstance(marker, dict) or marker.get("status") not in { + "managed_placement_removed", + "referenced_placement_detached", + }: + return False + expected_ownership = ( + StoryArcPlacementOwnership.MANAGED + if marker.get("status") == "managed_placement_removed" + else StoryArcPlacementOwnership.REFERENCED + ) + if not _story_arc_removal_marker_matches( + marker, + work=work, + action=action, + membership_id=membership_id, + expected_status=str(marker["status"]), + expected_ownership=expected_ownership, + ): + raise ValueError("Import Story Arc completed removal checkpoint changed") + return True + + +def _prepared_story_arc_removal_checkpoint( + work: StoryArcSyncWork, + *, + action: ImportJobAction, + membership_id: int, +) -> tuple[int, StoryArcPlacementOwnership] | None: + marker = dict(work.last_result or {}).get("rollback") + if not isinstance(marker, dict) or marker.get("status") != "placement_removal_prepared": + return None + ownership_raw = marker.get("placement_ownership") + if not isinstance(ownership_raw, str): + raise ValueError("Import Story Arc prepared removal checkpoint changed") + try: + ownership = StoryArcPlacementOwnership(ownership_raw) + except (TypeError, ValueError) as exc: + raise ValueError("Import Story Arc prepared removal checkpoint changed") from exc + if ownership not in { + StoryArcPlacementOwnership.MANAGED, + StoryArcPlacementOwnership.REFERENCED, + } or not _story_arc_removal_marker_matches( + marker, + work=work, + action=action, + membership_id=membership_id, + expected_status="placement_removal_prepared", + expected_ownership=ownership, + ): + raise ValueError("Import Story Arc prepared removal checkpoint changed") + return int(marker["placement_id"]), ownership + + +def _story_arc_removal_marker_matches( + marker: dict[str, object], + *, + work: StoryArcSyncWork, + action: ImportJobAction, + membership_id: int, + expected_status: str, + expected_ownership: StoryArcPlacementOwnership, +) -> bool: + placement_id = marker.get("placement_id") + return bool( + set(marker) + == { + "schema_version", + "status", + "import_job_id", + "import_action_id", + "sync_work_id", + "membership_id", + "desired_generation", + "placement_id", + "placement_ownership", + } + and marker.get("schema_version") == 1 + and marker.get("status") == expected_status + and marker.get("import_job_id") == action.import_job_id + and marker.get("import_action_id") == action.id + and marker.get("sync_work_id") == work.id + and marker.get("membership_id") == membership_id + and marker.get("desired_generation") == work.desired_generation + and isinstance(placement_id, int) + and not isinstance(placement_id, bool) + and placement_id > 0 + and marker.get("placement_ownership") == expected_ownership.value + ) + + +def _recovered_story_arc_removal_status( + membership: IssueStoryArc, + *, + placement_id: int, + placement_ownership: StoryArcPlacementOwnership, +) -> str: + """Validate the transactionally paired membership checkpoint after row deletion.""" + result = dict(membership.last_materialization_result or {}) + if placement_ownership is StoryArcPlacementOwnership.MANAGED: + valid = ( + set(result) + == { + "schema_version", + "status", + "placement_id", + "artifact_removed", + "canonical_preserved", + } + and result.get("schema_version") == 1 + and result.get("status") == "placement_removed" + and result.get("placement_id") == placement_id + and isinstance(result.get("artifact_removed"), bool) + and result.get("canonical_preserved") is True + ) + recovered_status = "managed_placement_removed" + else: + valid = ( + set(result) + == { + "schema_version", + "status", + "placement_id", + "artifact_removed", + "canonical_preserved", + "referenced_artifact_preserved", + } + and result.get("schema_version") == 1 + and result.get("status") == "placement_reference_removed" + and result.get("placement_id") == placement_id + and result.get("artifact_removed") is False + and result.get("canonical_preserved") is True + and result.get("referenced_artifact_preserved") is True + ) + recovered_status = "referenced_placement_detached" + if not valid or membership.sync_eligible is not False: + raise ValueError("Import Story Arc membership removal checkpoint is missing or changed") + return recovered_status + + +async def _rollback_attached_story_arc_reference( + session: AsyncSession, + action: ImportJobAction, + payload: dict[str, Any], +) -> None: + """Detach import-owned database evidence without touching the user artifact.""" + if payload.get("journal_state") == "prepared" and payload.get("placement_id") is None: + return + if payload.get("journal_state") != "completed": + raise ValueError("Referenced story-arc placement journal is incomplete") + placement_id = _positive_int(payload.get("placement_id"), "placement_id") + membership_id = _positive_int(payload.get("issue_story_arc_id"), "issue_story_arc_id") + imported_entry_id = _positive_int( + payload.get("imported_story_arc_entry_id"), + "imported_story_arc_entry_id", + ) + if ( + _positive_int(payload.get("source_import_job_id"), "source_import_job_id") + != action.import_job_id + ): + raise ValueError("Referenced story-arc placement journal job changed") + placement = await session.get(StoryArcPlacement, placement_id) + if placement is None: + return + await _require_story_arc_entry_ownership( + session, + action=action, + imported_story_arc_entry_id=imported_entry_id, + membership_id=membership_id, + ) + expected_identity = { + "issue_story_arc_id": membership_id, + "placement_path": _required_string(payload, "placement_path"), + "mode": StoryArcPlacementMode.REFERENCE_ONLY.value, + "ownership": StoryArcPlacementOwnership.REFERENCED.value, + "source_kind": _required_string(payload, "source_kind"), + "source_import_job_id": action.import_job_id, + "creating_action_id": int(action.id), + } + if _referenced_placement_rollback_identity(placement) != expected_identity: + raise ValueError("Referenced story-arc placement ownership changed; rollback refused") + await session.delete(placement) + + +async def _rollback_created_story_arc_membership( + session: AsyncSession, + action: ImportJobAction, + payload: dict[str, Any], +) -> None: + membership_id = _positive_int(payload.get("membership_id"), "membership_id") + story_arc_id = _positive_int(payload.get("story_arc_id"), "story_arc_id") + expected_after = _payload_mapping(payload, "expected_after") + membership = await session.get(IssueStoryArc, membership_id) + if membership is None: + return + await _require_story_arc_entry_ownership( + session, + action=action, + imported_story_arc_entry_id=_positive_int( + payload.get("imported_story_arc_entry_id"), + "imported_story_arc_entry_id", + ), + membership_id=membership_id, + ) + if membership.story_arc_id != story_arc_id or _membership_state(membership) != expected_after: + raise ValueError("Story-arc membership changed after import; rollback refused") + placement_count = int( + await session.scalar( + sa_select(sa_func.count()) + .select_from(StoryArcPlacement) + .where(StoryArcPlacement.issue_story_arc_id == membership_id) + ) + or 0 + ) + if placement_count: + raise ValueError("Story-arc membership has placements; rollback refused") + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise ValueError("Story arc disappeared before membership rollback") + revision_after = _non_negative_int(payload.get("arc_revision_after"), "arc_revision_after") + revision_before = _non_negative_int(payload.get("arc_revision_before"), "arc_revision_before") + if int(arc.revision) != revision_after: + raise ValueError("Story arc changed after import; rollback refused") + await session.delete(membership) + arc.revision = revision_before + + +async def _rollback_updated_story_arc_membership( + session: AsyncSession, + action: ImportJobAction, + payload: dict[str, Any], +) -> None: + membership_id = _positive_int(payload.get("membership_id"), "membership_id") + story_arc_id = _positive_int(payload.get("story_arc_id"), "story_arc_id") + expected_after = _payload_mapping(payload, "expected_after") + restore_before = _payload_mapping(payload, "restore_before") + membership = await session.get(IssueStoryArc, membership_id) + if membership is None: + raise ValueError("Updated story-arc membership disappeared; rollback refused") + await _require_story_arc_entry_ownership( + session, + action=action, + imported_story_arc_entry_id=_positive_int( + payload.get("imported_story_arc_entry_id"), + "imported_story_arc_entry_id", + ), + membership_id=membership_id, + ) + if membership.story_arc_id != story_arc_id or _membership_state(membership) != expected_after: + raise ValueError("Story-arc membership changed after import; rollback refused") + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise ValueError("Story arc disappeared before membership rollback") + revision_after = _non_negative_int(payload.get("arc_revision_after"), "arc_revision_after") + revision_before = _non_negative_int(payload.get("arc_revision_before"), "arc_revision_before") + if int(arc.revision) != revision_after: + raise ValueError("Story arc changed after import; rollback refused") + _restore_membership_state(membership, restore_before) + arc.revision = revision_before + + +async def _rollback_created_story_arc_external_identity( + session: AsyncSession, + action: ImportJobAction, + payload: dict[str, Any], +) -> None: + identity_id = _positive_int(payload.get("external_identity_id"), "external_identity_id") + story_arc_id = _positive_int(payload.get("story_arc_id"), "story_arc_id") + expected_after = _payload_mapping(payload, "expected_after") + identity = await session.get(StoryArcExternalIdentity, identity_id) + if identity is None: + return + await _require_story_arc_ownership( + session, + action=action, + imported_story_arc_id=_positive_int( + payload.get("imported_story_arc_id"), "imported_story_arc_id" + ), + story_arc_id=story_arc_id, + ) + if ( + identity.story_arc_id != story_arc_id + or _external_identity_state(identity) != expected_after + ): + raise ValueError("Story-arc external identity changed after import; rollback refused") + await session.delete(identity) + + +async def _rollback_story_arc_policy_update( + session: AsyncSession, + action: ImportJobAction, + payload: dict[str, Any], +) -> None: + story_arc_id = _positive_int(payload.get("story_arc_id"), "story_arc_id") + expected_after = _payload_mapping(payload, "expected_after") + restore_before = _payload_mapping(payload, "restore_before") + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise ValueError("Updated story arc disappeared; rollback refused") + await _require_story_arc_ownership( + session, + action=action, + imported_story_arc_id=_positive_int( + payload.get("imported_story_arc_id"), "imported_story_arc_id" + ), + story_arc_id=story_arc_id, + ) + if _story_arc_policy_state(arc) != expected_after: + raise ValueError("Story arc changed after import; rollback refused") + _restore_story_arc_policy_state(arc, restore_before) + + +async def _rollback_created_story_arc( + session: AsyncSession, + action: ImportJobAction, + payload: dict[str, Any], +) -> None: + story_arc_id = _positive_int(payload.get("story_arc_id"), "story_arc_id") + expected_after = _payload_mapping(payload, "expected_after") + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + return + await _require_story_arc_ownership( + session, + action=action, + imported_story_arc_id=_positive_int( + payload.get("imported_story_arc_id"), "imported_story_arc_id" + ), + story_arc_id=story_arc_id, + ) + if arc.source_import_job_id != action.import_job_id: + raise ValueError("Story arc is not owned by this import; rollback refused") + if _story_arc_created_state(arc) != expected_after: + raise ValueError("Story arc changed after import; rollback refused") + membership_count = int( + await session.scalar( + sa_select(sa_func.count()) + .select_from(IssueStoryArc) + .where(IssueStoryArc.story_arc_id == story_arc_id) + ) + or 0 + ) + identity_count = int( + await session.scalar( + sa_select(sa_func.count()) + .select_from(StoryArcExternalIdentity) + .where(StoryArcExternalIdentity.story_arc_id == story_arc_id) + ) + or 0 + ) + placement_count = int( + await session.scalar( + sa_select(sa_func.count()) + .select_from(StoryArcPlacement) + .join( + IssueStoryArc, + IssueStoryArc.id == StoryArcPlacement.issue_story_arc_id, + ) + .where(IssueStoryArc.story_arc_id == story_arc_id) + ) + or 0 + ) + if membership_count or identity_count or placement_count: + raise ValueError("Story arc still has related rows; rollback refused") + await session.delete(arc) + + +async def _require_story_arc_ownership( + session: AsyncSession, + *, + action: ImportJobAction, + imported_story_arc_id: int, + story_arc_id: int, +) -> None: + staged_arc_id = await session.scalar( + sa_select(ImportedStoryArc.id).where( + ImportedStoryArc.id == imported_story_arc_id, + ImportedStoryArc.import_job_id == action.import_job_id, + ImportedStoryArc.materialized_story_arc_id == story_arc_id, + ) + ) + if staged_arc_id is None: + raise ValueError("Story-arc rollback ownership changed; rollback refused") + + +async def _require_story_arc_entry_ownership( + session: AsyncSession, + *, + action: ImportJobAction, + imported_story_arc_entry_id: int, + membership_id: int, +) -> None: + staged_entry_id = await session.scalar( + sa_select(ImportedStoryArcEntry.id) + .join( + ImportedStoryArc, + ImportedStoryArc.id == ImportedStoryArcEntry.imported_story_arc_id, + ) + .where( + ImportedStoryArcEntry.id == imported_story_arc_entry_id, + ImportedStoryArcEntry.materialized_membership_id == membership_id, + ImportedStoryArc.import_job_id == action.import_job_id, + ) + ) + if staged_entry_id is None: + raise ValueError("Story-arc membership rollback ownership changed; rollback refused") + + +def _story_arc_policy_state(arc: StoryArc) -> dict[str, object]: + return { + "monitored": bool(arc.monitored), + "search_missing": bool(arc.search_missing), + "include_upcoming": bool(arc.include_upcoming), + "sync_enabled": bool(arc.sync_enabled), + "target_library_root_id": arc.target_library_root_id, + "policy_schema_version": arc.policy_schema_version, + "policy_snapshot": dict(arc.policy_snapshot or {}), + "revision": int(arc.revision), + } + + +def _story_arc_created_state(arc: StoryArc) -> dict[str, object]: + return { + "name": arc.name, + "normalized_name": arc.normalized_name, + "description": arc.description, + "comicvine_id": arc.comicvine_id, + "publisher_id": arc.publisher_id, + "comicvine_url": arc.comicvine_url, + "source_kind": arc.source_kind.value, + "lifecycle": arc.lifecycle.value, + "source_import_job_id": arc.source_import_job_id, + "diagnostics": dict(arc.diagnostics or {}), + **_story_arc_policy_state(arc), + } + + +def _membership_state(membership: IssueStoryArc) -> dict[str, object]: + return { + "story_arc_id": int(membership.story_arc_id), + "issue_id": membership.issue_id, + "sequence_number": int(membership.sequence_number), + "source_ordinal": int(membership.source_ordinal), + "legacy_sequence_was_null": bool(membership.legacy_sequence_was_null), + "resolution_state": membership.resolution_state.value, + "source_kind": membership.source_kind.value, + "source_entry_id": membership.source_entry_id, + "source_arc_id": membership.source_arc_id, + "source_issue_id": membership.source_issue_id, + "source_series_id": membership.source_series_id, + "source_issue_number_text": membership.source_issue_number_text, + "source_series_name": membership.source_series_name, + "source_issue_title": membership.source_issue_title, + "source_publisher": membership.source_publisher, + "source_release_date_text": membership.source_release_date_text, + "source_issue_date_text": membership.source_issue_date_text, + "resolution_confidence": membership.resolution_confidence, + "resolution_method": membership.resolution_method, + "evidence": dict(membership.evidence or {}), + "sync_eligible": bool(membership.sync_eligible), + "last_materialization_result": dict(membership.last_materialization_result or {}), + } + + +def _external_identity_state(identity: StoryArcExternalIdentity) -> dict[str, object]: + return { + "story_arc_id": int(identity.story_arc_id), + "source": identity.source, + "namespace": identity.namespace, + "external_id": identity.external_id, + "source_url": identity.source_url, + "evidence": dict(identity.evidence or {}), + } + + +def _referenced_placement_rollback_identity( + placement: StoryArcPlacement, +) -> dict[str, object]: + """Return only immutable ownership fields; observed drift is intentionally excluded.""" + return { + "issue_story_arc_id": int(placement.issue_story_arc_id), + "placement_path": placement.placement_path, + "mode": placement.mode.value, + "ownership": placement.ownership.value, + "source_kind": placement.source_kind.value, + "source_import_job_id": placement.source_import_job_id, + "creating_action_id": placement.creating_action_id, + } + + +def _restore_story_arc_policy_state(arc: StoryArc, state: dict[str, object]) -> None: + arc.monitored = _bool_value(state, "monitored") + arc.search_missing = _bool_value(state, "search_missing") + arc.include_upcoming = _bool_value(state, "include_upcoming") + arc.sync_enabled = _bool_value(state, "sync_enabled") + arc.target_library_root_id = _optional_positive_int( + state.get("target_library_root_id"), "target_library_root_id" + ) + arc.policy_schema_version = _optional_non_negative_int( + state.get("policy_schema_version"), "policy_schema_version" + ) + arc.policy_snapshot = dict(_mapping_value(state.get("policy_snapshot"), "policy_snapshot")) + arc.revision = _non_negative_int(state.get("revision"), "revision") + + +def _restore_membership_state( + membership: IssueStoryArc, + state: dict[str, object], +) -> None: + if _positive_int(state.get("story_arc_id"), "story_arc_id") != membership.story_arc_id: + raise ValueError("Membership rollback story arc does not match") + membership.issue_id = _optional_positive_int(state.get("issue_id"), "issue_id") + membership.sequence_number = _non_negative_int(state.get("sequence_number"), "sequence_number") + membership.source_ordinal = _non_negative_int(state.get("source_ordinal"), "source_ordinal") + membership.legacy_sequence_was_null = _bool_value(state, "legacy_sequence_was_null") + membership.resolution_state = StoryArcResolutionState( + _required_string(state, "resolution_state") + ) + membership.source_kind = StoryArcSourceKind(_required_string(state, "source_kind")) + for field in ( + "source_entry_id", + "source_arc_id", + "source_issue_id", + "source_series_id", + "source_issue_number_text", + "source_series_name", + "source_issue_title", + "source_publisher", + "source_release_date_text", + "source_issue_date_text", + "resolution_method", + ): + setattr(membership, field, _optional_string(state.get(field), field)) + confidence = state.get("resolution_confidence") + if confidence is not None and ( + isinstance(confidence, bool) or not isinstance(confidence, int | float) + ): + raise ValueError("Invalid resolution_confidence in story-arc rollback action") + membership.resolution_confidence = float(confidence) if confidence is not None else None + membership.evidence = dict(_mapping_value(state.get("evidence"), "evidence")) + membership.sync_eligible = _bool_value(state, "sync_eligible") + membership.last_materialization_result = dict( + _mapping_value(state.get("last_materialization_result"), "last_materialization_result") + ) + + +def _payload_mapping(payload: dict[str, Any], key: str) -> dict[str, object]: + return dict(_mapping_value(payload.get(key), key)) + + +def _mapping_value(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict): + raise ValueError(f"Invalid {label} in story-arc rollback action") + return value + + +def _positive_int(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"Invalid {label} in story-arc rollback action") + return value + + +def _non_negative_int(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"Invalid {label} in story-arc rollback action") + return value + + +def _optional_positive_int(value: object, label: str) -> int | None: + return None if value is None else _positive_int(value, label) + + +def _optional_non_negative_int(value: object, label: str) -> int | None: + return None if value is None else _non_negative_int(value, label) + + +def _bool_value(state: dict[str, object], key: str) -> bool: + value = state.get(key) + if not isinstance(value, bool): + raise ValueError(f"Invalid {key} in story-arc rollback action") + return value + + +def _required_string(state: dict[str, object], key: str) -> str: + value = state.get(key) + if not isinstance(value, str) or not value: + raise ValueError(f"Invalid {key} in story-arc rollback action") + return value + + +def _optional_string(value: object, label: str) -> str | None: + if value is not None and not isinstance(value, str): + raise ValueError(f"Invalid {label} in story-arc rollback action") + return value + + async def _restore_imported_file_paths_after_folder_rollback( session: AsyncSession, *, diff --git a/src/pullbox/services/import_job_archive.py b/src/pullbox/services/import_job_archive.py new file mode 100644 index 00000000..735ff1d4 --- /dev/null +++ b/src/pullbox/services/import_job_archive.py @@ -0,0 +1,42 @@ +"""Non-destructive import history archival.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.models.import_job import ImportControlRequest, ImportJob, ImportJobStatus + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +_ARCHIVABLE_STATUSES = frozenset( + { + ImportJobStatus.COMPLETED, + ImportJobStatus.FAILED, + ImportJobStatus.CANCELLED, + ImportJobStatus.ROLLED_BACK, + } +) + + +async def set_import_job_archived( + session: AsyncSession, + job_id: int, + *, + archived: bool, +) -> ImportJob: + """Hide or restore one finished import without deleting its evidence.""" + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if ( + job.status not in _ARCHIVABLE_STATUSES + or job.control_request is not ImportControlRequest.NONE + ): + raise ValidationError("Only an idle, finished import can be archived") + job.archived_at = datetime.now(UTC) if archived else None + await session.flush() + return job diff --git a/src/pullbox/services/import_job_controls.py b/src/pullbox/services/import_job_controls.py index 4a8c378e..fdd87546 100644 --- a/src/pullbox/services/import_job_controls.py +++ b/src/pullbox/services/import_job_controls.py @@ -23,6 +23,9 @@ sync_paused_job_state, sync_progress_snapshot_state, ) +from pullbox.services.story_arc_sync_queue import ( + discard_unpublished_import_story_arc_sync_work, +) if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -45,6 +48,15 @@ def __call__( JobDeletedLogger = Callable[[int, ImportJobStatus], None] +def _is_story_arc_placement_wait(job: ImportJob) -> bool: + """Return whether an import is waiting on separately scheduled placements.""" + return ( + job.status == ImportJobStatus.IMPORTING + and job.import_started_at is not None + and dict(job.progress_snapshot or {}).get("phase") == "story_arc_placements" + ) + + async def _flush_job_with_sqlite_lock_retry( session: AsyncSession, job: ImportJob, @@ -62,7 +74,12 @@ async def _flush_job_with_sqlite_lock_retry( if not is_sqlite_locked_error(exc) or attempt == SQLITE_LOCK_RETRY_ATTEMPTS: raise await asyncio.sleep(sqlite_lock_retry_delay(attempt)) - reloaded = await session.get(ImportJob, job_id) + reloaded = await session.get( + ImportJob, + job_id, + populate_existing=True, + with_for_update=True, + ) if reloaded is None: raise NotFoundError("ImportJob", job_id) from exc mutate(reloaded) @@ -124,12 +141,16 @@ async def cancel_job( if ( job.status in {ImportJobStatus.PAUSED, ImportJobStatus.STALLED} and job.import_started_at is None + and dict(dict(job.progress_snapshot or {}).get("deferred_recovery") or {}).get("state") + not in {"queued", "catalogs", "prepared"} ): + await discard_unpublished_import_story_arc_sync_work(session, (job_id,)) await session.delete(job) await session.flush() return "deleted" if job.status in deletable: + await discard_unpublished_import_story_arc_sync_work(session, (job_id,)) log_job_deleted(job_id, job.status) await session.delete(job) await session.flush() @@ -145,7 +166,12 @@ async def pause_job( log_event: ImportEventLogger, ) -> ImportJob: """Persist or request a pause at the nearest resumable checkpoint.""" - job = await session.get(ImportJob, job_id) + job = await session.get( + ImportJob, + job_id, + populate_existing=True, + with_for_update=True, + ) if job is None: raise NotFoundError("ImportJob", job_id) @@ -157,6 +183,8 @@ async def pause_job( ImportJobStatus.IMPORTING, }: raise ValidationError(f"Cannot pause job in {job.status} state") + if _is_story_arc_placement_wait(job): + raise ValidationError("Import cannot be paused while story arc placements are finishing") def _apply_pause(target: ImportJob) -> None: if target.status not in { @@ -167,6 +195,10 @@ def _apply_pause(target: ImportJob) -> None: ImportJobStatus.IMPORTING, }: raise ValidationError(f"Cannot pause job in {target.status} state") + if _is_story_arc_placement_wait(target): + raise ValidationError( + "Import cannot be paused while story arc placements are finishing" + ) if target.status == ImportJobStatus.SCANNING and target.import_started_at is None: snapshot = dict(target.progress_snapshot or {}) target.control_request = ImportControlRequest.PAUSE @@ -218,6 +250,15 @@ async def resume_job( raise ValidationError(f"Cannot resume job in {job.status} state") phase = str((job.progress_snapshot or {}).get("phase") or "") + if ( + job.status is ImportJobStatus.STALLED + and job.import_started_at is not None + and phase == "story_arc_placements" + ): + raise ValidationError( + "Retry the failed Story Arc placement work or cancel the import; " + "the completed canonical import must not be replayed." + ) mode = snapshot_mode_for_job(job) if mode == "scan" and job.import_started_at is not None: if phase == "importing": @@ -265,20 +306,31 @@ async def request_cancel( log_event: ImportEventLogger, ) -> ImportJob: """Request cooperative cancellation or immediate discard for paused scans.""" - job = await session.get(ImportJob, job_id) + job = await session.get( + ImportJob, + job_id, + populate_existing=True, + with_for_update=True, + ) if job is None: raise NotFoundError("ImportJob", job_id) if ( job.status in {ImportJobStatus.PAUSED, ImportJobStatus.STALLED} and job.import_started_at is None + and dict(dict(job.progress_snapshot or {}).get("deferred_recovery") or {}).get("state") + not in {"queued", "catalogs", "prepared"} ): await session.delete(job) await session.flush() return job def _apply_cancel(target: ImportJob) -> None: - if ( + recovery = dict(dict(target.progress_snapshot or {}).get("deferred_recovery") or {}) + if recovery.get("state") in {"queued", "catalogs", "prepared"}: + target.status = ImportJobStatus.CANCELLING + target.control_request = ImportControlRequest.CANCEL + elif _is_story_arc_placement_wait(target) or ( target.status in {ImportJobStatus.PAUSED, ImportJobStatus.STALLED} and target.import_started_at is not None ): @@ -298,6 +350,7 @@ def _apply_cancel(target: ImportJob) -> None: raise ValidationError(f"Cannot cancel job in {target.status} state") target.error_message = "Import cancelled by user." if target.status == ImportJobStatus.ROLLING_BACK: + target.story_arc_placement_followup_pending = False target.progress_snapshot = initialize_progress_snapshot( target, mode="rollback", @@ -351,6 +404,7 @@ async def request_rollback( job.status = ImportJobStatus.ROLLING_BACK job.control_request = ImportControlRequest.NONE + job.story_arc_placement_followup_pending = False job.progress_snapshot = initialize_progress_snapshot( job, mode="rollback", diff --git a/src/pullbox/services/import_job_creation.py b/src/pullbox/services/import_job_creation.py index b5602680..1c4763e4 100644 --- a/src/pullbox/services/import_job_creation.py +++ b/src/pullbox/services/import_job_creation.py @@ -2,24 +2,49 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING, Any, Protocol from sqlalchemy import select as sa_select -from pullbox.core.exceptions import ValidationError -from pullbox.core.library_policy import load_library_ingest_policy, load_search_on_add_default -from pullbox.models.import_job import ImportControlRequest, ImportJob, ImportJobStatus +from pullbox.core.exceptions import ConfigurationError, ValidationError +from pullbox.core.library_file_ownership import ( + ReferencedFileValidationError, + resolve_referenced_source_root, +) +from pullbox.core.library_layout import resolve_source_layout_spec +from pullbox.core.library_policy import ( + load_effective_library_ingest_policy, + load_library_ingest_policy, + load_search_on_add_default, +) +from pullbox.core.library_root_resolution import resolve_library_root +from pullbox.models.import_job import ( + ImportControlRequest, + ImportFileHandlingMode, + ImportJob, + ImportJobStatus, + ImportSourceType, +) +from pullbox.services.import_mylar3_path_preflight import Mylar3PathPreflightAnalyzer +from pullbox.services.import_mylar3_path_validation import validate_mylar3_path_map_targets from pullbox.services.import_policy_snapshot import apply_ingest_policy_to_import_job +from pullbox.services.import_root_policy_activation import ( + apply_future_root_policy_to_ingest_policy, + build_future_root_policy_snapshot, +) from pullbox.services.import_workflow_state import ( ACTIVE_IMPORT_JOB_STATUSES, initialize_progress_snapshot, ) +from pullbox.services.library_root_management import validate_managed_library_root if TYPE_CHECKING: from collections.abc import Awaitable from sqlalchemy.ext.asyncio import AsyncSession + from pullbox.models.library import LibraryRoot from pullbox.schemas.import_job import ImportJobCreate @@ -44,6 +69,87 @@ async def create_job( log_event: ImportEventLogger, ) -> ImportJob: """Create an import job record without starting scan execution.""" + if ( + request.file_handling_mode == ImportFileHandlingMode.IN_PLACE + and request.source_type == ImportSourceType.FILESYSTEM + ): + try: + await resolve_referenced_source_root( + session, + Path(request.source_path), + None, + ) + except ReferencedFileValidationError as exc: + raise ValidationError(exc.message) from exc + + target_root: LibraryRoot | None = None + requires_managed_destination = ( + request.file_handling_mode == ImportFileHandlingMode.MANAGED_COPY + or request.target_library_root_id is not None + or request.future_layout_requested + ) + if requires_managed_destination: + try: + target_root = await resolve_library_root( + session, + Path(request.source_path), + request.target_library_root_id, + ) + except ConfigurationError as exc: + raise ValidationError(exc.message) from exc + if target_root is not None: + await validate_managed_library_root(target_root) + resolved_target_library_root_id = target_root.id if target_root is not None else None + if request.future_layout_requested and resolved_target_library_root_id is None: + raise ValidationError("Future library layout requires a target library root.") + if request.source_type == ImportSourceType.MYLAR3 and request.mylar3_path_map_confirmed: + await validate_mylar3_path_map_targets( + session, + request.mylar3_path_map, + file_handling_mode=request.file_handling_mode, + ) + from pullbox.schemas.import_mylar3_path_preflight import MylarPathMappingDraft + + try: + preview = await Mylar3PathPreflightAnalyzer().analyze( + session, + request.source_path, + auto_detect=False, + file_handling_mode=request.file_handling_mode, + mappings=[ + MylarPathMappingDraft( + stored_prefix=stored_prefix, + pullbox_prefix=pullbox_prefix, + ) + for stored_prefix, pullbox_prefix in request.mylar3_path_map.items() + ], + ) + except (OSError, ValueError) as exc: + raise ValidationError( + "The confirmed Mylar path mapping could not be revalidated." + ) from exc + if ( + preview.requires_unresolved_acknowledgement + and not request.mylar3_allow_unresolved_paths + ): + raise ValidationError( + "Review and acknowledge the unavailable Mylar paths before starting the scan." + ) + if preview.requires_unresolved_acknowledgement and ( + request.mylar3_unresolved_fingerprint != preview.unresolved_fingerprint + ): + raise ValidationError( + "The unavailable Mylar paths changed or have not been reviewed. " + "Analyze the paths again and acknowledge the current report." + ) + allowed = preview.can_confirm or ( + request.mylar3_allow_unresolved_paths and preview.can_continue_with_unresolved + ) + if not allowed or preview.path_map != request.mylar3_path_map: + raise ValidationError( + "The confirmed Mylar path mapping preview is blocked. Analyze the paths again." + ) + active_job_id = await session.scalar( sa_select(ImportJob.id) .where(ImportJob.status.in_(ACTIVE_IMPORT_JOB_STATUSES)) @@ -61,7 +167,31 @@ async def create_job( raise ValidationError("Search on add is now controlled by the global import policy.") monitored = request.monitored or search_on_add - ingest_policy = await load_library_ingest_policy(session) + baseline_ingest_policy = ( + await load_effective_library_ingest_policy( + session, + resolved_target_library_root_id, + ) + if resolved_target_library_root_id is not None + else await load_library_ingest_policy(session) + ) + future_root_policy_snapshot = ( + build_future_root_policy_snapshot( + request.future_root_policy.model_dump(mode="json"), + baseline_ingest_policy, + ) + if request.future_root_policy is not None + else None + ) + ingest_policy = ( + apply_future_root_policy_to_ingest_policy( + baseline_ingest_policy, + future_root_policy_snapshot, + ) + if future_root_policy_snapshot is not None + else baseline_ingest_policy + ) + source_layout_snapshot = resolve_source_layout_spec(request.source_layout.to_core()).to_dict() job = ImportJob( source_path=request.source_path, @@ -70,18 +200,35 @@ async def create_job( status=ImportJobStatus.PENDING, monitored=monitored, search_on_add=search_on_add, - target_library_root_id=request.target_library_root_id, + target_library_root_id=resolved_target_library_root_id, mylar3_path_map=request.mylar3_path_map, + mylar3_path_map_confirmed=request.mylar3_path_map_confirmed, cv_match_threshold=request.cv_match_threshold, min_files_per_series=request.min_files_per_series, file_formats=request.file_formats, progress_snapshot={}, progress_revision=0, control_request=ImportControlRequest.NONE, + file_handling_mode=request.file_handling_mode, + source_layout_snapshot=source_layout_snapshot, + future_layout_requested=request.future_layout_requested, + future_root_policy_snapshot=future_root_policy_snapshot, + future_root_policy_applied_at=None, + story_arc_import_requested=request.story_arc_import_requested, + story_arc_materialization_requested=request.story_arc_materialization_requested, ) apply_ingest_policy_to_import_job(job, ingest_policy) session.add(job) await session.flush() + if future_root_policy_snapshot is not None: + apply_ingest_policy_to_import_job( + job, + apply_future_root_policy_to_ingest_policy( + baseline_ingest_policy, + future_root_policy_snapshot, + source_import_job_id=job.id, + ), + ) job.progress_snapshot = initialize_progress_snapshot( job, mode="scan", diff --git a/src/pullbox/services/import_job_execution.py b/src/pullbox/services/import_job_execution.py index 44953165..3eab15b2 100644 --- a/src/pullbox/services/import_job_execution.py +++ b/src/pullbox/services/import_job_execution.py @@ -74,6 +74,7 @@ ProcessSeriesFilesFunc, RaiseIfCancelledFunc, RecordActionFunc, + RecordActionsFunc, ReportFileProgressFunc, SeriesServiceFunc, SlowItemDelayFunc, @@ -81,13 +82,38 @@ from pullbox.services.import_job_execution_types import ( ExecutionItemPlan as _ExecutionItemPlan, ) +from pullbox.services.import_managed_copy_preflight import ( + ManagedCopyPreflightError, + reopen_review_after_managed_copy_preflight_failure, + validate_managed_copy_preflight, +) from pullbox.services.import_progress_runtime import ( ImportProgressSettings, current_item_payload, import_group_progress_plan, - weighted_import_progress_pct, + import_work_progress, +) +from pullbox.services.import_retry_helpers import require_retained_import_destination +from pullbox.services.import_root_policy_activation import ( + RootPolicyActivationConflictError, + activate_future_root_policy, +) +from pullbox.services.import_story_arc_materialization import ( + StoryArcMaterializationResult, + materialize_confirmed_story_arcs, +) +from pullbox.services.import_story_arc_placement_completion import ( + seal_import_story_arc_placement_origin, +) +from pullbox.services.import_story_arc_resolution import ( + StoryArcResolutionResult, + resolve_staged_story_arc_entries, +) +from pullbox.services.import_story_arc_review import ( + auto_confirm_trusted_logical_story_arcs, ) from pullbox.services.import_workflow_state import ( + deferred_recovery_scope, emit_live_progress, ) @@ -214,12 +240,38 @@ async def execute_import_job( estimate_remaining_seconds: EstimateRemainingFunc, maybe_slow_item_delay: SlowItemDelayFunc, progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None = None, + record_actions: RecordActionsFunc | None = None, ) -> None: """Execute confirmed new-series imports plus duplicate-series file merges.""" loaded_job = await session.get(ImportJob, job_id) if loaded_job is None: raise NotFoundError("ImportJob", job_id) job = loaded_job + require_retained_import_destination(job) + + try: + await validate_managed_copy_preflight(session, job, stage="execution") + except ManagedCopyPreflightError as exc: + await reopen_review_after_managed_copy_preflight_failure(session, job, exc) + capacity = exc.snapshot + await log_event( + session, + job_id, + "ERROR", + "managed_copy_preflight_blocked", + message=exc.message, + reason=exc.reason.value, + target_library_root_id=( + capacity.target_library_root_id + if capacity is not None + else job.target_library_root_id + ), + selected_source_bytes=(capacity.selected_source_bytes if capacity else None), + reserve_bytes=(capacity.reserve_bytes if capacity else None), + required_bytes=(capacity.required_bytes if capacity else None), + free_bytes=(capacity.free_bytes if capacity else None), + ) + return if job.import_started_at is None: job.import_started_at = datetime.now(UTC) @@ -234,6 +286,12 @@ async def execute_import_job( confirmed_ids={item.id for item in confirmed_items}, ) execution_items = _build_execution_item_plans(confirmed_items, duplicate_items) + recovery_scope = deferred_recovery_scope(job) + if recovery_scope is not None: + authorized_ids = set(recovery_scope) + confirmed_items = [item for item in confirmed_items if item.id in authorized_ids] + duplicate_items = [item for item in duplicate_items if item.id in authorized_ids] + execution_items = _build_execution_item_plans(confirmed_items, duplicate_items) await log_event( session, job_id, @@ -297,6 +355,8 @@ def queue_catalog_hydration(series_id: int, search_on_add: bool) -> None: runtime_revision_state=runtime_revision_state, ) + # Execution plans contain only remaining work, including after pause/restart. + work_started_at = datetime.now(UTC) _prime_series_prefetch_window( series_service=series_service, execution_items=execution_items, @@ -310,10 +370,10 @@ def queue_catalog_hydration(series_id: int, search_on_add: bool) -> None: job_id=job_id, job=job, job_started_at=job_started_at, + work_started_at=work_started_at, progress_callback=progress_callback, emit_progress=emit_progress, emit_live_progress=emit_live_progress, - estimate_remaining_seconds=estimate_remaining_seconds, group_progress_plans=group_progress_plans, shared_progress_settings=shared_progress_settings, group_progress_weights=group_progress_weights, @@ -357,9 +417,9 @@ def queue_catalog_hydration(series_id: int, search_on_add: bool) -> None: job_id=job_id, job=job, job_started_at=job_started_at, + work_started_at=work_started_at, progress_callback=progress_callback, progress_session_factory=progress_session_factory, - estimate_remaining_seconds=estimate_remaining_seconds, group_progress_plans=group_progress_plans, shared_progress_settings=shared_progress_settings, group_progress_weights=group_progress_weights, @@ -461,6 +521,7 @@ def queue_catalog_hydration(series_id: int, search_on_add: bool) -> None: job, item, process_series_files=process_series_files, + record_action=record_action, log_event=log_event, report_file_progress=report_file_progress, ) @@ -477,6 +538,34 @@ def queue_catalog_hydration(series_id: int, search_on_add: bool) -> None: total_files_imported += files_ok total_files_failed += files_err + policy_was_pending = job.future_root_policy_applied_at is None + try: + policy_action = await activate_future_root_policy( + session, + job, + successful_registration_count=total_files_imported, + ) + except RootPolicyActivationConflictError as exc: + job.error_message = exc.message + await log_event( + session, + job_id, + "ERROR", + "library_root_policy_activation_conflict", + message=exc.message, + target_library_root_id=job.target_library_root_id, + ) + else: + if policy_action is not None and policy_was_pending: + await log_event( + session, + job_id, + "INFO", + "library_root_policy_applied", + message="Future library layout activated for the selected root.", + target_library_root_id=job.target_library_root_id, + policy_revision=policy_action.payload.get("applied_revision"), + ) await session.commit() except JobPausedError: @@ -533,10 +622,10 @@ def queue_catalog_hydration(series_id: int, search_on_add: bool) -> None: item = refreshed_item if progress_callback: - progress = weighted_import_progress_pct( + work_progress = import_work_progress( group_progress_weights, current_group_index=idx, - current_group_progress_pct=100, + current_group_completed_weight=group_progress_weights[idx], ) job.series_imported = imported_count job.series_failed = failed_count @@ -551,15 +640,14 @@ def queue_catalog_hydration(series_id: int, search_on_add: bool) -> None: job_id=job_id, status=ImportJobStatus.IMPORTING, phase="importing", - progress=progress, + progress=work_progress.progress_pct, message=f"Processed {idx + 1}/{len(execution_items)} review groups", current_series=item_raw_series_name, current_series_status=( item.status if item is not None else ImportSeriesStatus.FAILED ), - estimated_seconds_remaining=estimate_remaining_seconds( - job_started_at, - progress, + estimated_seconds_remaining=work_progress.remaining_seconds( + work_started_at ), series_imported=imported_count, series_failed=failed_count, @@ -601,26 +689,151 @@ def queue_catalog_hydration(series_id: int, search_on_add: bool) -> None: job.series_failed = failed_count job.total_files_imported = total_files_imported job.total_files_failed = total_files_failed - job.status = ImportJobStatus.COMPLETED - job.import_completed_at = datetime.now(UTC) - await session.flush() + if recovery_scope is not None: + from pullbox.services.import_counters import ( + recompute_file_counters, + recompute_series_counters, + ) - await log_event( + await recompute_file_counters(session, job, series_ids=list(recovery_scope)) + await recompute_series_counters(session, job) + snapshot = dict(job.progress_snapshot or {}) + recovery = dict(snapshot.get("deferred_recovery") or {}) + recovery["state"] = "completed" + job.status = ImportJobStatus.COMPLETED + job.progress_snapshot = { + **snapshot, + "deferred_recovery": recovery, + "status": "completed", + "phase": "done", + "progress": 100, + "message": "Deferred file recovery completed. Remaining decisions are in Follow-up.", + } + await log_event( + session, + job_id, + "INFO", + "import_deferred_recovery_completed", + message="Completed the scoped recovery without executing unrelated review groups.", + series_ids=list(recovery_scope), + ) + for request in pending_catalog_hydrations: + _schedule_catalog_hydration( + session, + series_service=series_service, + series_id=request.series_id, + search_on_add=request.search_on_add, + ) + await session.flush() + return + job, story_arc_materialization = await _execute_story_arc_materialization( session, - job_id, - "INFO", - "import_completed", - message=( - f"Import complete: {imported_count} series imported, " - f"{failed_count} series failed, " - f"{total_files_imported} files imported, " - f"{total_files_failed} files failed" - ), - imported=imported_count, - failed=failed_count, - files_imported=total_files_imported, - files_failed=total_files_failed, + job, + job_id=job_id, + raise_if_cancelled=raise_if_cancelled, + record_action=record_action, + record_actions=record_actions, + log_event=log_event, + emit_progress=emit_progress, + estimate_remaining_seconds=estimate_remaining_seconds, + progress_callback=progress_callback, + runtime_revision_state=runtime_revision_state, + job_started_at=job_started_at, ) + job.series_imported = imported_count + job.series_failed = failed_count + job.total_files_imported = total_files_imported + job.total_files_failed = total_files_failed + placement_counts = await seal_import_story_arc_placement_origin(session, job_id) + if placement_counts.total: + job.status = ImportJobStatus.IMPORTING + job.import_completed_at = None + snapshot = dict(job.progress_snapshot or {}) + snapshot.update( + { + "status": ImportJobStatus.IMPORTING.value, + "mode": "import", + "phase": "story_arc_placements", + "progress": 99, + "message": ( + "Creating the approved story-arc copies and links after the import " + "transaction commits..." + ), + "story_arc_placements_total": placement_counts.total, + "story_arc_placements_queued": placement_counts.queued, + "story_arc_placements_running": placement_counts.running, + "story_arc_placements_retry_wait": placement_counts.retry_wait, + "story_arc_placements_completed": placement_counts.completed, + "story_arc_placements_failed": placement_counts.failed, + "story_arc_placements_cancelled": placement_counts.cancelled, + } + ) + job.progress_snapshot = snapshot + await session.flush() + await log_event( + session, + job_id, + "INFO", + "story_arc_placements_queued", + message=(f"Tracking {placement_counts.total} approved story-arc placements."), + total=placement_counts.total, + queued=placement_counts.queued, + reused=story_arc_materialization.managed_placements_reused, + ) + if progress_callback is not None: + runtime_revision_state["value"] += 1 + job.progress_revision = runtime_revision_state["value"] + await emit_progress( + session, + job, + ImportProgressEvent( + job_id=job_id, + status=ImportJobStatus.IMPORTING, + mode="import", + phase="story_arc_placements", + progress=99, + message=( + "Creating the approved story-arc copies and links after the " + "import transaction commits..." + ), + estimated_seconds_remaining=None, + series_found=job_series_found, + series_imported=imported_count, + series_failed=failed_count, + total_files_imported=total_files_imported, + total_files_failed=total_files_failed, + story_arc_placements_total=placement_counts.total, + story_arc_placements_queued=placement_counts.queued, + story_arc_placements_running=placement_counts.running, + story_arc_placements_retry_wait=placement_counts.retry_wait, + story_arc_placements_failed=placement_counts.failed, + story_arc_placements_completed=placement_counts.completed, + story_arc_placements_cancelled=placement_counts.cancelled, + progress_revision=runtime_revision_state["value"], + ), + progress_callback, + ) + else: + job.status = ImportJobStatus.COMPLETED + job.import_completed_at = datetime.now(UTC) + await session.flush() + + await log_event( + session, + job_id, + "INFO", + "import_completed", + message=( + f"Import complete: {imported_count} series imported, " + f"{failed_count} series failed, " + f"{total_files_imported} files imported, " + f"{total_files_failed} files failed" + ), + imported=imported_count, + failed=failed_count, + files_imported=total_files_imported, + files_failed=total_files_failed, + ) for request in pending_catalog_hydrations: _schedule_catalog_hydration( session, @@ -630,6 +843,185 @@ def queue_catalog_hydration(series_id: int, search_on_add: bool) -> None: ) +async def _execute_story_arc_materialization( + session: AsyncSession, + job: ImportJob, + *, + job_id: int, + raise_if_cancelled: RaiseIfCancelledFunc, + record_action: RecordActionFunc, + record_actions: RecordActionsFunc | None, + log_event: LogEventFunc, + emit_progress: EmitProgressFunc, + estimate_remaining_seconds: EstimateRemainingFunc, + progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None, + runtime_revision_state: dict[str, int], + job_started_at: datetime | None, +) -> tuple[ImportJob, StoryArcMaterializationResult]: + """Resolve and register confirmed arcs through restart-safe durable pages.""" + + async def cancellation_checkpoint() -> None: + await raise_if_cancelled(session, job_id) + + async def durable_story_arc_checkpoint() -> None: + """Commit one page, yield the writer, then read control in a fresh transaction.""" + await session.commit() + await asyncio.sleep(0) + try: + await raise_if_cancelled(session, job_id) + except BaseException: + await session.rollback() + raise + await session.commit() + + try: + build_snapshot = dict(job.progress_snapshot or {}) + build_snapshot.update( + { + "status": ImportJobStatus.IMPORTING.value, + "mode": "import", + "phase": "story_arcs", + "progress": 98, + "message": "Registering approved story arcs in durable batches...", + } + ) + job.progress_snapshot = build_snapshot + await session.flush() + await durable_story_arc_checkpoint() + resolution = await resolve_staged_story_arc_entries( + session, + import_job_id=job_id, + cancellation_check=cancellation_checkpoint, + durable_checkpoint=durable_story_arc_checkpoint, + ) + await auto_confirm_trusted_logical_story_arcs(session, job_id) + await durable_story_arc_checkpoint() + materialization = await materialize_confirmed_story_arcs( + session, + import_job_id=job_id, + cancellation_check=cancellation_checkpoint, + durable_checkpoint=durable_story_arc_checkpoint, + record_action=record_action, + record_actions=record_actions, + ) + except (JobPausedError, JobCancelledError): + raise + except Exception as exc: + # Earlier canonical and story-arc pages may already be durable. Discard + # only the current page, preserve every committed ownership pointer and + # journal row, and leave the optional arc work for follow-up without + # invalidating the canonical comic import. + await session.rollback() + persisted_job = await session.get(ImportJob, job_id) + if persisted_job is None: + raise NotFoundError("ImportJob", job_id) from exc + failure_message = "Some story arcs need follow-up; canonical comics imported successfully." + persisted_job.status = ImportJobStatus.IMPORTING + persisted_job.import_completed_at = None + persisted_job.error_message = failure_message + failure_snapshot = dict(persisted_job.progress_snapshot or {}) + failure_snapshot.update( + { + "status": ImportJobStatus.IMPORTING.value, + "mode": "import", + # Keep the seal fence valid so the outer import can publish any + # already-durable placement work before it completes. + "phase": "story_arcs", + "progress": 99, + "message": failure_message, + } + ) + persisted_job.progress_snapshot = failure_snapshot + await log_event( + session, + job_id, + "ERROR", + "story_arc_materialization_failed", + message=failure_message, + failure_type=type(exc).__name__, + ) + await session.commit() + return persisted_job, StoryArcMaterializationResult(arcs_failed=1) + + warning_codes = sorted({warning.code for warning in materialization.warnings}) + level = "WARNING" if materialization.arcs_failed else "INFO" + if materialization.arcs_failed: + job.error_message = ( + "Some story arcs could not be registered; canonical files remain imported." + ) + await log_event( + session, + job_id, + level, + "story_arc_materialization_completed", + message=( + f"Story-arc registration complete: {materialization.arcs_examined} examined, " + f"{materialization.arcs_failed} failed." + ), + **_story_arc_log_counts(resolution, materialization), + warning_codes=warning_codes, + ) + if ( + progress_callback is not None + and materialization.arcs_examined + and not materialization.managed_placements_queued + ): + runtime_revision_state["value"] += 1 + job.progress_revision = runtime_revision_state["value"] + await emit_progress( + session, + job, + ImportProgressEvent( + job_id=job_id, + status=ImportJobStatus.IMPORTING, + mode="import", + phase="importing", + progress=99, + message=( + f"Registered {materialization.arcs_examined - materialization.arcs_failed}/" + f"{materialization.arcs_examined} story arcs." + ), + estimated_seconds_remaining=estimate_remaining_seconds(job_started_at, 99), + series_found=int(job.series_found or 0), + series_imported=int(job.series_imported or 0), + series_failed=int(job.series_failed or 0), + total_files_imported=int(job.total_files_imported or 0), + total_files_failed=int(job.total_files_failed or 0), + progress_revision=runtime_revision_state["value"], + ), + progress_callback, + ) + return job, materialization + + +def _story_arc_log_counts( + resolution: StoryArcResolutionResult, + materialization: StoryArcMaterializationResult, +) -> dict[str, int]: + """Return path- and identity-free counters safe for durable job logs.""" + return { + "entries_examined": resolution.entries_examined, + "entries_resolved": resolution.resolved, + "entries_pending": resolution.pending, + "entries_missing": resolution.missing, + "entries_ambiguous": resolution.ambiguous, + "entries_conflicted": resolution.conflicts, + "entries_skipped": resolution.skipped, + "files_linked": resolution.linked_files, + "arcs_examined": materialization.arcs_examined, + "arcs_created": materialization.arcs_created, + "arcs_merged": materialization.arcs_merged, + "arcs_reused": materialization.arcs_reused, + "arcs_failed": materialization.arcs_failed, + "memberships_created": materialization.memberships_created, + "memberships_reused": materialization.memberships_reused, + "resolved_memberships": materialization.resolved_entries, + "unresolved_memberships": materialization.unresolved_entries, + "managed_placements_queued": materialization.managed_placements_queued, + "managed_placements_reused": materialization.managed_placements_reused, + } + + async def _load_confirmed_import_series( session: AsyncSession, job_id: int, diff --git a/src/pullbox/services/import_job_execution_items.py b/src/pullbox/services/import_job_execution_items.py index 7fc9888f..c22b739c 100644 --- a/src/pullbox/services/import_job_execution_items.py +++ b/src/pullbox/services/import_job_execution_items.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextlib import suppress from typing import TYPE_CHECKING, Any from sqlalchemy import select as sa_select @@ -10,14 +11,28 @@ ImportedFile, ImportedFileStatus, ImportedSeries, + ImportFileHandlingMode, ImportJob, ImportSeriesStatus, ) +from pullbox.models.issue import IssueType from pullbox.models.series import Series from pullbox.providers.base import IssueSummary from pullbox.services.import_catalog_hydration import schedule_catalog_hydration +from pullbox.services.import_file_issue_signals import ( + candidate_issue_number, + candidate_issue_number_text, +) from pullbox.services.import_file_resolution import load_importable_files +from pullbox.services.import_job_actions import ( + build_series_cover_cache_action_payload, + build_series_cover_path_updated_action_payload, + build_series_created_action_payload, + build_series_folder_created_action_payload, + build_series_monitoring_updated_action_payload, +) from pullbox.services.import_job_execution_progress import progress_session_factory_for_runtime +from pullbox.services.import_split_series import apply_import_preferred_series_root if TYPE_CHECKING: from collections.abc import Callable @@ -71,6 +86,9 @@ async def execute_new_series( importable_files = await load_importable_files(session, item) if not importable_files: + if await retain_imported_series_outcome(session, item): + await session.flush() + return 0, 0, imported_count + 1, failed_count, True item.status = ImportSeriesStatus.FAILED item.error_message = "No eligible files available for import" failed_count += 1 @@ -91,8 +109,36 @@ async def execute_new_series( # are running. That prevents autoflush from opening a SQLite write # transaction before the slow work is finished. with session.no_autoflush: - existing_series_id = await session.scalar( - sa_select(Series.id).where(Series.comicvine_id == cv_id) + current_library_root_id = ( + None + if job.file_handling_mode == ImportFileHandlingMode.IN_PLACE + else job.target_library_root_id + ) + existing_series_row = ( + await session.execute( + sa_select( + Series.id, + Series.cover_path, + Series.path, + Series.library_root_id, + Series.preferred_library_root_id, + Series.monitored, + ).where(Series.comicvine_id == cv_id) + ) + ).one_or_none() + existing_series_id = existing_series_row[0] if existing_series_row is not None else None + existing_series_cover_path = ( + existing_series_row[1] if existing_series_row is not None else None + ) + existing_series_path = existing_series_row[2] if existing_series_row is not None else None + existing_series_library_root_id = ( + existing_series_row[3] if existing_series_row is not None else None + ) + existing_series_preferred_root_id = ( + existing_series_row[4] if existing_series_row is not None else None + ) + existing_series_monitored = ( + bool(existing_series_row[5]) if existing_series_row is not None else False ) targeted_descriptor = getattr( type(series_service), @@ -112,7 +158,7 @@ async def execute_new_series( new_series = await add_from_import_review_targeted( session, import_series=item, - library_root_id=job.target_library_root_id, + library_root_id=current_library_root_id, search_on_add=job.search_on_add, issue_summaries=targeted_issue_summaries_for_import_files(importable_files), ) @@ -125,7 +171,7 @@ async def execute_new_series( new_series = await add_from_comicvine_prefetched( session, comicvine_id=cv_id, - library_root_id=job.target_library_root_id, + library_root_id=current_library_root_id, search_on_add=job.search_on_add, series_meta=series_meta, issue_summaries=issue_summaries, @@ -134,19 +180,93 @@ async def execute_new_series( new_series = await series_service.add_from_comicvine( session, comicvine_id=cv_id, - library_root_id=job.target_library_root_id, + library_root_id=current_library_root_id, search_on_add=job.search_on_add, ) new_series_id = new_series.id + # Persist the review-row ownership link before file work begins. A + # cooperative cancellation can interrupt that work, and the rollback + # journal must still be able to prove which import created this series. + item.series_id = new_series_id + await apply_import_preferred_series_root( + session, + job, + series_id=new_series_id, + record_action=record_action, + ) if existing_series_id is None: await record_action( session, job, phase="import", action_type="series_created", - payload={"series_id": new_series_id, "import_series_id": item_id}, + payload=await build_series_created_action_payload( + session, + series_id=new_series_id, + import_series_id=item_id, + ), + ) + else: + monitoring_action_payload = await build_series_monitoring_updated_action_payload( + session, + series_id=new_series_id, + import_series_id=item_id, + previous_monitored=existing_series_monitored, + ) + if monitoring_action_payload is not None: + await record_action( + session, + job, + phase="import", + action_type="series_monitoring_updated", + payload=monitoring_action_payload, + ) + folder_action_payload = await build_series_folder_created_action_payload( + session, + series_id=new_series_id, + import_series_id=item_id, + previous_series_path=existing_series_path, + previous_library_root_id=existing_series_library_root_id, + previous_preferred_library_root_id=existing_series_preferred_root_id, + ) + if folder_action_payload is not None: + await record_action( + session, + job, + phase="import", + action_type="series_folder_created", + payload=folder_action_payload, + ) + cover_action_payload = await build_series_cover_cache_action_payload( + session, + series_id=new_series_id, + import_series_id=item_id, + previous_cover_path=existing_series_cover_path, ) + if cover_action_payload is not None: + await record_action( + session, + job, + phase="import", + action_type="series_cover_cache_created", + payload=cover_action_payload, + ) + else: + cover_path_action_payload = await build_series_cover_path_updated_action_payload( + session, + series_id=new_series_id, + import_series_id=item_id, + previous_cover_path=existing_series_cover_path, + ) + if cover_path_action_payload is not None: + await record_action( + session, + job, + phase="import", + action_type="series_cover_path_updated", + payload=cover_path_action_payload, + ) await log_event( session, @@ -195,6 +315,9 @@ def queue_or_schedule_catalog_hydration() -> None: queue_or_schedule_catalog_hydration() await session.flush() return files_ok, files_err, imported_count, failed_count, True + if await retain_imported_series_outcome(session, item): + await session.flush() + return files_ok, files_err, imported_count + 1, failed_count, True item.status = ImportSeriesStatus.FAILED item.error_message = "No eligible files available for import" failed_count += 1 @@ -218,6 +341,30 @@ def queue_or_schedule_catalog_hydration() -> None: return files_ok, files_err, imported_count, failed_count, True +async def retain_imported_series_outcome(session: AsyncSession, item: ImportedSeries) -> bool: + """An exhausted retry must not erase an earlier successful partial import.""" + if item.series_id is None: + return False + imported_file_id = await session.scalar( + sa_select(ImportedFile.id) + .where( + ImportedFile.import_series_id == item.id, + ImportedFile.status == ImportedFileStatus.IMPORTED, + ) + .limit(1) + ) + if imported_file_id is None: + return False + if item.error_message: + item.diagnostics = { + **dict(item.diagnostics or {}), + "previous_series_error": item.error_message, + } + item.status = ImportSeriesStatus.IMPORTED + item.error_message = None + return True + + async def has_safety_blocked_files(session: AsyncSession, imported_series_id: int) -> bool: """Return whether a review row has deferred resource-safety file decisions.""" safety_file_id = await session.scalar( @@ -235,14 +382,14 @@ def targeted_issue_summaries_for_import_files(files: list[ImportedFile]) -> list """Build Step 4 issue summaries from review-time file matches.""" summaries: list[IssueSummary] = [] seen_provider_ids: set[str] = set() - seen_numbers: set[float] = set() + seen_issue_keys: set[str] = set() for imp_file in files: + if not ensure_target_issue_summary_for_import_file(imp_file): + continue diagnostics = imp_file.diagnostics if isinstance(imp_file.diagnostics, dict) else {} summary_payload = diagnostics.get("target_issue_summary") issue_number: float | None if not isinstance(summary_payload, dict): - if imp_file.matched_issue_cv_id is not None and imp_file.matched_issue_id is None: - raise ValueError("Import file is missing required target_issue_summary diagnostics") continue provider_id = str(summary_payload.get("provider_id") or "").strip() @@ -254,14 +401,16 @@ def targeted_issue_summaries_for_import_files(files: list[ImportedFile]) -> list release_date = summary_payload.get("release_date") cover_url = summary_payload.get("cover_url") issue_type = str(summary_payload["issue_type"]) + issue_number_text = str(summary_payload.get("issue_number_text") or "").strip() or None if not provider_id or issue_number is None: continue number_key = float(issue_number) - if provider_id in seen_provider_ids or number_key in seen_numbers: + issue_key = issue_number_text or str(number_key) + if provider_id in seen_provider_ids or issue_key in seen_issue_keys: continue seen_provider_ids.add(provider_id) - seen_numbers.add(number_key) + seen_issue_keys.add(issue_key) summaries.append( IssueSummary( provider_id=provider_id, @@ -270,11 +419,65 @@ def targeted_issue_summaries_for_import_files(files: list[ImportedFile]) -> list release_date=str(release_date) if release_date else None, cover_url=str(cover_url) if cover_url else None, issue_type=issue_type, + issue_number_text=issue_number_text, ) ) return summaries +def ensure_target_issue_summary_for_import_file(imp_file: ImportedFile) -> bool: + """Repair the cached issue target needed to create a new series in Step 4.""" + if imp_file.matched_issue_id is not None or imp_file.matched_issue_cv_id is None: + return True + + diagnostics = dict(imp_file.diagnostics or {}) + existing = diagnostics.get("target_issue_summary") + payload = dict(existing) if isinstance(existing, dict) else {} + provider_id = str(payload.get("provider_id") or imp_file.matched_issue_cv_id).strip() + raw_issue_number = payload.get("issue_number") + issue_number: float | None = None + if raw_issue_number is not None: + with suppress(ValueError): + issue_number = float(str(raw_issue_number)) + if issue_number is None: + issue_number = candidate_issue_number(imp_file) + if not provider_id or issue_number is None: + diagnostics.update( + { + "reason": "target_issue_summary_unavailable", + "rejection_reason": ( + "The matched issue target is incomplete and must be reviewed before import." + ), + } + ) + imp_file.diagnostics = diagnostics + return False + + raw_issue_type = payload.get("issue_type") or diagnostics.get("source_issue_type") + try: + issue_type = IssueType(str(raw_issue_type or IssueType.ISSUE.value)) + except ValueError: + issue_type = IssueType.ISSUE + issue_number_text = str(payload.get("issue_number_text") or "").strip() + if not issue_number_text: + issue_number_text = candidate_issue_number_text(imp_file) or "" + repaired = { + "provider_id": provider_id, + "issue_number": issue_number, + "title": payload.get("title"), + "release_date": payload.get("release_date"), + "cover_url": payload.get("cover_url"), + "issue_type": issue_type.value, + } + if issue_number_text: + repaired["issue_number_text"] = issue_number_text + diagnostics["target_issue_summary"] = repaired + diagnostics.pop("reason", None) + diagnostics.pop("rejection_reason", None) + imp_file.diagnostics = diagnostics + return True + + def schedule_catalog_hydration_for_series( session: AsyncSession, *, @@ -297,6 +500,7 @@ async def execute_duplicate_series_merge( item: ImportedSeries, *, process_series_files: ProcessSeriesFilesFunc, + record_action: RecordActionFunc, log_event: LogEventFunc, report_file_progress: ReportFileProgressFunc | None = None, ) -> tuple[int, int, bool]: @@ -306,6 +510,13 @@ async def execute_duplicate_series_merge( await session.commit() return 0, 0, False + await apply_import_preferred_series_root( + session, + job, + series_id=item.series_id, + record_action=record_action, + ) + await log_event( session, job.id, diff --git a/src/pullbox/services/import_job_execution_progress.py b/src/pullbox/services/import_job_execution_progress.py index 4f63eec9..41a26ffb 100644 --- a/src/pullbox/services/import_job_execution_progress.py +++ b/src/pullbox/services/import_job_execution_progress.py @@ -6,12 +6,16 @@ from typing import TYPE_CHECKING, Any import structlog +from sqlalchemy import and_, or_ +from sqlalchemy import func as sa_func +from sqlalchemy import select as sa_select from sqlalchemy.ext.asyncio import async_sessionmaker from pullbox.core.exceptions import JobCancelledError, JobPausedError from pullbox.core.sqlite_lock import is_sqlite_locked_error from pullbox.models.import_job import ( ImportedFile, + ImportedFileStatus, ImportedSeries, ImportJob, ImportJobStatus, @@ -22,16 +26,15 @@ ActiveFileProgressSettings, active_file_progress_pct, ) -from pullbox.services.import_file_resolution import load_importable_files from pullbox.services.import_progress_runtime import ( ImportGroupProgressPlan, ImportProgressFileProfile, ImportProgressSettings, current_item_payload, - import_group_file_progress_pct, - import_group_metadata_progress_pct, + import_group_file_completed_weight, + import_group_metadata_completed_weight, import_group_progress_plan, - weighted_import_progress_pct, + import_work_progress, ) if TYPE_CHECKING: @@ -41,7 +44,6 @@ from sqlalchemy.ext.asyncio import AsyncSession from pullbox.services.import_job_execution_types import ( - EstimateRemainingFunc, ExecutionItemPlan, RaiseIfCancelledFunc, ReportFileProgressFunc, @@ -49,6 +51,36 @@ logger = structlog.get_logger(__name__) +_IMPORT_PROGRESS_PLAN_BATCH_SIZE = 500 +_IMPORTABLE_FILE_STATUSES = ( + ImportedFileStatus.MATCHED, + ImportedFileStatus.CONFIRMED, +) + + +async def reconcile_durable_import_execution_counters( + session: AsyncSession, + job: ImportJob, +) -> None: + """Rebuild execution totals from file and series rows committed before interruption.""" + file_status_result = await session.execute( + sa_select(ImportedFile.status, sa_func.count(ImportedFile.id)) + .where(ImportedFile.import_job_id == job.id) + .group_by(ImportedFile.status) + ) + file_status_counts = {status: count for status, count in file_status_result.all()} + series_status_result = await session.execute( + sa_select(ImportedSeries.status, sa_func.count(ImportedSeries.id)) + .where(ImportedSeries.import_job_id == job.id) + .group_by(ImportedSeries.status) + ) + series_status_counts = {status: count for status, count in series_status_result.all()} + + job.total_files_imported = file_status_counts.get(ImportedFileStatus.IMPORTED, 0) + job.total_files_failed = file_status_counts.get(ImportedFileStatus.FAILED, 0) + job.series_imported = series_status_counts.get(ImportSeriesStatus.IMPORTED, 0) + job.series_failed = series_status_counts.get(ImportSeriesStatus.FAILED, 0) + async def build_import_group_progress_plans( session: AsyncSession, @@ -56,27 +88,60 @@ async def build_import_group_progress_plans( settings: ImportProgressSettings, ) -> dict[int, ImportGroupProgressPlan]: """Build weighted progress plans for selected Step 4 review groups.""" - plans: dict[int, ImportGroupProgressPlan] = {} - for item_plan in execution_items: - item = await session.get(ImportedSeries, item_plan.item_id) - if item is None: - plans[item_plan.item_id] = import_group_progress_plan(settings, []) - continue - files = await load_importable_files( - session, - item, - duplicate_mode=item_plan.mode == "duplicate", + modes_by_item_id = {item.item_id: item.mode for item in execution_items} + profiles_by_item_id: dict[int, list[ImportProgressFileProfile]] = { + item.item_id: [] for item in execution_items + } + + for start in range(0, len(execution_items), _IMPORT_PROGRESS_PLAN_BATCH_SIZE): + batch = execution_items[start : start + _IMPORT_PROGRESS_PLAN_BATCH_SIZE] + batch_ids = [item.item_id for item in batch] + files_result = await session.execute( + sa_select(ImportedFile) + .select_from(ImportedSeries) + .join( + ImportedFile, + and_( + ImportedFile.import_job_id == ImportedSeries.import_job_id, + ImportedFile.import_series_id == ImportedSeries.id, + ), + ) + .where( + ImportedSeries.id.in_(batch_ids), + or_( + ImportedFile.status.in_(_IMPORTABLE_FILE_STATUSES), + and_( + ImportedFile.status == ImportedFileStatus.CONFLICT, + ImportedFile.is_preferred.is_(True), + ), + ), + ) + .order_by(ImportedFile.import_series_id.asc(), ImportedFile.id.asc()) ) - profiles = [ - ImportProgressFileProfile( - file_id=imp_file.id, - file_path=imp_file.file_path, - file_size=imp_file.file_size, + for imp_file in files_result.scalars().all(): + item_id = int(imp_file.import_series_id) + mode = modes_by_item_id.get(item_id) + if mode is None: + continue + if mode == "duplicate" and ( + imp_file.status not in _IMPORTABLE_FILE_STATUSES or not imp_file.include_in_import + ): + continue + profiles_by_item_id[item_id].append( + ImportProgressFileProfile( + file_id=imp_file.id, + file_path=imp_file.file_path, + file_size=imp_file.file_size, + ) ) - for imp_file in files - ] - plans[item_plan.item_id] = import_group_progress_plan(settings, profiles) - return plans + + return { + item.item_id: import_group_progress_plan( + settings, + profiles_by_item_id[item.item_id], + ) + for item in execution_items + } def progress_session_factory_for_runtime( @@ -140,9 +205,9 @@ def build_report_file_progress_callback( job_id: int, job: ImportJob, job_started_at: datetime | None, + work_started_at: datetime, progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None, progress_session_factory: async_sessionmaker[AsyncSession] | None, - estimate_remaining_seconds: EstimateRemainingFunc, group_progress_plans: dict[int, ImportGroupProgressPlan], shared_progress_settings: ImportProgressSettings, group_progress_weights: list[float], @@ -185,15 +250,15 @@ async def report_file_progress( series_id, import_group_progress_plan(shared_progress_settings, []), ) - group_progress_pct = import_group_file_progress_pct( + group_completed_weight = import_group_file_completed_weight( group_plan, file_index=file_index, current_file_pct=current_file_pct, ) - overall_progress = weighted_import_progress_pct( + work_progress = import_work_progress( group_progress_weights, current_group_index=group_index, - current_group_progress_pct=group_progress_pct, + current_group_completed_weight=group_completed_weight, ) loop_now = monotonic_time() emitted_at_value = progress_state.get("emitted_at") @@ -231,7 +296,7 @@ async def report_file_progress( ephemeral_progress=not persist_progress, mode="import", phase="importing", - progress=overall_progress, + progress=work_progress.progress_pct, message=( f"Processing file {file_index}/{max(total_files, 1)} " f"in review group {group_index + 1}/{max(total_groups, 1)}" @@ -247,10 +312,7 @@ async def report_file_progress( current_file_progress_unit=unit, current_series=series_name, current_series_status=ImportSeriesStatus.IMPORTING, - estimated_seconds_remaining=estimate_remaining_seconds( - job_started_at, - overall_progress, - ), + estimated_seconds_remaining=work_progress.remaining_seconds(work_started_at), series_imported=int(stats["series_imported"]), series_failed=int(stats["series_failed"]), series_found=series_found, @@ -295,10 +357,10 @@ def build_series_metadata_progress_emitter( job_id: int, job: ImportJob, job_started_at: datetime | None, + work_started_at: datetime, progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None, emit_progress: Callable[..., Awaitable[None]], emit_live_progress: Callable[..., Awaitable[None]], - estimate_remaining_seconds: EstimateRemainingFunc, group_progress_plans: dict[int, ImportGroupProgressPlan], shared_progress_settings: ImportProgressSettings, group_progress_weights: list[float], @@ -326,14 +388,14 @@ async def emit_series_metadata_progress( series_id, import_group_progress_plan(shared_progress_settings, []), ) - group_progress = import_group_metadata_progress_pct( + group_completed_weight = import_group_metadata_completed_weight( group_plan, metadata_progress_pct=current_item_progress_pct, ) - progress = weighted_import_progress_pct( + work_progress = import_work_progress( group_progress_weights, current_group_index=group_index, - current_group_progress_pct=group_progress, + current_group_completed_weight=group_completed_weight, ) current_stats = stats() event = ImportProgressEvent( @@ -341,16 +403,13 @@ async def emit_series_metadata_progress( status=ImportJobStatus.IMPORTING, mode="import", phase="importing", - progress=progress, + progress=work_progress.progress_pct, message=message, current_series_id=series_id, current_series_name=series_name, current_series=series_name, current_series_status=ImportSeriesStatus.IMPORTING, - estimated_seconds_remaining=estimate_remaining_seconds( - job_started_at, - progress, - ), + estimated_seconds_remaining=work_progress.remaining_seconds(work_started_at), series_imported=current_stats["series_imported"], series_failed=current_stats["series_failed"], series_found=series_found, diff --git a/src/pullbox/services/import_job_execution_types.py b/src/pullbox/services/import_job_execution_types.py index 07e733d9..34d3594f 100644 --- a/src/pullbox/services/import_job_execution_types.py +++ b/src/pullbox/services/import_job_execution_types.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Literal, Protocol if TYPE_CHECKING: - from collections.abc import Awaitable, Callable + from collections.abc import Awaitable, Callable, Sequence from datetime import datetime from sqlalchemy.ext.asyncio import AsyncSession @@ -20,6 +20,7 @@ from pullbox.models.series import Series from pullbox.providers.base import IssueSummary from pullbox.schemas.import_job import ImportProgressEvent + from pullbox.services.import_job_actions import ImportJobActionSpec class SeriesServiceFunc(Protocol): @@ -103,6 +104,15 @@ async def __call__( ) -> ImportJobAction: ... +class RecordActionsFunc(Protocol): + async def __call__( + self, + session: AsyncSession, + job: ImportJob, + specs: Sequence[ImportJobActionSpec], + ) -> list[ImportJobAction]: ... + + class LogEventFunc(Protocol): async def __call__( self, diff --git a/src/pullbox/services/import_known_series_recovery.py b/src/pullbox/services/import_known_series_recovery.py new file mode 100644 index 00000000..d077924d --- /dev/null +++ b/src/pullbox/services/import_known_series_recovery.py @@ -0,0 +1,308 @@ +"""Recovery of legacy series rejections from agreeing saved local identities.""" + +from __future__ import annotations + +import json +from collections import Counter +from dataclasses import asdict, dataclass +from hashlib import sha256 +from itertools import batched +from typing import TYPE_CHECKING, Any + +from sqlalchemy import exists, select + +from pullbox.core.issue_numbers import parse_issue_number_text +from pullbox.core.name_matcher import NameMatcher +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportSeriesStatus, + ImportSourceType, +) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile +from pullbox.models.series import Series +from pullbox.providers.base import IssueSummary +from pullbox.services.import_file_match_targets import trusted_source_issue_identity_matches_target +from pullbox.services.import_source_metadata import ( + build_import_metadata_conflict, + source_metadata_for_import_file, +) +from pullbox.services.import_terminal_recovery import allows_terminal_import_recovery + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +@dataclass(frozen=True) +class KnownSeriesRecovery: + file_id: int + series_id: int + cv_id: int + match_method: str + summary: dict[str, Any] + evidence_digest: str + + +def _positive_id(value: object) -> int | None: + try: + number = int(str(value)) + except (ValueError, TypeError): + return None + return number if number > 0 else None + + +def _known_identity( + item: ImportedSeries, + files: list[ImportedFile], + source_type: ImportSourceType, +) -> tuple[int, str] | None: + if ( + item.user_selected_cv_id is not None + or item.series_id is not None + or any( + file.status in {ImportedFileStatus.CONFLICT, ImportedFileStatus.IMPORTED} + for file in files + ) + ): + return None + diagnostics = dict(item.diagnostics or {}) + candidate = diagnostics.get("selected_candidate") + candidate = candidate if isinstance(candidate, dict) else {} + cv_id = _positive_id(candidate.get("cv_id")) + method = str(candidate.get("match_method") or "") + if candidate.get("title") and NameMatcher.normalize( + str(candidate["title"]) + ) != NameMatcher.normalize(item.raw_series_name): + return None + if source_type is ImportSourceType.MYLAR3: + mylar_ids = { + _positive_id(file.diagnostics.get("comicvine_series_id")) + for file in files + if isinstance(file.diagnostics, dict) + and isinstance(file.diagnostics.get("metadata_signals"), dict) + and file.diagnostics["metadata_signals"].get("comicvine_series_id") == "mylar3" + } - {None} + if len(mylar_ids) != 1: + return None + mylar_id = next(iter(mylar_ids)) + if cv_id is not None and (cv_id != mylar_id or method != "mylar3_cv_id"): + return None + if item.cv_id not in (None, mylar_id): + return None + assert mylar_id is not None + return mylar_id, "mylar3_cv_id" + if cv_id is None or method not in {"comicinfo_cv_id", "folder_cv_id"}: + return None + if item.cv_id not in (None, cv_id): + return None + # Folder imports have no authoritative Mylar row to resolve series disagreements. + conflicts = diagnostics.get("identity_conflicts", []) + if not isinstance(conflicts, list) or any( + not isinstance(conflict, dict) or conflict.get("field") != "comicvine_issue_id" + for conflict in conflicts + ): + return None + return cv_id, method + + +def _file_plan( + item: ImportedSeries, file: ImportedFile, cv_id: int, method: str +) -> KnownSeriesRecovery | None: + if file.status not in { + ImportedFileStatus.MATCHED, + ImportedFileStatus.CONFIRMED, + ImportedFileStatus.NO_MATCH, + }: + return None + diagnostics = dict(file.diagnostics or {}) + if diagnostics.get("kind") in { + "metadata_conflict", + "source_scope_review", + "source_layout_review", + }: + return None + if file.library_file_id is not None or file.conflict_group_id is not None: + return None + if (file.match_method or "").startswith(("manual", "orphan_recovery")): + return None + if any( + diagnostics.get(key) for key in ("safety_block", "source_revalidation", "safety_exception") + ): + return None + if ( + file.status is ImportedFileStatus.NO_MATCH + and diagnostics.get("kind") != "series_no_match_file" + ): + return None + target = ImportedSeries( + raw_series_name=item.raw_series_name, + raw_year=item.raw_year, + cv_id=cv_id, + cv_match_method=method, + ) + if not trusted_source_issue_identity_matches_target(target, file, file.comicvine_issue_id): + return None + if file.matched_issue_cv_id not in (None, file.comicvine_issue_id): + return None + metadata = source_metadata_for_import_file(target, file) + if metadata.issue_number is None: + return None + if ( + build_import_metadata_conflict( + metadata=metadata, + target_series_title=item.raw_series_name, + target_series_year=item.raw_year, + target_issue_number=metadata.issue_number, + target_issue_cv_id=file.comicvine_issue_id, + target_issue_title=None, + ) + is not None + ): + return None + raw_summary = diagnostics.get("target_issue_summary") + if raw_summary is not None: + if ( + not isinstance(raw_summary, dict) + or _positive_id(raw_summary.get("provider_id")) != file.comicvine_issue_id + ): + return None + try: + number, number_text = parse_issue_number_text( + str(raw_summary.get("issue_number_text") or raw_summary.get("issue_number")) + ) + except ValueError: + return None + if number != metadata.issue_number: + return None + summary = dict(raw_summary) + else: + if file.status is not ImportedFileStatus.NO_MATCH: + return None + number, number_text = parse_issue_number_text(str(metadata.issue_number)) + summary = asdict( + IssueSummary( + provider_id=str(file.comicvine_issue_id), + issue_number=number, + issue_number_text=number_text, + title=None, + release_date=None, + cover_url=None, + issue_type=metadata.issue_type.value, + ) + ) + evidence = { + "series": [item.id, item.updated_at.isoformat(), item.diagnostics], + "file": [file.id, file.updated_at.isoformat(), file.diagnostics, file.source_signature], + "identity": [cv_id, method, file.matched_issue_id, file.matched_issue_cv_id], + } + return KnownSeriesRecovery( + file.id, + item.id, + cv_id, + method, + summary, + sha256(json.dumps(evidence, sort_keys=True).encode()).hexdigest(), + ) + + +async def load_known_series_recovery( + session: AsyncSession, + job_id: int, +) -> tuple[KnownSeriesRecovery, ...]: + """Return only exact recoverable file identities without modifying the session.""" + job = await session.get(ImportJob, job_id) + if job is None or not allows_terminal_import_recovery(job): + return () + plans: list[KnownSeriesRecovery] = [] + matched_local_ids: dict[int, int | None] = {} + ready_files_by_series: dict[int, set[int]] = {} + cursor = 0 + while True: + items = list( + ( + await session.scalars( + select(ImportedSeries) + .where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.id > cursor, + ImportedSeries.status.in_( + (ImportSeriesStatus.NO_MATCH, ImportSeriesStatus.FAILED) + ), + ImportedSeries.diagnostics["reason"].as_string() + == "trusted_source_identity_conflict", + ) + .order_by(ImportedSeries.id) + .limit(100) + ) + ).all() + ) + if not items: + break + cursor = items[-1].id + for item in items: + files = list( + ( + await session.scalars( + select(ImportedFile) + .where( + ImportedFile.import_series_id == item.id, + ) + .order_by(ImportedFile.id) + ) + ).all() + ) + identity = _known_identity(item, files, job.source_type) + if identity is None: + continue + ready_files_by_series[item.id] = { + file.id + for file in files + if file.status in {ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED} + } + for file in files: + plan = _file_plan(item, file, *identity) + if plan is not None: + plans.append(plan) + matched_local_ids[file.id] = file.matched_issue_id + issue_ids = sorted({int(plan.summary["provider_id"]) for plan in plans}) + local_targets = {} + for ids in batched(issue_ids, 400): + for issue, series_cv_id, owned in ( + await session.execute( + select(Issue, Series.comicvine_id, exists().where(LibraryFile.issue_id == Issue.id)) + .join(Series, Series.id == Issue.series_id) + .where(Issue.comicvine_id.in_(ids)) + ) + ).all(): + local_targets[issue.comicvine_id] = (issue, series_cv_id, owned) + # Comic Vine issue IDs are globally unique, including across legacy series rows. + counts = Counter(int(plan.summary["provider_id"]) for plan in plans) + result = [] + for plan in plans: + if counts[int(plan.summary["provider_id"])] != 1: + continue + local = local_targets.get(int(plan.summary["provider_id"])) + if local is not None: + issue, series_cv_id, owned = local + if ( + series_cv_id != plan.cv_id + or owned + or issue.issue_number != plan.summary["issue_number"] + or matched_local_ids[plan.file_id] not in (None, issue.id) + ): + continue + elif matched_local_ids[plan.file_id] is not None: + continue + result.append(plan) + eligible_ids = {plan.file_id for plan in result} + # Step 4 consumes all ready files in a series. Never revive a parent if that + # would implicitly authorize an unpreviewed ready file or manual decision. + return tuple( + sorted( + (plan for plan in result if ready_files_by_series[plan.series_id] <= eligible_ids), + key=lambda plan: plan.file_id, + ) + ) diff --git a/src/pullbox/services/import_layout_analysis.py b/src/pullbox/services/import_layout_analysis.py new file mode 100644 index 00000000..b58b73da --- /dev/null +++ b/src/pullbox/services/import_layout_analysis.py @@ -0,0 +1,531 @@ +"""Bounded, read-only source layout analysis for collection imports.""" + +from __future__ import annotations + +import asyncio +import hashlib +import os +import time +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath + +from pullbox.core.collection_scanner import COMIC_EXTENSIONS, IGNORE_DIRS +from pullbox.core.filesystem_policy import is_sensitive_path, resolve_preview_source +from pullbox.core.issue_numbers import format_issue_number +from pullbox.core.library_layout import ( + CompiledSourceLayout, + ImportLayoutMode, + LayoutClassification, + LayoutValueError, + SourceLayoutMatch, + SourceLayoutSpec, + compile_source_layout, + resolve_source_layout_spec, +) +from pullbox.core.source_metadata import SourceMetadataExtractor + +_GENERIC_OR_TYPE_CONTAINERS = frozenset( + { + "annual", + "annuals", + "books", + "collection", + "collections", + "comics", + "downloads", + "imports", + "incoming", + "library", + "media", + "special", + "specials", + "staging", + "volume", + "volumes", + } +) + + +@dataclass(frozen=True, slots=True) +class LayoutAnalysisBudget: + """Server-owned hard limits for one preflight analysis.""" + + max_directories: int = 2_000 + max_files: int = 5_000 + max_examples_per_cluster: int = 3 + deadline_seconds: float = 2.0 + + def __post_init__(self) -> None: + if self.max_directories < 1: + raise ValueError("max_directories must be positive") + if self.max_files < 1: + raise ValueError("max_files must be positive") + if self.max_examples_per_cluster < 1: + raise ValueError("max_examples_per_cluster must be positive") + if self.deadline_seconds < 0: + raise ValueError("deadline_seconds cannot be negative") + + +@dataclass(frozen=True, slots=True) +class LayoutExample: + """One sanitized root-relative example from a detected cluster.""" + + relative_path: str + publisher: str | None + series: str | None + year: int | None + issue_number: str | None + issue_title: str | None + evidence: list[str] + warnings: list[str] + + +@dataclass(frozen=True, slots=True) +class LayoutClusterSummary: + """Bounded summary of paths sharing one interpreted layout.""" + + cluster_id: str + classification: LayoutClassification + file_count: int + directory_count: int + confidence: str + proposed_series_path_template: str | None + proposed_issue_filename_template: str | None + examples: list[LayoutExample] + + +@dataclass(frozen=True, slots=True) +class LayoutAnalysisResult: + """Read-only preflight result; not durable job state.""" + + effective_spec: SourceLayoutSpec + classification: LayoutClassification + clusters: list[LayoutClusterSummary] + directories_considered: int + files_considered: int + files_fitting: int + files_ambiguous: int + files_outside_root: int + archive_probes: int + can_keep_in_place: bool + can_apply_future_policy: bool + partial: bool + warnings: list[str] + + +@dataclass(slots=True) +class _ClusterAccumulator: + classification: LayoutClassification + confidence: str + proposed_series_path_template: str | None + proposed_issue_filename_template: str | None + file_count: int = 0 + directories: set[str] = field(default_factory=set) + examples: list[LayoutExample] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True) +class _AnalyzedPath: + cluster_key: str + classification: LayoutClassification + confidence: str + match: SourceLayoutMatch | None + proposed_series_path_template: str | None + proposed_issue_filename_template: str | None + evidence: list[str] + warnings: list[str] + + +class ImportLayoutAnalyzer: + """Analyze comic paths without provider calls, archive probes, or writes.""" + + def __init__(self, *, extensions: frozenset[str] | None = None) -> None: + self._extensions = extensions or COMIC_EXTENSIONS + self._metadata_extractor = SourceMetadataExtractor() + self._series_matcher = compile_source_layout( + SourceLayoutSpec(mode=ImportLayoutMode.PRESET, preset="series_folders") + ) + self._publisher_matcher = compile_source_layout( + SourceLayoutSpec(mode=ImportLayoutMode.PRESET, preset="publisher_series") + ) + + async def analyze( + self, + root_path: str | Path, + *, + spec: SourceLayoutSpec | None = None, + budget: LayoutAnalysisBudget | None = None, + cancel_event: asyncio.Event | None = None, + ) -> LayoutAnalysisResult: + """Return a deterministic, bounded source-layout analysis.""" + root = resolve_preview_source(root_path) + if not root.is_dir(): + raise ValueError("Layout analysis root must be a directory") + + effective_spec = resolve_source_layout_spec(spec or SourceLayoutSpec()) + selected_matcher = ( + compile_source_layout(effective_spec) + if effective_spec.mode != ImportLayoutMode.AUTO + else None + ) + limits = budget or LayoutAnalysisBudget() + deadline = time.monotonic() + limits.deadline_seconds + warnings: list[str] = [] + clusters: dict[str, _ClusterAccumulator] = {} + directories_considered = 0 + files_considered = 0 + files_fitting = 0 + files_ambiguous = 0 + files_outside_root = 0 + partial = False + sensitive_skipped = False + + self._raise_if_cancelled(cancel_event) + if self._deadline_reached(deadline): + return self._empty_partial_result(effective_spec, "deadline_reached") + + walk_errors: list[str] = [] + + def on_walk_error(_error: OSError) -> None: + walk_errors.append("unreadable_path_skipped") + + for current_root, dir_names, file_names in os.walk( + root, + topdown=True, + onerror=on_walk_error, + followlinks=False, + ): + self._raise_if_cancelled(cancel_event) + if self._deadline_reached(deadline): + partial = True + _append_once(warnings, "deadline_reached") + break + if directories_considered >= limits.max_directories: + partial = True + _append_once(warnings, "directory_limit_reached") + break + + current = Path(current_root) + if not _is_within_root(current, root): + dir_names.clear() + continue + safe_dir_names: list[str] = [] + for name in sorted(dir_names): + if name in IGNORE_DIRS or name.startswith("."): + continue + candidate = current / name + if candidate.is_symlink(): + partial = True + _append_once(warnings, "symlink_directory_skipped") + continue + if is_sensitive_path(candidate): + sensitive_skipped = True + _append_once(warnings, "sensitive_directory_skipped") + continue + safe_dir_names.append(name) + dir_names[:] = safe_dir_names + file_names.sort() + directories_considered += 1 + + for file_name in file_names: + self._raise_if_cancelled(cancel_event) + if self._deadline_reached(deadline): + partial = True + _append_once(warnings, "deadline_reached") + break + path = current / file_name + if path.suffix.lower() not in self._extensions or file_name.startswith("._"): + continue + if files_considered >= limits.max_files: + partial = True + _append_once(warnings, "file_limit_reached") + break + if not _is_within_root(path, root): + files_outside_root += 1 + files_ambiguous += 1 + _append_once(warnings, "outside_root_skipped") + continue + + relative_path = path.relative_to(root).as_posix() + files_considered += 1 + analyzed = self._analyze_path( + relative_path, + effective_spec=effective_spec, + selected_matcher=selected_matcher, + ) + if analyzed.classification == LayoutClassification.NEEDS_REVIEW: + files_ambiguous += 1 + else: + files_fitting += 1 + self._accumulate( + clusters, + analyzed, + relative_path=relative_path, + max_examples=limits.max_examples_per_cluster, + ) + if files_considered % 64 == 0: + await asyncio.sleep(0) + + if partial: + break + + for warning in walk_errors: + _append_once(warnings, warning) + if walk_errors: + partial = True + + partial = partial or sensitive_skipped + summaries = self._summaries(clusters) + classification = _overall_classification(summaries) + can_keep_in_place = files_outside_root == 0 and not walk_errors + can_apply_future_policy = bool( + not partial + and files_ambiguous == 0 + and len(summaries) == 1 + and classification == LayoutClassification.NORMAL_LIBRARY + and summaries[0].proposed_series_path_template is not None + and ( + effective_spec.mode == ImportLayoutMode.AUTO + or summaries[0].proposed_series_path_template == effective_spec.series_path_template + ) + ) + return LayoutAnalysisResult( + effective_spec=effective_spec, + classification=classification, + clusters=summaries, + directories_considered=directories_considered, + files_considered=files_considered, + files_fitting=files_fitting, + files_ambiguous=files_ambiguous, + files_outside_root=files_outside_root, + archive_probes=0, + can_keep_in_place=can_keep_in_place, + can_apply_future_policy=can_apply_future_policy, + partial=partial, + warnings=warnings, + ) + + def _analyze_path( + self, + relative_path: str, + *, + effective_spec: SourceLayoutSpec, + selected_matcher: CompiledSourceLayout | None, + ) -> _AnalyzedPath: + if selected_matcher is not None: + try: + matched = selected_matcher.match(relative_path) + except LayoutValueError: + matched = None + if matched is not None: + return _AnalyzedPath( + cluster_key=f"selected:{effective_spec.series_path_template}", + classification=LayoutClassification.NORMAL_LIBRARY, + confidence="high", + match=matched, + proposed_series_path_template=effective_spec.series_path_template, + proposed_issue_filename_template=effective_spec.issue_filename_template, + evidence=["selected_layout_match"], + warnings=[], + ) + if not effective_spec.fallback_to_auto: + return self._needs_review_path(relative_path, "selected_layout_no_match") + + return self._auto_analyze_path(relative_path) + + def _auto_analyze_path(self, relative_path: str) -> _AnalyzedPath: + parts = PurePosixPath(relative_path).parts + depth = len(parts) - 1 + if depth == 1: + matcher = self._series_matcher + key = "auto:series_folders" + confidence = "high" + template = "{Series}" + elif depth == 2: + possible_publisher = parts[0].strip().casefold() + possible_series = parts[1].strip().casefold() + if ( + possible_publisher in _GENERIC_OR_TYPE_CONTAINERS + or possible_series in _GENERIC_OR_TYPE_CONTAINERS + or possible_publisher.isdigit() + ): + return self._needs_review_path( + relative_path, + "generic_or_type_container_requires_review", + ) + matcher = self._publisher_matcher + key = "auto:publisher_series" + confidence = "high" + template = "{Publisher}/{Series}" + elif depth == 0: + metadata = self._metadata_extractor.from_release_title( + PurePosixPath(relative_path).name + ) + if metadata.series_name and metadata.issue_number is not None: + match = SourceLayoutMatch( + relative_path=relative_path, + series=metadata.series_name, + year=metadata.year, + issue_number=format_issue_number(metadata.issue_number), + ) + return _AnalyzedPath( + cluster_key="auto:loose_files", + classification=LayoutClassification.NORMAL_LIBRARY, + confidence="low", + match=match, + proposed_series_path_template=None, + proposed_issue_filename_template="{Series} {Issue}", + evidence=["loose_filename_identity"], + warnings=["loose_files_require_review"], + ) + return self._needs_review_path(relative_path, "loose_file_without_strong_identity") + else: + return self._needs_review_path(relative_path, "unrecognized_directory_depth") + + try: + matched = matcher.match(relative_path) + except LayoutValueError: + matched = None + if matched is None: + return self._needs_review_path(relative_path, "registered_layout_no_match") + return _AnalyzedPath( + cluster_key=key, + classification=LayoutClassification.NORMAL_LIBRARY, + confidence=confidence, + match=matched, + proposed_series_path_template=template, + proposed_issue_filename_template=None, + evidence=[key.replace(":", "_")], + warnings=[], + ) + + def _needs_review_path(self, relative_path: str, reason: str) -> _AnalyzedPath: + metadata = self._metadata_extractor.from_release_title(PurePosixPath(relative_path).name) + match = SourceLayoutMatch( + relative_path=relative_path, + series=metadata.series_name, + year=metadata.year, + issue_number=( + format_issue_number(metadata.issue_number) + if metadata.issue_number is not None + else None + ), + ) + return _AnalyzedPath( + cluster_key=f"needs_review:{reason}", + classification=LayoutClassification.NEEDS_REVIEW, + confidence="low", + match=match, + proposed_series_path_template=None, + proposed_issue_filename_template=None, + evidence=[], + warnings=[reason], + ) + + @staticmethod + def _accumulate( + clusters: dict[str, _ClusterAccumulator], + analyzed: _AnalyzedPath, + *, + relative_path: str, + max_examples: int, + ) -> None: + accumulator = clusters.setdefault( + analyzed.cluster_key, + _ClusterAccumulator( + classification=analyzed.classification, + confidence=analyzed.confidence, + proposed_series_path_template=analyzed.proposed_series_path_template, + proposed_issue_filename_template=analyzed.proposed_issue_filename_template, + ), + ) + accumulator.file_count += 1 + accumulator.directories.add(PurePosixPath(relative_path).parent.as_posix()) + if len(accumulator.examples) >= max_examples: + return + matched = analyzed.match + accumulator.examples.append( + LayoutExample( + relative_path=relative_path, + publisher=matched.publisher if matched is not None else None, + series=matched.series if matched is not None else None, + year=matched.year if matched is not None else None, + issue_number=matched.issue_number if matched is not None else None, + issue_title=matched.issue_title if matched is not None else None, + evidence=list(analyzed.evidence), + warnings=list(analyzed.warnings), + ) + ) + + @staticmethod + def _summaries(clusters: dict[str, _ClusterAccumulator]) -> list[LayoutClusterSummary]: + summaries: list[LayoutClusterSummary] = [] + for key in sorted(clusters): + cluster = clusters[key] + cluster_id = hashlib.sha256(key.encode()).hexdigest()[:12] + summaries.append( + LayoutClusterSummary( + cluster_id=cluster_id, + classification=cluster.classification, + file_count=cluster.file_count, + directory_count=len(cluster.directories), + confidence=cluster.confidence, + proposed_series_path_template=cluster.proposed_series_path_template, + proposed_issue_filename_template=cluster.proposed_issue_filename_template, + examples=list(cluster.examples), + ) + ) + return summaries + + @staticmethod + def _raise_if_cancelled(cancel_event: asyncio.Event | None) -> None: + if cancel_event is not None and cancel_event.is_set(): + raise asyncio.CancelledError + + @staticmethod + def _deadline_reached(deadline: float) -> bool: + return time.monotonic() >= deadline + + @staticmethod + def _empty_partial_result( + spec: SourceLayoutSpec, + warning: str, + ) -> LayoutAnalysisResult: + return LayoutAnalysisResult( + effective_spec=spec, + classification=LayoutClassification.NEEDS_REVIEW, + clusters=[], + directories_considered=0, + files_considered=0, + files_fitting=0, + files_ambiguous=0, + files_outside_root=0, + archive_probes=0, + can_keep_in_place=True, + can_apply_future_policy=False, + partial=True, + warnings=[warning], + ) + + +def _is_within_root(path: Path, root: Path) -> bool: + try: + resolved = path.resolve(strict=True) + except OSError: + return False + return not is_sensitive_path(resolved) and (resolved == root or resolved.is_relative_to(root)) + + +def _overall_classification( + clusters: list[LayoutClusterSummary], +) -> LayoutClassification: + if not clusters: + return LayoutClassification.NEEDS_REVIEW + if len(clusters) > 1: + return LayoutClassification.MIXED + return clusters[0].classification + + +def _append_once(values: list[str], value: str) -> None: + if value not in values: + values.append(value) diff --git a/src/pullbox/services/import_library_adoption.py b/src/pullbox/services/import_library_adoption.py new file mode 100644 index 00000000..74d9b07a --- /dev/null +++ b/src/pullbox/services/import_library_adoption.py @@ -0,0 +1,790 @@ +"""Build a clean managed library from a completed reference-only import.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +from sqlalchemy import delete, func, insert, select + +from pullbox.core.config_resolver import get_application_secret +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.library_policy import load_effective_library_ingest_policy +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportFileHandlingMode, + ImportJob, + ImportJobLog, + ImportJobStatus, + ImportSeriesStatus, +) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile, LibraryFileStorageMode, LibraryRoot +from pullbox.models.series import Series +from pullbox.schemas.import_job import ImportProgressEvent +from pullbox.services.import_completed_cleanup import ( + CompletedImportCleanupAction, + count_completed_import_cleanup_scope, +) +from pullbox.services.import_policy_snapshot import apply_ingest_policy_to_import_job +from pullbox.services.import_workflow_state import ( + ACTIVE_IMPORT_JOB_STATUSES, + emit_progress, + phase_progress, + raise_if_job_cancelled, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + + +_TOKEN_SALT: Final = "completed-import-clean-library-v1" +_TOKEN_MAX_AGE_SECONDS: Final = 15 * 60 +_TOKEN_VERSION: Final = 1 +_SERIES_BATCH_SIZE: Final = 500 +_FILE_BATCH_SIZE: Final = 2_000 +_PREPARATION_PROGRESS_END: Final = 5 + + +@dataclass(frozen=True, slots=True) +class CleanLibraryImportPreview: + """Exact source-preserving adoption scope shown before job creation.""" + + source_job_id: int + target_root_id: int + eligible_file_count: int + eligible_series_count: int + total_bytes: int + source_preserved: bool + preview_token: str + + +@dataclass(frozen=True, slots=True) +class CleanLibraryImportResult: + """New managed-copy import created from a completed reference import.""" + + source_job_id: int + job_id: int + eligible_file_count: int + eligible_series_count: int + + +@dataclass(frozen=True, slots=True) +class _AdoptionSnapshot: + file_count: int + series_count: int + total_bytes: int + digest: str + + +def _serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(get_application_secret(), salt=_TOKEN_SALT) + + +async def _load_source_job(session: AsyncSession, source_job_id: int) -> ImportJob: + job = await session.get(ImportJob, source_job_id, populate_existing=True) + if job is None: + raise NotFoundError("ImportJob", source_job_id) + if job.status is not ImportJobStatus.COMPLETED: + raise ValidationError("The source import must be complete before standardization.") + if job.archived_at is not None: + raise ValidationError("Restore the archived import before standardizing its library.") + return job + + +async def _require_mixed_folder_repairs_complete( + session: AsyncSession, + source_job_id: int, +) -> None: + repair_count, _file_count = await count_completed_import_cleanup_scope( + session, + source_job_id, + CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES, + ) + if repair_count: + raise ValidationError( + "Resolve mixed-folder files before building the clean Pullbox library." + ) + + +def _eligible_sources(source_job_id: int) -> tuple[ColumnElement[bool], ...]: + return ( + ImportedFile.import_job_id == source_job_id, + ImportedFile.status == ImportedFileStatus.IMPORTED, + ImportedFile.matched_issue_id == Issue.id, + ImportedFile.library_file_id == LibraryFile.id, + LibraryFile.issue_id == Issue.id, + LibraryFile.storage_mode == LibraryFileStorageMode.REFERENCED, + LibraryFile.file_path == ImportedFile.file_path, + ) + + +async def _build_snapshot(session: AsyncSession, source_job_id: int) -> _AdoptionSnapshot: + digest = sha256() + series_ids: set[int] = set() + file_count = 0 + total_bytes = 0 + stream = await session.stream( + select( + ImportedFile.id, + LibraryFile.id, + Issue.id, + Issue.series_id, + LibraryFile.file_path, + LibraryFile.file_size, + LibraryFile.updated_at, + ImportedFile.updated_at, + ) + .select_from(ImportedFile) + .join(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .join(Issue, Issue.id == LibraryFile.issue_id) + .where(*_eligible_sources(source_job_id)) + .order_by(ImportedFile.id) + .execution_options(yield_per=2_000) + ) + async for row in stream: + file_count += 1 + series_ids.add(int(row[3])) + total_bytes += int(row[5]) + digest.update( + ( + f"{int(row[0])}|{int(row[1])}|{int(row[2])}|{row[4]}|" + f"{row[6].isoformat()}|{row[7].isoformat()}\n" + ).encode() + ) + return _AdoptionSnapshot( + file_count=file_count, + series_count=len(series_ids), + total_bytes=total_bytes, + digest=digest.hexdigest(), + ) + + +def _paths_overlap(first: str, second: str) -> bool: + first_path = Path(first).resolve(strict=False) + second_path = Path(second).resolve(strict=False) + return ( + first_path == second_path + or first_path.is_relative_to(second_path) + or second_path.is_relative_to(first_path) + ) + + +async def _load_target_root( + session: AsyncSession, + target_root_id: int, + source_job_id: int, +) -> LibraryRoot: + root = await session.get(LibraryRoot, target_root_id) + if root is None: + raise NotFoundError("LibraryRoot", target_root_id) + if not root.enabled or not root.allow_managed_writes: + raise ValidationError("Choose an enabled library root that allows managed writes.") + source_root_paths = set( + ( + await session.scalars( + select(LibraryRoot.path) + .select_from(ImportedFile) + .join(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .join(Issue, Issue.id == LibraryFile.issue_id) + .join(LibraryRoot, LibraryRoot.id == LibraryFile.library_root_id) + .where(*_eligible_sources(source_job_id)) + .distinct() + ) + ).all() + ) + if any(_paths_overlap(root.path, source_path) for source_path in source_root_paths): + raise ValidationError( + "Choose a separate managed library root that does not overlap the Mylar source." + ) + return root + + +async def preview_clean_library_import( + session: AsyncSession, + source_job_id: int, + *, + target_root_id: int, + actor_id: int, +) -> CleanLibraryImportPreview: + """Preview referenced files that can become a clean managed library.""" + await _load_source_job(session, source_job_id) + await _require_mixed_folder_repairs_complete(session, source_job_id) + snapshot = await _build_snapshot(session, source_job_id) + if snapshot.file_count == 0: + raise ValidationError("This import has no referenced files available to standardize.") + await _load_target_root(session, target_root_id, source_job_id) + token = str( + _serializer().dumps( + { + "version": _TOKEN_VERSION, + "source_job_id": source_job_id, + "target_root_id": target_root_id, + "actor_id": actor_id, + "snapshot": { + "file_count": snapshot.file_count, + "series_count": snapshot.series_count, + "total_bytes": snapshot.total_bytes, + "digest": snapshot.digest, + }, + } + ) + ) + return CleanLibraryImportPreview( + source_job_id=source_job_id, + target_root_id=target_root_id, + eligible_file_count=snapshot.file_count, + eligible_series_count=snapshot.series_count, + total_bytes=snapshot.total_bytes, + source_preserved=True, + preview_token=token, + ) + + +def _snapshot_from_token( + token: str, + *, + source_job_id: int, + target_root_id: int, + actor_id: int, +) -> _AdoptionSnapshot: + try: + payload = _serializer().loads(token, max_age=_TOKEN_MAX_AGE_SECONDS) + except SignatureExpired as exc: + raise ValidationError("The clean-library preview expired. Preview it again.") from exc + except BadSignature as exc: + raise ValidationError("The clean-library preview is invalid. Preview it again.") from exc + if not isinstance(payload, Mapping): + raise ValidationError("The clean-library preview is invalid. Preview it again.") + if ( + payload.get("version") != _TOKEN_VERSION + or payload.get("source_job_id") != source_job_id + or payload.get("target_root_id") != target_root_id + or payload.get("actor_id") != actor_id + ): + raise ValidationError("The clean-library scope changed. Preview it again.") + raw_snapshot = payload.get("snapshot") + if not isinstance(raw_snapshot, Mapping): + raise ValidationError("The clean-library preview is invalid. Preview it again.") + try: + snapshot = _AdoptionSnapshot( + file_count=int(raw_snapshot["file_count"]), + series_count=int(raw_snapshot["series_count"]), + total_bytes=int(raw_snapshot["total_bytes"]), + digest=str(raw_snapshot["digest"]), + ) + except (KeyError, TypeError, ValueError) as exc: + raise ValidationError("The clean-library preview is invalid. Preview it again.") from exc + if ( + snapshot.file_count <= 0 + or snapshot.series_count <= 0 + or snapshot.total_bytes < 0 + or not snapshot.digest + ): + raise ValidationError("The clean-library preview is invalid. Preview it again.") + return snapshot + + +def _adoption_diagnostics( + *, + source_diagnostics: object, + source_job_id: int, + source_imported_file_id: int, + source_library_file_id: int, + source_path: str, + source_library_root_id: int, + source_signature: object, +) -> dict[str, object]: + diagnostics = dict(source_diagnostics) if isinstance(source_diagnostics, dict) else {} + diagnostics["library_adoption"] = { + "schema_version": 1, + "source_import_job_id": source_job_id, + "source_imported_file_id": source_imported_file_id, + "source_library_file_id": source_library_file_id, + "source_path": source_path, + "source_library_root_id": source_library_root_id, + "source_signature": (dict(source_signature) if isinstance(source_signature, dict) else {}), + "source_storage_mode": LibraryFileStorageMode.REFERENCED.value, + "source_preserved": True, + } + return diagnostics + + +async def _create_adoption_series( + session: AsyncSession, + *, + job: ImportJob, + source_job_id: int, + batch_callback: Callable[[int], Awaitable[None]] | None = None, +) -> dict[int, int]: + imported_series_by_id: dict[int, int] = {} + last_series_id = 0 + while True: + rows = ( + await session.execute( + select( + Series.id, + Series.title, + Series.year_start, + Series.comicvine_id, + func.count(ImportedFile.id), + func.min(LibraryFile.file_path), + ) + .select_from(ImportedFile) + .join(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .join(Issue, Issue.id == LibraryFile.issue_id) + .join(Series, Series.id == Issue.series_id) + .where(*_eligible_sources(source_job_id)) + .where(Series.id > last_series_id) + .group_by(Series.id, Series.title, Series.year_start, Series.comicvine_id) + .order_by(Series.id) + .limit(_SERIES_BATCH_SIZE) + ) + ).all() + if not rows: + break + pending: list[ImportedSeries] = [] + for series_id, title, year_start, comicvine_id, file_count, sample_path in rows: + imported_series = ImportedSeries( + import_job_id=job.id, + raw_series_name=title, + raw_year=year_start, + file_count=int(file_count), + files_total=int(file_count), + files_matched=int(file_count), + sample_paths=[str(sample_path)] if sample_path else [], + source_folder=str(Path(str(sample_path)).parent) if sample_path else None, + has_files=True, + cv_id=comicvine_id, + cv_title=title, + cv_year=year_start, + cv_match_score=1.0, + cv_match_method="clean_library_adoption", + status=ImportSeriesStatus.DUPLICATE, + selected_for_import=True, + series_id=series_id, + diagnostics={ + "kind": "clean_library_adoption", + "source_import_job_id": source_job_id, + "source_preserved": True, + }, + ) + session.add(imported_series) + pending.append(imported_series) + await session.flush() + imported_series_by_id.update( + {int(item.series_id): int(item.id) for item in pending if item.series_id is not None} + ) + last_series_id = int(rows[-1][0]) + if batch_callback is not None: + await batch_callback(len(imported_series_by_id)) + return imported_series_by_id + + +async def _create_adoption_files( + session: AsyncSession, + *, + job_id: int, + source_job_id: int, + imported_series_by_id: dict[int, int], + batch_callback: Callable[[int], Awaitable[None]] | None = None, +) -> None: + completed = 0 + last_file_id = 0 + while True: + rows = ( + ( + await session.execute( + select( + ImportedFile.id.label("source_imported_file_id"), + ImportedFile.diagnostics.label("source_diagnostics"), + LibraryFile.id.label("source_library_file_id"), + LibraryFile.file_path, + LibraryFile.file_name, + LibraryFile.file_size, + LibraryFile.file_format, + LibraryFile.has_comicinfo, + LibraryFile.source_signature, + LibraryFile.library_root_id.label("source_library_root_id"), + Issue.id.label("issue_id"), + Issue.comicvine_id.label("issue_comicvine_id"), + Issue.issue_number, + Issue.issue_number_text, + Series.id.label("series_id"), + Series.title.label("series_title"), + Series.year_start, + ) + .select_from(ImportedFile) + .join(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .join(Issue, Issue.id == LibraryFile.issue_id) + .join(Series, Series.id == Issue.series_id) + .where(*_eligible_sources(source_job_id)) + .where(ImportedFile.id > last_file_id) + .order_by(ImportedFile.id) + .limit(_FILE_BATCH_SIZE) + ) + ) + .mappings() + .all() + ) + if not rows: + break + pending: list[dict[str, object]] = [] + for row in rows: + series_id = int(row["series_id"]) + issue_number = float(row["issue_number"]) + issue_number_text = row["issue_number_text"] + pending.append( + { + "import_job_id": job_id, + "import_series_id": imported_series_by_id[series_id], + "file_path": str(row["file_path"]), + "file_name": str(row["file_name"]), + "file_size": int(row["file_size"]), + "file_format": row["file_format"].value, + "parsed_series": str(row["series_title"]), + "parsed_issue_number": issue_number, + "parsed_year": row["year_start"], + "has_comicinfo": bool(row["has_comicinfo"]), + "comicvine_issue_id": row["issue_comicvine_id"], + "issue_number_raw": ( + str(issue_number_text) if issue_number_text else f"{issue_number:g}" + ), + "status": ImportedFileStatus.CONFIRMED, + "matched_issue_id": int(row["issue_id"]), + "matched_issue_cv_id": row["issue_comicvine_id"], + "match_confidence": "high", + "match_method": "clean_library_adoption", + "include_in_import": True, + "source_signature": dict(row["source_signature"] or {}), + "diagnostics": _adoption_diagnostics( + source_diagnostics=row["source_diagnostics"], + source_job_id=source_job_id, + source_imported_file_id=int(row["source_imported_file_id"]), + source_library_file_id=int(row["source_library_file_id"]), + source_path=str(row["file_path"]), + source_library_root_id=int(row["source_library_root_id"]), + source_signature=row["source_signature"], + ), + } + ) + await session.execute(insert(ImportedFile), pending) + completed += len(pending) + last_file_id = int(rows[-1]["source_imported_file_id"]) + if batch_callback is not None: + await batch_callback(completed) + + +def _snapshot_payload(snapshot: _AdoptionSnapshot) -> dict[str, object]: + return { + "file_count": snapshot.file_count, + "series_count": snapshot.series_count, + "total_bytes": snapshot.total_bytes, + "digest": snapshot.digest, + } + + +def _snapshot_from_job(job: ImportJob) -> _AdoptionSnapshot: + raw = dict(job.progress_snapshot or {}).get("clean_library_source_snapshot") + if not isinstance(raw, Mapping): + raise ValidationError("The clean-library preparation scope is unavailable.") + try: + return _AdoptionSnapshot( + file_count=int(raw["file_count"]), + series_count=int(raw["series_count"]), + total_bytes=int(raw["total_bytes"]), + digest=str(raw["digest"]), + ) + except (KeyError, TypeError, ValueError) as exc: + raise ValidationError("The clean-library preparation scope is invalid.") from exc + + +async def _find_active_clean_library_import( + session: AsyncSession, + *, + source_job_id: int, + target_root_id: int, +) -> ImportJob | None: + jobs = list( + ( + await session.scalars( + select(ImportJob) + .where( + ImportJob.status.in_( + { + ImportJobStatus.IMPORTING, + ImportJobStatus.PAUSING, + ImportJobStatus.PAUSED, + ImportJobStatus.STALLED, + ImportJobStatus.CANCELLING, + ImportJobStatus.ROLLING_BACK, + } + ), + ImportJob.target_library_root_id == target_root_id, + ) + .order_by(ImportJob.id.desc()) + ) + ).all() + ) + for job in jobs: + progress = dict(job.progress_snapshot or {}) + if ( + progress.get("clean_library_adoption") is True + and int(progress.get("source_import_job_id") or 0) == source_job_id + ): + return job + return None + + +async def prepare_clean_library_import( + session: AsyncSession, + job_id: int, + *, + progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None = None, +) -> bool: + """Materialize a clean-library plan inside the durable import worker.""" + job = await session.get(ImportJob, job_id, populate_existing=True) + if job is None: + raise NotFoundError("ImportJob", job_id) + progress = dict(job.progress_snapshot or {}) + if progress.get("clean_library_adoption") is not True: + return False + if progress.get("clean_library_adoption_prepared") is True: + return False + + source_job_id = int(progress.get("source_import_job_id") or 0) + if source_job_id <= 0 or job.target_library_root_id is None: + raise ValidationError("The clean-library preparation context is incomplete.") + expected_snapshot = _snapshot_from_job(job) + await _load_source_job(session, source_job_id) + await _require_mixed_folder_repairs_complete(session, source_job_id) + await _load_target_root(session, int(job.target_library_root_id), source_job_id) + current_snapshot = await _build_snapshot(session, source_job_id) + if current_snapshot != expected_snapshot: + raise ValidationError( + "The clean-library source changed after preview. Open the organizer and try again." + ) + + await session.execute(delete(ImportedFile).where(ImportedFile.import_job_id == job_id)) + await session.execute(delete(ImportedSeries).where(ImportedSeries.import_job_id == job_id)) + + total_units = expected_snapshot.series_count + expected_snapshot.file_count + + async def report_progress( + completed_units: int, + *, + message: str, + item_label: str, + ) -> None: + await raise_if_job_cancelled(session, job_id) + job.progress_snapshot = { + **dict(job.progress_snapshot or {}), + "clean_library_adoption": True, + "clean_library_adoption_prepared": False, + "source_import_job_id": source_job_id, + "clean_library_source_snapshot": _snapshot_payload(expected_snapshot), + } + await emit_progress( + session, + job, + ImportProgressEvent( + job_id=job_id, + status=ImportJobStatus.IMPORTING, + mode="import", + phase="clean_library_preparing", + progress=phase_progress( + 0, + _PREPARATION_PROGRESS_END, + completed_units, + total_units, + ), + message=message, + current_file_name=item_label, + current_file_stage="clean_library_preparing", + current_file_progress_current=completed_units, + current_file_progress_total=total_units, + current_file_progress_pct=round((completed_units / total_units) * 100), + current_file_progress_unit="items", + ), + progress_callback, + ) + + await report_progress( + 0, + message="Preparing the clean-library work plan...", + item_label="Preparing library records", + ) + imported_series_by_id = await _create_adoption_series( + session, + job=job, + source_job_id=source_job_id, + batch_callback=lambda completed: report_progress( + completed, + message=(f"Prepared {completed:,} of {expected_snapshot.series_count:,} series."), + item_label="Preparing series", + ), + ) + await _create_adoption_files( + session, + job_id=job_id, + source_job_id=source_job_id, + imported_series_by_id=imported_series_by_id, + batch_callback=lambda completed: report_progress( + expected_snapshot.series_count + completed, + message=(f"Prepared {completed:,} of {expected_snapshot.file_count:,} files."), + item_label="Preparing files", + ), + ) + + refreshed_job = await session.get(ImportJob, job_id, populate_existing=True) + if refreshed_job is None: + raise NotFoundError("ImportJob", job_id) + refreshed_job.series_found = expected_snapshot.series_count + refreshed_job.series_duplicate = expected_snapshot.series_count + refreshed_job.total_files_found = expected_snapshot.file_count + refreshed_job.total_files_matched = expected_snapshot.file_count + refreshed_job.progress_snapshot = { + **dict(refreshed_job.progress_snapshot or {}), + "clean_library_adoption": True, + "clean_library_adoption_prepared": True, + "source_import_job_id": source_job_id, + "clean_library_source_snapshot": _snapshot_payload(expected_snapshot), + } + session.add( + ImportJobLog( + import_job_id=refreshed_job.id, + level="INFO", + event="clean_library_adoption_prepared", + message=( + f"Prepared {expected_snapshot.file_count} referenced files for a clean managed " + "library." + ), + data={ + "source_import_job_id": source_job_id, + "target_library_root_id": refreshed_job.target_library_root_id, + "eligible_file_count": expected_snapshot.file_count, + "eligible_series_count": expected_snapshot.series_count, + "total_bytes": expected_snapshot.total_bytes, + "source_preserved": True, + }, + ) + ) + await report_progress( + total_units, + message="Clean-library plan ready. Starting file processing...", + item_label="Library plan ready", + ) + refreshed_job.progress_snapshot = { + **dict(refreshed_job.progress_snapshot or {}), + "clean_library_adoption_prepared": True, + } + await session.commit() + return True + + +async def create_clean_library_import( + session: AsyncSession, + source_job_id: int, + *, + target_root_id: int, + actor_id: int, + preview_token: str, +) -> CleanLibraryImportResult: + """Create a managed-copy import that adopts exact referenced library files.""" + source_job = await _load_source_job(session, source_job_id) + await _require_mixed_folder_repairs_complete(session, source_job_id) + target_root = await _load_target_root(session, target_root_id, source_job_id) + snapshot = _snapshot_from_token( + preview_token, + source_job_id=source_job_id, + target_root_id=target_root_id, + actor_id=actor_id, + ) + + active_job = await _find_active_clean_library_import( + session, + source_job_id=source_job_id, + target_root_id=target_root_id, + ) + if active_job is not None: + return CleanLibraryImportResult( + source_job_id=source_job_id, + job_id=int(active_job.id), + eligible_file_count=snapshot.file_count, + eligible_series_count=snapshot.series_count, + ) + + active_job_id = await session.scalar( + select(ImportJob.id) + .where(ImportJob.status.in_(ACTIVE_IMPORT_JOB_STATUSES)) + .order_by(ImportJob.created_at.desc()) + .limit(1) + ) + if active_job_id is not None: + raise ValidationError( + "Only one import can be active at a time. " + "Finish, discard, or roll back the current import first." + ) + + policy = await load_effective_library_ingest_policy(session, target_root) + job = ImportJob( + source_path=source_job.source_path, + selected_file_paths=[], + source_type=source_job.source_type, + status=ImportJobStatus.IMPORTING, + target_library_root_id=target_root.id, + monitored=False, + search_on_add=False, + file_handling_mode=ImportFileHandlingMode.MANAGED_COPY, + source_layout_snapshot=dict(source_job.source_layout_snapshot or {}), + mylar3_path_map=dict(source_job.mylar3_path_map or {}), + mylar3_path_map_confirmed=source_job.mylar3_path_map_confirmed, + progress_snapshot={ + "mode": "import", + "phase": "queued", + "progress": 0, + "message": "Preparing a clean Pullbox-managed library.", + "source_import_job_id": source_job.id, + "clean_library_adoption": True, + "clean_library_adoption_prepared": False, + "clean_library_source_snapshot": _snapshot_payload(snapshot), + }, + ) + apply_ingest_policy_to_import_job(job, policy) + session.add(job) + await session.flush() + + job.series_found = snapshot.series_count + job.series_duplicate = snapshot.series_count + job.total_files_found = snapshot.file_count + job.total_files_matched = snapshot.file_count + session.add( + ImportJobLog( + import_job_id=job.id, + level="INFO", + event="clean_library_adoption_queued", + message=(f"Queued {snapshot.file_count} referenced files for a clean managed library."), + data={ + "source_import_job_id": source_job.id, + "target_library_root_id": target_root.id, + "eligible_file_count": snapshot.file_count, + "eligible_series_count": snapshot.series_count, + "total_bytes": snapshot.total_bytes, + "source_preserved": True, + }, + ) + ) + await session.flush() + return CleanLibraryImportResult( + source_job_id=source_job.id, + job_id=job.id, + eligible_file_count=snapshot.file_count, + eligible_series_count=snapshot.series_count, + ) diff --git a/src/pullbox/services/import_logical_groups.py b/src/pullbox/services/import_logical_groups.py index 5842de22..f1cfff15 100644 --- a/src/pullbox/services/import_logical_groups.py +++ b/src/pullbox/services/import_logical_groups.py @@ -208,6 +208,7 @@ async def _mark_library_owned_ambiguous_candidate_conflict( imp_file.diagnostics = { **dict(imp_file.diagnostics or {}), "kind": "series_conflict_file", + "reason": "library_owned_ambiguous_candidate", "target_state": "series_match_conflict", "rejection_reason": ( "A same-year single-issue library item has the same crossover title tokens." diff --git a/src/pullbox/services/import_managed_copy_preflight.py b/src/pullbox/services/import_managed_copy_preflight.py new file mode 100644 index 00000000..d7de457f --- /dev/null +++ b/src/pullbox/services/import_managed_copy_preflight.py @@ -0,0 +1,733 @@ +"""Managed-copy destination capability and capacity preflight.""" + +from __future__ import annotations + +import asyncio +import enum +import shutil +import tempfile +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Literal + +from sqlalchemy import func as sa_func +from sqlalchemy import or_ as sa_or +from sqlalchemy import select as sa_select +from sqlalchemy import update as sa_update + +from pullbox.config import get_settings +from pullbox.core.exceptions import ConfigurationError, ValidationError +from pullbox.core.library_root_resolution import resolve_library_root +from pullbox.models.import_job import ( + ImportControlRequest, + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportFileHandlingMode, + ImportJob, + ImportJobStatus, + ImportSeriesStatus, +) +from pullbox.models.library import LibraryFile, LibraryRoot +from pullbox.models.story_arc import ImportedStoryArcStatus, StoryArcResolutionState +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.services.library_root_management import validate_managed_library_root + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +_ONE_GIB = 1024**3 +_CAPACITY_SNAPSHOT_KEY = "managed_copy_capacity" +_STORY_ARC_PAGE_SIZE = 250 +_STORY_ARC_ENTRY_PAGE_SIZE = 1_000 +_CONVERSION_WORKSPACE_MULTIPLIER = 2 +ManagedCopyPreflightStage = Literal["confirmation", "execution"] + + +class ManagedCopyPreflightFailure(enum.StrEnum): + """Stable failure reasons for a managed-copy preflight block.""" + + TARGET_MISSING = "target_missing" + TARGET_DISABLED = "target_disabled" + TARGET_REFERENCE_ONLY = "target_reference_only" + TARGET_UNAVAILABLE = "target_unavailable" + CAPACITY_UNKNOWN = "capacity_unknown" + CAPACITY_INSUFFICIENT = "capacity_insufficient" + + +@dataclass(frozen=True, slots=True) +class ManagedCopyTargetCapacitySnapshot: + """One path-free managed-root capacity result.""" + + target_library_root_id: int + selected_source_bytes: int + reserve_bytes: int + required_bytes: int + free_bytes: int | None + status: str + + def as_dict(self) -> dict[str, int | str | None]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class ConversionWorkspaceCapacitySnapshot: + """Path-free estimate and live capacity for the system temp filesystem.""" + + selected_source_bytes: int + active_worker_count: int + estimated_workspace_bytes: int + reserve_bytes: int + required_bytes: int + free_bytes: int | None + status: str + + def as_dict(self) -> dict[str, int | str | None]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class ManagedCopyCapacitySnapshot: + """Path-free capacity evidence persisted with an import job. + + Schema version 1 remains byte-for-byte compatible for the common single + root/no-conversion case. Version 2 adds bounded per-root and temporary + conversion-workspace evidence while retaining every v1 top-level field. + """ + + schema_version: int + stage: ManagedCopyPreflightStage + target_library_root_id: int | None + selected_source_bytes: int + reserve_bytes: int + required_bytes: int + free_bytes: int | None + status: str + target_capacities: tuple[ManagedCopyTargetCapacitySnapshot, ...] = () + conversion_workspace: ConversionWorkspaceCapacitySnapshot | None = None + + def as_dict(self) -> dict[str, object]: + """Return the JSON-safe durable representation.""" + result: dict[str, object] = { + "schema_version": self.schema_version, + "stage": self.stage, + "target_library_root_id": self.target_library_root_id, + "selected_source_bytes": self.selected_source_bytes, + "reserve_bytes": self.reserve_bytes, + "required_bytes": self.required_bytes, + "free_bytes": self.free_bytes, + "status": self.status, + } + if self.schema_version >= 2: + result["target_capacities"] = [target.as_dict() for target in self.target_capacities] + result["conversion_workspace"] = ( + self.conversion_workspace.as_dict() + if self.conversion_workspace is not None + else None + ) + return result + + +class ManagedCopyPreflightError(ValidationError): + """Managed-copy validation failed before any library mutation.""" + + def __init__( + self, + reason: ManagedCopyPreflightFailure, + message: str, + *, + snapshot: ManagedCopyCapacitySnapshot | None = None, + ) -> None: + self.reason = reason + self.snapshot = snapshot + details: dict[str, object] = {"reason": reason.value} + if snapshot is not None: + details["capacity"] = snapshot.as_dict() + super().__init__(message, details=details) + + +def managed_copy_capacity_reserve(selected_source_bytes: int) -> int: + """Return the required fixed-or-proportional free-space reserve.""" + if selected_source_bytes < 0: + raise ValueError("Selected source bytes cannot be negative") + ten_percent = (selected_source_bytes + 9) // 10 + return max(_ONE_GIB, ten_percent) + + +async def selected_managed_copy_source_bytes( + session: AsyncSession, + job_id: int, +) -> int: + """Sum files that remain selected for managed placement.""" + selected_new_series = ImportedSeries.status.in_( + [ImportSeriesStatus.CONFIRMED, ImportSeriesStatus.IMPORTING] + ) + selected_duplicate_series = ( + ImportedSeries.status == ImportSeriesStatus.DUPLICATE + ) & ImportedFile.include_in_import.is_(True) + from pullbox.services.import_workflow_state import deferred_recovery_scope + + job = await session.get(ImportJob, job_id) + scope = deferred_recovery_scope(job) if job is not None else None + total = await session.scalar( + sa_select(sa_func.coalesce(sa_func.sum(ImportedFile.file_size), 0)) + .join(ImportedSeries, ImportedFile.import_series_id == ImportedSeries.id) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status.in_([ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED]), + sa_or(selected_new_series, selected_duplicate_series), + *([ImportedSeries.id.in_(scope)] if scope is not None else []), + ) + ) + return max(int(total or 0), 0) + + +def _selected_import_file( + imp_file: ImportedFile | None, + imported_series: ImportedSeries | None, + *, + job_id: int, + matched_issue_id: int | None, +) -> bool: + if ( + imp_file is None + or imported_series is None + or imp_file.import_job_id != job_id + or imported_series.import_job_id != job_id + or imp_file.status not in {ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED} + or imp_file.matched_issue_id is None + or ( + matched_issue_id is not None and int(imp_file.matched_issue_id) != int(matched_issue_id) + ) + ): + return False + return bool( + imported_series.status in {ImportSeriesStatus.CONFIRMED, ImportSeriesStatus.IMPORTING} + or ( + imported_series.status is ImportSeriesStatus.DUPLICATE + and imp_file.include_in_import is True + ) + ) + + +def _story_arc_copy_target_id(snapshot: object) -> int | None: + if not isinstance(snapshot, dict) or snapshot.get("activation") != "confirmed": + return None + placement = snapshot.get("placement_policy") + if not isinstance(placement, dict) or placement.get("mode") != "copy": + return None + target_id = placement.get("target_library_root_id") + if isinstance(target_id, bool) or not isinstance(target_id, int) or target_id < 1: + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.TARGET_MISSING, + "A confirmed Story Arc copy policy no longer has a valid managed root.", + ) + return target_id + + +async def selected_story_arc_copy_source_bytes_by_root( + session: AsyncSession, + job_id: int, +) -> dict[int, int]: + """Return confirmed Story Arc COPY bytes grouped by actual target root. + + Use the same minimum-id canonical LibraryFile chosen by materialization + when one already exists. Otherwise, a selected import file is the best + pre-execution size evidence for the job-owned artifact that will be + registered before Story Arc materialization. + Each arc entry is charged independently because one issue may intentionally + produce a separate copy in more than one Story Arc destination. + """ + totals: dict[int, int] = {} + after_arc_id = 0 + while True: + arc_rows = ( + await session.execute( + sa_select( + ImportedStoryArc.id, + ImportedStoryArc.proposed_policy_snapshot, + ) + .where( + ImportedStoryArc.import_job_id == job_id, + ImportedStoryArc.status == ImportedStoryArcStatus.CONFIRMED, + ImportedStoryArc.selected_for_import.is_(True), + ImportedStoryArc.id > after_arc_id, + ) + .order_by(ImportedStoryArc.id.asc()) + .limit(_STORY_ARC_PAGE_SIZE) + ) + ).all() + if not arc_rows: + break + copy_root_by_arc_id = { + int(row.id): target_id + for row in arc_rows + if (target_id := _story_arc_copy_target_id(row.proposed_policy_snapshot)) is not None + } + if copy_root_by_arc_id: + await _accumulate_story_arc_copy_entry_page_bytes( + session, + job_id=job_id, + copy_root_by_arc_id=copy_root_by_arc_id, + totals=totals, + ) + after_arc_id = int(arc_rows[-1].id) + return totals + + +async def _accumulate_story_arc_copy_entry_page_bytes( + session: AsyncSession, + *, + job_id: int, + copy_root_by_arc_id: dict[int, int], + totals: dict[int, int], +) -> None: + after_entry_id = 0 + while True: + rows = ( + await session.execute( + sa_select( + ImportedStoryArcEntry, + ImportedFile, + ImportedSeries, + ) + .outerjoin(ImportedFile, ImportedFile.id == ImportedStoryArcEntry.import_file_id) + .outerjoin(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .where( + ImportedStoryArcEntry.imported_story_arc_id.in_(copy_root_by_arc_id), + ImportedStoryArcEntry.selected_for_import.is_(True), + ImportedStoryArcEntry.resolution_state == StoryArcResolutionState.RESOLVED, + ImportedStoryArcEntry.id > after_entry_id, + ) + .order_by(ImportedStoryArcEntry.id.asc()) + .limit(_STORY_ARC_ENTRY_PAGE_SIZE) + ) + ).all() + if not rows: + break + issue_ids = { + int(row[0].matched_issue_id) for row in rows if row[0].matched_issue_id is not None + } + canonical_sizes = await _canonical_library_file_sizes(session, issue_ids) + for entry, imp_file, imported_series in rows: + issue_id = int(entry.matched_issue_id) if entry.matched_issue_id is not None else None + if issue_id is not None and issue_id in canonical_sizes: + # Materialization selects the minimum-id canonical file. Any + # file registered later by this job receives a larger id, so + # current canonical evidence must win when it already exists. + size = canonical_sizes[issue_id] + elif _selected_import_file( + imp_file, + imported_series, + job_id=job_id, + matched_issue_id=issue_id, + ): + # No canonical file exists yet. The selected job-owned file is + # the source that normal import will register before Story Arc + # materialization, so its staged size is the fail-closed fallback. + size = max(int(imp_file.file_size or 0), 0) + else: + size = 0 + root_id = copy_root_by_arc_id[int(entry.imported_story_arc_id)] + totals[root_id] = totals.get(root_id, 0) + size + after_entry_id = int(rows[-1][0].id) + + +async def _canonical_library_file_sizes( + session: AsyncSession, + issue_ids: set[int], +) -> dict[int, int]: + if not issue_ids: + return {} + canonical = ( + sa_select( + LibraryFile.issue_id.label("issue_id"), + sa_func.min(LibraryFile.id).label("library_file_id"), + ) + .where(LibraryFile.issue_id.in_(issue_ids)) + .group_by(LibraryFile.issue_id) + .subquery() + ) + rows = ( + await session.execute( + sa_select(LibraryFile.issue_id, LibraryFile.file_size).join( + canonical, + LibraryFile.id == canonical.c.library_file_id, + ) + ) + ).all() + return { + int(issue_id): max(int(file_size or 0), 0) + for issue_id, file_size in rows + if issue_id is not None + } + + +async def estimate_conversion_workspace_source_bytes( + session: AsyncSession, + job: ImportJob, + *, + worker_count: int | None = None, +) -> tuple[int, int]: + """Return largest concurrently staged non-CBZ source bytes and worker count.""" + if not ( + job.move_to_library + and (job.convert_to_preferred_format or job.update_embedded_comicinfo_from_match) + ): + return 0, 0 + configured_workers = ( + get_settings().import_file_worker_count if worker_count is None else worker_count + ) + active_limit = max(int(configured_workers), 1) + selected_new_series = ImportedSeries.status.in_( + [ImportSeriesStatus.CONFIRMED, ImportSeriesStatus.IMPORTING] + ) + selected_duplicate_series = ( + ImportedSeries.status == ImportSeriesStatus.DUPLICATE + ) & ImportedFile.include_in_import.is_(True) + from pullbox.services.import_workflow_state import deferred_recovery_scope + + scope = deferred_recovery_scope(job) + sizes = list( + ( + await session.scalars( + sa_select(ImportedFile.file_size) + .join(ImportedSeries, ImportedFile.import_series_id == ImportedSeries.id) + .where( + ImportedFile.import_job_id == job.id, + ImportedFile.status.in_( + [ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED] + ), + sa_or(selected_new_series, selected_duplicate_series), + *([ImportedSeries.id.in_(scope)] if scope is not None else []), + sa_func.lower(ImportedFile.file_format) != "cbz", + ) + .order_by(ImportedFile.file_size.desc(), ImportedFile.id.asc()) + .limit(active_limit) + ) + ).all() + ) + return sum(max(int(size or 0), 0) for size in sizes), len(sizes) + + +async def validate_managed_copy_preflight( + session: AsyncSession, + job: ImportJob, + *, + stage: ManagedCopyPreflightStage, +) -> ManagedCopyCapacitySnapshot | None: + """Revalidate every actual copy destination plus conversion workspace.""" + selected_bytes_by_root: dict[int, int] = {} + if job.file_handling_mode != ImportFileHandlingMode.IN_PLACE: + selected_source_bytes = await selected_managed_copy_source_bytes(session, job.id) + # Preserve the original managed-copy contract even for sparse/legacy + # review rows with zero recorded bytes: validate the configured root and + # retain the fixed reserve evidence in the v1 snapshot. + job_root = await _resolve_job_managed_root(session, job) + selected_bytes_by_root[job_root.id] = selected_source_bytes + + from pullbox.services.import_workflow_state import deferred_recovery_scope + + story_arc_bytes = ( + await selected_story_arc_copy_source_bytes_by_root(session, job.id) + if deferred_recovery_scope(job) is None + else {} + ) + for root_id, selected_source_bytes in story_arc_bytes.items(): + selected_bytes_by_root[root_id] = ( + selected_bytes_by_root.get(root_id, 0) + selected_source_bytes + ) + + conversion_source_bytes, conversion_workers = await estimate_conversion_workspace_source_bytes( + session, job + ) + if not selected_bytes_by_root and conversion_source_bytes == 0: + return None + + target_snapshots: list[ManagedCopyTargetCapacitySnapshot] = [] + for root_id in sorted(selected_bytes_by_root): + root = await _load_managed_root(session, root_id) + selected_source_bytes = selected_bytes_by_root[root_id] + reserve_bytes = managed_copy_capacity_reserve(selected_source_bytes) + required_bytes = selected_source_bytes + reserve_bytes + capabilities = await _validate_live_managed_root(root) + free_bytes = _free_bytes_from_capabilities(capabilities) + status = ( + "unknown" + if free_bytes is None + else "insufficient" + if free_bytes < required_bytes + else "ready" + ) + target = ManagedCopyTargetCapacitySnapshot( + target_library_root_id=root.id, + selected_source_bytes=selected_source_bytes, + reserve_bytes=reserve_bytes, + required_bytes=required_bytes, + free_bytes=free_bytes, + status=status, + ) + target_snapshots.append(target) + if status != "ready": + snapshot = _target_result_snapshot( + stage=stage, + target=target, + targets=target_snapshots, + force_v2=len(selected_bytes_by_root) > 1, + ) + _record_capacity_snapshot(job, snapshot) + if status == "unknown": + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.CAPACITY_UNKNOWN, + "Available space for a selected managed library root could not be determined.", + snapshot=snapshot, + ) + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.CAPACITY_INSUFFICIENT, + "A selected managed library root does not have enough free space for this import.", + snapshot=snapshot, + ) + + workspace = await _validate_conversion_workspace( + selected_source_bytes=conversion_source_bytes, + active_worker_count=conversion_workers, + ) + if workspace is not None and workspace.status != "ready": + snapshot = _workspace_result_snapshot( + stage=stage, + targets=target_snapshots, + workspace=workspace, + ) + _record_capacity_snapshot(job, snapshot) + if workspace.status == "unknown": + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.CAPACITY_UNKNOWN, + "Available space for the temporary conversion workspace could not be determined.", + snapshot=snapshot, + ) + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.CAPACITY_INSUFFICIENT, + "The temporary conversion workspace does not have enough free space for this import.", + snapshot=snapshot, + ) + + primary_root_id = ( + job.target_library_root_id + if job.target_library_root_id in selected_bytes_by_root + else min(selected_bytes_by_root) + ) + primary = next( + target for target in target_snapshots if target.target_library_root_id == primary_root_id + ) + snapshot = _target_result_snapshot( + stage=stage, + target=primary, + targets=target_snapshots, + workspace=workspace, + force_v2=len(target_snapshots) > 1 or workspace is not None, + ) + _record_capacity_snapshot(job, snapshot) + return snapshot + + +async def reopen_review_after_managed_copy_preflight_failure( + session: AsyncSession, + job: ImportJob, + error: ManagedCopyPreflightError, +) -> None: + """Return an unstarted execution to a retryable review state.""" + await session.execute( + sa_update(ImportedFile) + .where( + ImportedFile.import_job_id == job.id, + ImportedFile.status == ImportedFileStatus.CONFIRMED, + ) + .values(status=ImportedFileStatus.MATCHED) + ) + await session.execute( + sa_update(ImportedSeries) + .where( + ImportedSeries.import_job_id == job.id, + ImportedSeries.status.in_([ImportSeriesStatus.CONFIRMED, ImportSeriesStatus.IMPORTING]), + ) + .values(status=ImportSeriesStatus.MATCHED, selected_for_import=True) + ) + await session.execute( + sa_update(ImportedStoryArc) + .where( + ImportedStoryArc.import_job_id == job.id, + ImportedStoryArc.status == ImportedStoryArcStatus.CONFIRMED, + ) + .values(status=ImportedStoryArcStatus.READY) + ) + job.status = ImportJobStatus.REVIEW + job.control_request = ImportControlRequest.NONE + job.import_started_at = None + job.error_message = error.message + snapshot = dict(job.progress_snapshot or {}) + snapshot.update( + { + "status": ImportJobStatus.REVIEW.value, + "mode": "scan", + "phase": "review", + "progress": 100, + "message": error.message, + } + ) + if error.snapshot is not None: + snapshot[_CAPACITY_SNAPSHOT_KEY] = error.snapshot.as_dict() + job.progress_snapshot = snapshot + await session.flush() + + +async def _resolve_job_managed_root( + session: AsyncSession, + job: ImportJob, +) -> LibraryRoot: + if job.target_library_root_id is not None: + root = await session.get(LibraryRoot, job.target_library_root_id) + if root is None: + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.TARGET_MISSING, + "The selected managed library root does not exist.", + ) + else: + try: + root = await resolve_library_root(session, Path(job.source_path), None) + except ConfigurationError as exc: + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.TARGET_MISSING, + exc.message, + ) from exc + job.target_library_root_id = root.id + + return _require_managed_root_roles(root) + + +async def _load_managed_root(session: AsyncSession, root_id: int) -> LibraryRoot: + root = await session.get(LibraryRoot, root_id) + if root is None: + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.TARGET_MISSING, + "A selected managed library root does not exist.", + ) + return _require_managed_root_roles(root) + + +def _require_managed_root_roles(root: LibraryRoot) -> LibraryRoot: + if not root.enabled: + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.TARGET_DISABLED, + "A selected managed library root is disabled.", + ) + if not root.allow_managed_writes: + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.TARGET_REFERENCE_ONLY, + "A selected library root does not allow managed writes.", + ) + return root + + +async def _validate_live_managed_root(root: LibraryRoot) -> dict[str, object]: + try: + return await validate_managed_library_root(root) + except ValidationError as exc: + raise ManagedCopyPreflightError( + ManagedCopyPreflightFailure.TARGET_UNAVAILABLE, + exc.message, + ) from exc + + +def _free_bytes_from_capabilities(capabilities: dict[str, object]) -> int | None: + free_value = capabilities.get("free_bytes") + return free_value if isinstance(free_value, int) and not isinstance(free_value, bool) else None + + +async def _validate_conversion_workspace( + *, + selected_source_bytes: int, + active_worker_count: int, +) -> ConversionWorkspaceCapacitySnapshot | None: + if selected_source_bytes <= 0 or active_worker_count <= 0: + return None + # Each active conversion can hold extracted/rendered members plus a new CBZ + # at the same time. Two times the largest N concurrent source sizes is a + # conservative, deterministic estimate rather than a claim about final + # compression ratio; the fixed-or-10% reserve supplies additional headroom. + estimated_workspace_bytes = selected_source_bytes * _CONVERSION_WORKSPACE_MULTIPLIER + reserve_bytes = managed_copy_capacity_reserve(estimated_workspace_bytes) + required_bytes = estimated_workspace_bytes + reserve_bytes + free_bytes: int | None + try: + temp_root = Path(tempfile.gettempdir()).resolve(strict=True) + usage = await asyncio.to_thread(shutil.disk_usage, temp_root) + free_bytes = int(usage.free) + except (OSError, RuntimeError, ValueError): + free_bytes = None + status = ( + "unknown" + if free_bytes is None + else "insufficient" + if free_bytes < required_bytes + else "ready" + ) + return ConversionWorkspaceCapacitySnapshot( + selected_source_bytes=selected_source_bytes, + active_worker_count=active_worker_count, + estimated_workspace_bytes=estimated_workspace_bytes, + reserve_bytes=reserve_bytes, + required_bytes=required_bytes, + free_bytes=free_bytes, + status=status, + ) + + +def _target_result_snapshot( + *, + stage: ManagedCopyPreflightStage, + target: ManagedCopyTargetCapacitySnapshot, + targets: list[ManagedCopyTargetCapacitySnapshot], + workspace: ConversionWorkspaceCapacitySnapshot | None = None, + force_v2: bool, +) -> ManagedCopyCapacitySnapshot: + return ManagedCopyCapacitySnapshot( + schema_version=2 if force_v2 else 1, + stage=stage, + target_library_root_id=target.target_library_root_id, + selected_source_bytes=target.selected_source_bytes, + reserve_bytes=target.reserve_bytes, + required_bytes=target.required_bytes, + free_bytes=target.free_bytes, + status=target.status, + target_capacities=tuple(targets) if force_v2 else (), + conversion_workspace=workspace if force_v2 else None, + ) + + +def _workspace_result_snapshot( + *, + stage: ManagedCopyPreflightStage, + targets: list[ManagedCopyTargetCapacitySnapshot], + workspace: ConversionWorkspaceCapacitySnapshot, +) -> ManagedCopyCapacitySnapshot: + return ManagedCopyCapacitySnapshot( + schema_version=2, + stage=stage, + target_library_root_id=None, + selected_source_bytes=workspace.estimated_workspace_bytes, + reserve_bytes=workspace.reserve_bytes, + required_bytes=workspace.required_bytes, + free_bytes=workspace.free_bytes, + status=workspace.status, + target_capacities=tuple(targets), + conversion_workspace=workspace, + ) + + +def _record_capacity_snapshot( + job: ImportJob, + snapshot: ManagedCopyCapacitySnapshot, +) -> None: + progress_snapshot = dict(job.progress_snapshot or {}) + progress_snapshot[_CAPACITY_SNAPSHOT_KEY] = snapshot.as_dict() + job.progress_snapshot = progress_snapshot diff --git a/src/pullbox/services/import_metadata_priority.py b/src/pullbox/services/import_metadata_priority.py new file mode 100644 index 00000000..77d87325 --- /dev/null +++ b/src/pullbox/services/import_metadata_priority.py @@ -0,0 +1,92 @@ +"""Weighted coordination between catalog hydration and ComicInfo enrichment.""" + +from __future__ import annotations + +import asyncio +from contextlib import AbstractAsyncContextManager +from dataclasses import dataclass + +_CATALOGS_PER_COMICINFO_TURN = 3 + + +@dataclass +class _PriorityState: + condition: asyncio.Condition + pending_catalogs: int = 0 + catalog_credits: int = 0 + + +_state: _PriorityState | None = None +_state_loop: asyncio.AbstractEventLoop | None = None + + +def _priority_state() -> _PriorityState: + global _state, _state_loop + loop = asyncio.get_running_loop() + if _state is None or _state_loop is not loop: + _state = _PriorityState(asyncio.Condition()) + _state_loop = loop + return _state + + +def reset_import_metadata_priority() -> None: + """Reset process-local coordination between tests or event-loop restarts.""" + global _state, _state_loop + _state = None + _state_loop = None + + +class CatalogMetadataWork(AbstractAsyncContextManager["CatalogMetadataWork"]): + """Track a known catalog batch and expose per-series completion checkpoints.""" + + def __init__(self, units: int) -> None: + self._units = max(units, 0) + self._remaining = self._units + self._state: _PriorityState | None = None + + async def __aenter__(self) -> CatalogMetadataWork: + self._state = _priority_state() + async with self._state.condition: + if self._state.pending_catalogs == 0: + self._state.catalog_credits = 0 + self._state.pending_catalogs += self._units + self._state.condition.notify_all() + return self + + async def complete_one(self) -> None: + if self._state is None or self._remaining == 0: + return + async with self._state.condition: + self._remaining -= 1 + self._state.pending_catalogs -= 1 + self._state.catalog_credits += 1 + self._state.condition.notify_all() + + async def __aexit__(self, *_exc: object) -> None: + assert self._state is not None + async with self._state.condition: + self._state.pending_catalogs -= self._remaining + self._remaining = 0 + if self._state.pending_catalogs == 0: + self._state.catalog_credits = 0 + self._state.condition.notify_all() + + +def catalog_metadata_work(units: int) -> CatalogMetadataWork: + """Register catalog work before it competes for ComicVine capacity.""" + return CatalogMetadataWork(units) + + +async def wait_for_comicinfo_turn() -> None: + """Give each three catalog completions priority over one new file update.""" + state = _priority_state() + async with state.condition: + await state.condition.wait_for( + lambda: ( + state.pending_catalogs == 0 or state.catalog_credits >= _CATALOGS_PER_COMICINFO_TURN + ) + ) + if state.pending_catalogs == 0: + state.catalog_credits = 0 + else: + state.catalog_credits -= _CATALOGS_PER_COMICINFO_TURN diff --git a/src/pullbox/services/import_metadata_progress.py b/src/pullbox/services/import_metadata_progress.py new file mode 100644 index 00000000..d2a51295 --- /dev/null +++ b/src/pullbox/services/import_metadata_progress.py @@ -0,0 +1,269 @@ +"""Overall import metadata activity, derived from durable catalog and file state.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager, suppress +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import structlog +from sqlalchemy import func, select + +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobStatus, +) +from pullbox.models.operation_progress import ( + OperationProgress, + OperationProgressState, + OperationProgressTone, + OperationProgressType, +) +from pullbox.models.series import IssueCatalogState, Series +from pullbox.services.operation_progress import ( + OperationProgressMeasure, + OperationProgressUpdate, + publish_operation_progress, +) +from pullbox.services.operation_progress_dispatch import notify_activity_changed + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +logger = structlog.get_logger(__name__) +_REFRESH_SECONDS = 5.0 + + +@dataclass +class _MetadataActivity: + factory: async_sessionmaker[AsyncSession] + job_id: int + workers: int = 0 + task: asyncio.Task[None] | None = None + stop: asyncio.Event = field(default_factory=asyncio.Event) + lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + async def refresh(self) -> None: + try: + async with self.lock: + async with self.factory() as session: + update = await build_import_metadata_progress( + session, + job_id=self.job_id, + running=self.workers > 0, + ) + if update is None: + return + existing = await session.scalar( + select(OperationProgress).where( + OperationProgress.operation_type == update.operation_type, + OperationProgress.operation_key == update.operation_key, + ) + ) + if existing is not None and ( + existing.state == update.state + and existing.message == update.message + and existing.overall_current == update.overall.current + and existing.overall_total == update.overall.total + ): + return + await publish_operation_progress(session, update) + await session.commit() + await notify_activity_changed(update) + except Exception: + # Progress failure must never interrupt a catalog fetch or archive write. + logger.exception("import_metadata_activity_update_failed", job_id=self.job_id) + + async def run(self, stop: asyncio.Event) -> None: + while not stop.is_set(): + await self.refresh() + with suppress(TimeoutError): + await asyncio.wait_for(stop.wait(), timeout=_REFRESH_SECONDS) + + +_activities: dict[tuple[asyncio.AbstractEventLoop, int], _MetadataActivity] = {} + + +@asynccontextmanager +async def track_import_metadata_progress( + factory: async_sessionmaker[AsyncSession], + *, + job_ids: list[int], +) -> AsyncIterator[None]: + """Share one periodic summary across both metadata lanes for each import.""" + loop = asyncio.get_running_loop() + tracked = [] + for job_id in set(job_ids): + key = (loop, job_id) + activity = _activities.setdefault(key, _MetadataActivity(factory, job_id)) + activity.workers += 1 + if activity.task is None: + activity.stop = asyncio.Event() + activity.task = asyncio.create_task(activity.run(activity.stop)) + tracked.append((key, activity)) + try: + yield + finally: + for key, activity in tracked: + activity.workers -= 1 + if activity.workers: + continue + task, activity.task = activity.task, None + activity.stop.set() + if task is not None: + await task + await activity.refresh() + if activity.workers == 0: + _activities.pop(key, None) + + +async def catalog_hydration_import_job_ids( + factory: async_sessionmaker[AsyncSession], + *, + series_id: int | None = None, +) -> list[int]: + """Find import owners without expanding large series lists into SQL parameters.""" + try: + async with factory() as session: + statement = ( + select(ImportedSeries.import_job_id) + .join(ImportJob, ImportJob.id == ImportedSeries.import_job_id) + .join(Series, Series.id == ImportedSeries.series_id) + .where(ImportJob.status.in_([ImportJobStatus.IMPORTING, ImportJobStatus.COMPLETED])) + .distinct() + ) + if series_id is not None: + statement = statement.where(Series.id == series_id) + else: + statement = statement.where( + Series.issue_catalog_state == IssueCatalogState.HYDRATING + ) + return list((await session.scalars(statement)).all()) + except Exception: + # Activity ownership is observability only and cannot block real metadata work. + logger.exception("import_metadata_owner_lookup_failed", series_id=series_id) + return [] + + +async def build_import_metadata_progress( + session: AsyncSession, + *, + job_id: int, + running: bool, +) -> OperationProgressUpdate | None: + """Count finished work, not individual-file bytes or elapsed-time guesses.""" + job = await session.get(ImportJob, job_id) + if job is None: + return None + key = f"metadata:{job_id}" + if job.status != ImportJobStatus.COMPLETED: + existing = await session.scalar( + select(OperationProgress.id).where( + OperationProgress.operation_type == OperationProgressType.IMPORT, + OperationProgress.operation_key == key, + ) + ) + if existing is None: + return None + return OperationProgressUpdate( + operation_type=OperationProgressType.IMPORT, + operation_key=key, + revision=None, + state=OperationProgressState.CANCELLED, + phase="metadata_sync", + title="Import metadata sync", + message="Metadata sync stopped because the import is no longer complete.", + source_label=f"Import #{job_id}", + group_key="import_metadata", + ) + + series_ids = select(ImportedSeries.series_id).where(ImportedSeries.import_job_id == job_id) + catalog_rows = await session.execute( + select(Series.issue_catalog_state, Series.metadata_source, func.count()) + .where(Series.id.in_(series_ids), Series.comicvine_id.isnot(None)) + .group_by(Series.issue_catalog_state, Series.metadata_source) + ) + catalog_groups = list(catalog_rows.all()) + catalogs: dict[IssueCatalogState, int] = {} + profile_done = 0 + for catalog_state, metadata_source, count in catalog_groups: + catalogs[catalog_state] = catalogs.get(catalog_state, 0) + count + if metadata_source == "comicvine" or catalog_state == IssueCatalogState.COMPLETE: + profile_done += count + status = ImportedFile.diagnostics["comicinfo_enrichment"]["status"].as_string() + file_rows = await session.execute( + select(status, func.count()) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.IMPORTED, + status.in_(["pending", "complete", "failed"]), + ) + .group_by(status) + ) + files = {state: count for state, count in file_rows.all()} + catalog_total = sum(catalogs.values()) + file_total = sum(files.values()) + total = catalog_total + file_total + if not total: + return None + catalog_done = catalogs.get(IssueCatalogState.COMPLETE, 0) + file_done = files.get("complete", 0) + failed = catalogs.get(IssueCatalogState.FAILED, 0) + files.get("failed", 0) + finished = catalog_done + file_done + failed + pending = total - finished + state = OperationProgressState.RUNNING + tone = OperationProgressTone.INFO + attention = False + summary = ( + f"Series details: {profile_done:,} of {catalog_total:,}. " + f"Issue catalogs: {catalog_done:,} of {catalog_total:,}. " + f"ComicInfo files: {file_done:,} of {file_total:,}." + ) + if not pending: + state = OperationProgressState.FAILED if failed else OperationProgressState.COMPLETED + tone = OperationProgressTone.WARNING if failed else OperationProgressTone.SUCCESS + attention = bool(failed) + summary = ( + f"Metadata sync finished with {failed:,} failed updates. " + if failed + else "Metadata sync complete. " + ) + summary + elif not running: + state = OperationProgressState.PAUSED + tone = OperationProgressTone.WARNING + attention = True + summary = "Metadata sync paused. Check import logs for provider or file errors. " + summary + else: + summary = "Syncing metadata in the background. " + summary + return OperationProgressUpdate( + operation_type=OperationProgressType.IMPORT, + operation_key=key, + revision=None, + state=state, + phase="metadata_sync", + title="Import metadata sync", + message=summary, + source_label=f"Import #{job_id}", + group_key="import_metadata", + detail_url="/import?tab=history", + tone=tone, + attention_required=attention, + overall=OperationProgressMeasure(current=finished, total=total, unit="updates"), + detail_snapshot={ + "job_id": job_id, + "profiles_complete": profile_done, + "profiles_total": catalog_total, + "catalogs_complete": catalog_done, + "catalogs_total": catalog_total, + "files_complete": file_done, + "files_total": file_total, + "failed": failed, + "pending": pending, + }, + ) diff --git a/src/pullbox/services/import_misplaced_source_cleanup.py b/src/pullbox/services/import_misplaced_source_cleanup.py new file mode 100644 index 00000000..29ecbfa8 --- /dev/null +++ b/src/pullbox/services/import_misplaced_source_cleanup.py @@ -0,0 +1,894 @@ +"""Previewed source cleanup for exact cross-folder Mylar recoveries.""" + +from __future__ import annotations + +import asyncio +import enum +import os +import shutil +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from hashlib import sha256 +from pathlib import Path +from typing import TYPE_CHECKING, Final, cast + +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +from sqlalchemy import func, select + +from pullbox.config import get_settings +from pullbox.core.config_resolver import get_application_secret +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.filesystem_policy import is_invalid_path_text +from pullbox.core.library_file_ownership import ( + build_file_identity_signature, + validate_file_identity_signature, +) +from pullbox.models.audit_log import AuditEventType +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportJob, + ImportJobAction, + ImportJobStatus, + ImportSourceType, +) +from pullbox.models.library import LibraryFile, LibraryFileStorageMode, LibraryRoot +from pullbox.services.audit_service import AuditService +from pullbox.services.import_duplicate_copies import compute_content_hash +from pullbox.services.import_runtime_settings import load_import_utility_trash_folder +from pullbox.utilities.settings import ( + move_file_to_utility_trash, + resolve_trash_directory, + restore_file_from_utility_trash, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + + +class MisplacedSourceCleanupAction(enum.StrEnum): + """Explicit physical cleanup choices for one recovered Mylar source.""" + + RESTORE_RECORDED_PATH = "restore_recorded_path" + TRASH_IDENTICAL_DUPLICATE = "trash_identical_duplicate" + + +@dataclass(frozen=True, slots=True) +class MisplacedSourceCleanupPreview: + """One actor-bound preview for a physical source change.""" + + job_id: int + file_id: int + action: MisplacedSourceCleanupAction + file_name: str + source_path: str + destination_path: str | None + can_apply: bool + unavailable_reason: str + preview_token: str | None + + +@dataclass(frozen=True, slots=True) +class MisplacedSourceCleanupResult: + """Result of one explicit source cleanup.""" + + final_path: Path + + +@dataclass(frozen=True, slots=True) +class MisplacedSourceCleanupBulkPreview: + """Signed preview for every currently eligible verified misplaced file.""" + + job_id: int + affected_count: int + unavailable_count: int + examples: tuple[str, ...] + preview_token: str | None + + +@dataclass(frozen=True, slots=True) +class MisplacedSourceCleanupBulkResult: + """Outcome of one verified bulk source-organization operation.""" + + moved_count: int + skipped_count: int + + +@dataclass(frozen=True, slots=True) +class MisplacedSourceCleanupFilePage: + """One bounded page of exact source-cleanup candidates.""" + + items: tuple[ImportedFile, ...] + total: int + page: int + page_size: int + total_pages: int + + +@dataclass(frozen=True, slots=True) +class _CleanupContext: + job: ImportJob + imported_file: ImportedFile + library_file: LibraryFile + source: Path + destination: Path | None + signature: dict[str, int | str] + unavailable_reason: str + + +@dataclass(frozen=True, slots=True) +class _DuplicateCleanupContext: + job: ImportJob + imported_file: ImportedFile + canonical_file: ImportedFile + source: Path + signature: dict[str, int | str] + canonical_signature: dict[str, int | str] + content_hash: str + trash_dir: Path | None + unavailable_reason: str + + +_TOKEN_SALT: Final = "import-misplaced-source-cleanup-v1" +_VERIFIED_CROSS_FOLDER_METHODS: Final = ( + "verified_cross_folder_issue_identity", + "verified_cross_folder_series_issue_filename", +) +_TOKEN_MAX_AGE_SECONDS: Final = 15 * 60 + + +def _serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(get_application_secret(), salt=_TOKEN_SALT) + + +async def _load_completed_mylar_job(session: AsyncSession, job_id: int) -> ImportJob: + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if ( + job.status is not ImportJobStatus.COMPLETED + or job.source_type is not ImportSourceType.MYLAR3 + ): + raise ValidationError("Mylar source cleanup is available after a completed Mylar import.") + return job + + +def _cleanup_scope_filters( + job_id: int, + action: MisplacedSourceCleanupAction, +) -> tuple[ColumnElement[bool], ...]: + evidence = ImportedFile.diagnostics["mylar3_cross_folder_reconciliation"] + base: tuple[ColumnElement[bool], ...] = ( + ImportedFile.import_job_id == job_id, + evidence["method"].as_string().in_(_VERIFIED_CROSS_FOLDER_METHODS), + ) + if action is MisplacedSourceCleanupAction.RESTORE_RECORDED_PATH: + return ( + *base, + ImportedFile.status == ImportedFileStatus.IMPORTED, + evidence["role"].as_string() == "canonical", + evidence["restored_at"].as_string().is_(None), + ImportedFile.library_file_id.is_not(None), + ) + if action is MisplacedSourceCleanupAction.TRASH_IDENTICAL_DUPLICATE: + return ( + *base, + ImportedFile.status == ImportedFileStatus.DUPLICATE_FILE, + evidence["role"].as_string() == "identical_duplicate", + ImportedFile.diagnostics["misplaced_source_cleanup"]["action"].as_string().is_(None), + ImportedFile.duplicate_of_file_id.is_not(None), + ImportedFile.content_hash.is_not(None), + ) + raise ValidationError("This misplaced source cleanup action is not supported.") + + +async def count_misplaced_source_cleanup_files( + session: AsyncSession, + job_id: int, + action: MisplacedSourceCleanupAction, +) -> int: + """Count pending exact source-cleanup candidates without hydrating them.""" + await _load_completed_mylar_job(session, job_id) + return int( + ( + await session.scalar( + select(func.count(ImportedFile.id)).where(*_cleanup_scope_filters(job_id, action)) + ) + ) + or 0 + ) + + +async def list_misplaced_source_cleanup_files( + session: AsyncSession, + job_id: int, + action: MisplacedSourceCleanupAction, + *, + page: int = 1, + page_size: int = 25, +) -> MisplacedSourceCleanupFilePage: + """Return a deterministic, bounded page of pending source cleanups.""" + await _load_completed_mylar_job(session, job_id) + normalized_page = max(1, int(page)) + normalized_page_size = min(max(1, int(page_size)), 100) + filters = _cleanup_scope_filters(job_id, action) + total = int((await session.scalar(select(func.count(ImportedFile.id)).where(*filters))) or 0) + total_pages = max(1, (total + normalized_page_size - 1) // normalized_page_size) + normalized_page = min(normalized_page, total_pages) + items = tuple( + ( + await session.scalars( + select(ImportedFile) + .where(*filters) + .order_by(ImportedFile.file_name, ImportedFile.id) + .offset((normalized_page - 1) * normalized_page_size) + .limit(normalized_page_size) + ) + ).all() + ) + return MisplacedSourceCleanupFilePage( + items=items, + total=total, + page=normalized_page, + page_size=normalized_page_size, + total_pages=total_pages, + ) + + +def _inside(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def _safe_absolute_path(value: object, *, label: str) -> Path: + if not isinstance(value, str) or not value or is_invalid_path_text(value): + raise ValidationError(f"The recorded {label} is invalid. Re-scan before cleanup.") + path = Path(value).expanduser() + if not path.is_absolute() or ".." in path.parts: + raise ValidationError(f"The recorded {label} is invalid. Re-scan before cleanup.") + return path.absolute() + + +async def _reference_root_for_path( + session: AsyncSession, + *, + lexical_path: Path, + resolved_path: Path, + require_managed_writes: bool, +) -> tuple[LibraryRoot | None, str]: + roots = list( + await session.scalars( + select(LibraryRoot).where( + LibraryRoot.enabled.is_(True), + LibraryRoot.allow_referenced_registrations.is_(True), + ) + ) + ) + matches: list[LibraryRoot] = [] + for root in roots: + try: + lexical_root = Path(root.path).expanduser().absolute() + resolved_root = Path(root.path).expanduser().resolve(strict=True) + except (OSError, RuntimeError, ValueError): + continue + if _inside(lexical_path, lexical_root) and _inside(resolved_path, resolved_root): + matches.append(root) + if len(matches) != 1: + return None, "The file does not belong to one unambiguous enabled library root." + if require_managed_writes and not matches[0].allow_managed_writes: + return None, ( + "This library root does not allow managed writes. Enable managed writes only when " + "you are ready for Pullbox to change the Mylar source library." + ) + return matches[0], "" + + +async def _load_restore_context( + session: AsyncSession, + job_id: int, + file_id: int, +) -> _CleanupContext: + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if ( + job.status is not ImportJobStatus.COMPLETED + or job.source_type is not ImportSourceType.MYLAR3 + ): + raise ValidationError("Mylar source cleanup is available after a completed Mylar import.") + imported_file = await session.get(ImportedFile, file_id) + if imported_file is None or imported_file.import_job_id != job_id: + raise NotFoundError("ImportedFile", file_id) + evidence = dict(imported_file.diagnostics or {}).get("mylar3_cross_folder_reconciliation") + if ( + imported_file.status is not ImportedFileStatus.IMPORTED + or not isinstance(evidence, Mapping) + or evidence.get("method") not in _VERIFIED_CROSS_FOLDER_METHODS + or evidence.get("role") != "canonical" + ): + raise ValidationError( + "Only an imported, exactly identified misplaced file can be restored." + ) + if imported_file.library_file_id is None: + raise ValidationError("The misplaced file no longer has a Pullbox library reference.") + library_file = await session.get(LibraryFile, imported_file.library_file_id) + if ( + library_file is None + or library_file.storage_mode is not LibraryFileStorageMode.REFERENCED + or library_file.issue_id != imported_file.matched_issue_id + ): + raise ValidationError("The misplaced file reference changed after import.") + + source_lexical = _safe_absolute_path(imported_file.file_path, label="source path") + destination_lexical = _safe_absolute_path(evidence.get("recorded_path"), label="Mylar path") + try: + if source_lexical.is_symlink(): + raise ValidationError("Symlinked source files cannot use Mylar path restoration.") + source = source_lexical.resolve(strict=True) + destination = destination_lexical.resolve(strict=False) + signature = build_file_identity_signature(source) + validate_file_identity_signature(dict(imported_file.source_signature or {}), signature) + validate_file_identity_signature(dict(library_file.source_signature or {}), signature) + except ValidationError: + raise + except Exception as exc: + raise ValidationError( + "The misplaced source changed or is unavailable. Re-scan before cleanup." + ) from exc + if Path(library_file.file_path).expanduser().resolve(strict=True) != source: + raise ValidationError("The misplaced file reference changed after import.") + if source == destination: + return _CleanupContext(job, imported_file, library_file, source, None, signature, "") + + _source_root, source_reason = await _reference_root_for_path( + session, + lexical_path=source_lexical, + resolved_path=source, + require_managed_writes=False, + ) + destination_parent = destination.parent + try: + resolved_parent = destination_parent.resolve(strict=True) + except (OSError, RuntimeError, ValueError): + resolved_parent = destination_parent + _destination_root, destination_reason = await _reference_root_for_path( + session, + lexical_path=destination_lexical, + resolved_path=resolved_parent / destination.name, + require_managed_writes=False, + ) + unavailable_reason = source_reason or destination_reason + if not unavailable_reason and os.path.lexists(destination): + unavailable_reason = ( + "The Mylar-recorded destination already exists, so Pullbox left it unchanged." + ) + if not unavailable_reason and ( + not destination_parent.is_dir() or not os.access(destination_parent, os.W_OK) + ): + unavailable_reason = "The Mylar-recorded destination folder is not writable." + if not unavailable_reason and not os.access(source.parent, os.W_OK): + unavailable_reason = "The current source folder is not writable." + return _CleanupContext( + job, + imported_file, + library_file, + source, + destination, + signature, + unavailable_reason, + ) + + +async def _load_duplicate_context( + session: AsyncSession, + job_id: int, + file_id: int, +) -> _DuplicateCleanupContext: + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if ( + job.status is not ImportJobStatus.COMPLETED + or job.source_type is not ImportSourceType.MYLAR3 + ): + raise ValidationError("Mylar source cleanup is available after a completed Mylar import.") + imported_file = await session.get(ImportedFile, file_id) + if imported_file is None or imported_file.import_job_id != job_id: + raise NotFoundError("ImportedFile", file_id) + evidence = dict(imported_file.diagnostics or {}).get("mylar3_cross_folder_reconciliation") + cleaned = dict(imported_file.diagnostics or {}).get("misplaced_source_cleanup") + if ( + imported_file.status is not ImportedFileStatus.DUPLICATE_FILE + or not isinstance(evidence, Mapping) + or evidence.get("method") not in _VERIFIED_CROSS_FOLDER_METHODS + or evidence.get("role") != "identical_duplicate" + or isinstance(cleaned, Mapping) + ): + raise ValidationError( + "Only an untouched, hash-confirmed misplaced duplicate can be removed." + ) + if imported_file.duplicate_of_file_id is None or not imported_file.content_hash: + raise ValidationError("The duplicate no longer has exact canonical-file evidence.") + canonical = await session.get(ImportedFile, imported_file.duplicate_of_file_id) + if ( + canonical is None + or canonical.import_job_id != job_id + or canonical.content_hash != imported_file.content_hash + ): + raise ValidationError("The duplicate no longer has exact canonical-file evidence.") + + source_lexical = _safe_absolute_path(imported_file.file_path, label="duplicate path") + canonical_lexical = _safe_absolute_path(canonical.file_path, label="canonical path") + try: + if source_lexical.is_symlink() or canonical_lexical.is_symlink(): + raise ValidationError("Symlinked files cannot use duplicate source cleanup.") + source = source_lexical.resolve(strict=True) + canonical_source = canonical_lexical.resolve(strict=True) + signature = build_file_identity_signature(source) + canonical_signature = build_file_identity_signature(canonical_source) + validate_file_identity_signature(dict(imported_file.source_signature or {}), signature) + validate_file_identity_signature( + dict(canonical.source_signature or {}), canonical_signature + ) + except ValidationError: + raise + except Exception as exc: + raise ValidationError( + "The duplicate or canonical source changed after import. Leave both files unchanged." + ) from exc + unavailable_reason = "" + _root, root_reason = await _reference_root_for_path( + session, + lexical_path=source_lexical, + resolved_path=source, + require_managed_writes=True, + ) + unavailable_reason = root_reason + actual_hash = await asyncio.to_thread(compute_content_hash, str(source)) + canonical_hash = await asyncio.to_thread(compute_content_hash, str(canonical_source)) + if not unavailable_reason and ( + actual_hash is None + or canonical_hash is None + or actual_hash != imported_file.content_hash + or canonical_hash != imported_file.content_hash + ): + unavailable_reason = "The two files are no longer byte-identical, so Pullbox left both." + configured_trash = await load_import_utility_trash_folder(session) + settings = get_settings() + trash_dir = resolve_trash_directory( + configured_trash, + library_root=settings.library_root, + data_dir=settings.data_dir, + ) + if trash_dir is None: + unavailable_reason = unavailable_reason or ( + "Configure the Trash folder in Media Management before removing duplicates." + ) + return _DuplicateCleanupContext( + job, + imported_file, + canonical, + source, + signature, + canonical_signature, + imported_file.content_hash, + trash_dir, + unavailable_reason, + ) + + +async def preview_misplaced_source_cleanup( + session: AsyncSession, + job_id: int, + file_id: int, + action: MisplacedSourceCleanupAction, + *, + actor_id: int, +) -> MisplacedSourceCleanupPreview: + """Preview one exact cross-folder cleanup without mutating state.""" + if action is MisplacedSourceCleanupAction.RESTORE_RECORDED_PATH: + context = await _load_restore_context(session, job_id, file_id) + destination = context.destination + already_restored = destination is None + token = None + if not context.unavailable_reason and not already_restored: + token = str( + _serializer().dumps( + { + "job_id": job_id, + "file_id": file_id, + "actor_id": actor_id, + "action": action.value, + "source": str(context.source), + "destination": str(destination), + "signature": context.signature, + } + ) + ) + return MisplacedSourceCleanupPreview( + job_id=job_id, + file_id=file_id, + action=action, + file_name=context.imported_file.file_name, + source_path=str(context.source), + destination_path=str(destination) if destination is not None else None, + can_apply=token is not None, + unavailable_reason=( + "This file is already at the Mylar-recorded path." + if already_restored + else context.unavailable_reason + ), + preview_token=token, + ) + if action is MisplacedSourceCleanupAction.TRASH_IDENTICAL_DUPLICATE: + duplicate = await _load_duplicate_context(session, job_id, file_id) + token = None + if not duplicate.unavailable_reason and duplicate.trash_dir is not None: + token = str( + _serializer().dumps( + { + "job_id": job_id, + "file_id": file_id, + "actor_id": actor_id, + "action": action.value, + "source": str(duplicate.source), + "trash_dir": str(duplicate.trash_dir), + "signature": duplicate.signature, + "canonical_signature": duplicate.canonical_signature, + "content_hash": duplicate.content_hash, + } + ) + ) + return MisplacedSourceCleanupPreview( + job_id=job_id, + file_id=file_id, + action=action, + file_name=duplicate.imported_file.file_name, + source_path=str(duplicate.source), + destination_path=str(duplicate.trash_dir) if duplicate.trash_dir is not None else None, + can_apply=token is not None, + unavailable_reason=duplicate.unavailable_reason, + preview_token=token, + ) + raise ValidationError("This misplaced source cleanup action is not supported.") + + +async def _load_verified_restore_contexts( + session: AsyncSession, + job_id: int, +) -> tuple[list[_CleanupContext], int]: + file_ids = list( + await session.scalars( + select(ImportedFile.id) + .where( + *_cleanup_scope_filters( + job_id, + MisplacedSourceCleanupAction.RESTORE_RECORDED_PATH, + ) + ) + .order_by(ImportedFile.id) + ) + ) + contexts: list[_CleanupContext] = [] + unavailable_count = 0 + for file_id in file_ids: + try: + context = await _load_restore_context(session, job_id, int(file_id)) + except ValidationError: + unavailable_count += 1 + continue + if context.unavailable_reason or context.destination is None: + unavailable_count += 1 + continue + contexts.append(context) + return contexts, unavailable_count + + +def _restore_scope_digest(contexts: list[_CleanupContext]) -> str: + digest = sha256() + for context in contexts: + destination = context.destination + if destination is None: + continue + digest.update( + ( + f"{context.imported_file.id}\0{context.source}\0{destination}\0" + f"{sorted(context.signature.items())}\n" + ).encode() + ) + return digest.hexdigest() + + +async def _move_restore_source(context: _CleanupContext) -> Path: + if context.destination is None: + raise ValidationError("This file is already at the Mylar-recorded path.") + await asyncio.to_thread(shutil.move, str(context.source), str(context.destination)) + return context.destination.resolve(strict=True) + + +async def _update_restore_registration( + session: AsyncSession, + context: _CleanupContext, + final_path: Path, +) -> None: + final_signature = build_file_identity_signature(final_path) + stat_result = final_path.stat() + context.imported_file.file_path = str(final_path) + context.imported_file.file_name = final_path.name + context.imported_file.source_signature = final_signature + context.library_file.file_path = str(final_path) + context.library_file.file_name = final_path.name + context.library_file.file_size = stat_result.st_size + context.library_file.file_modified_at = datetime.fromtimestamp(stat_result.st_mtime, UTC) + context.library_file.source_signature = final_signature + diagnostics = dict(context.imported_file.diagnostics or {}) + evidence = dict(diagnostics.get("mylar3_cross_folder_reconciliation") or {}) + evidence.update( + { + "restored_at": datetime.now(UTC).isoformat(), + "restored_path": str(final_path), + } + ) + diagnostics["mylar3_cross_folder_reconciliation"] = evidence + context.imported_file.diagnostics = diagnostics + registration_action = await _load_registration_action( + session, + job_id=context.job.id, + imported_file_id=context.imported_file.id, + ) + if registration_action is not None: + registration_payload = dict(registration_action.payload or {}) + registration_payload.update( + { + "destination_path": str(final_path), + "destination_signature": final_signature, + "original_source_path": str(final_path), + } + ) + registration_action.payload = registration_payload + + +async def _restore_source_moves(moved: list[tuple[Path, Path]]) -> None: + for source, destination in reversed(moved): + if os.path.lexists(destination) and not os.path.lexists(source): + await asyncio.to_thread(shutil.move, str(destination), str(source)) + + +async def preview_verified_misplaced_source_cleanup( + session: AsyncSession, + job_id: int, + *, + actor_id: int, +) -> MisplacedSourceCleanupBulkPreview: + """Preview every currently eligible exact misplaced-file restoration.""" + await _load_completed_mylar_job(session, job_id) + contexts, unavailable_count = await _load_verified_restore_contexts(session, job_id) + token = None + if contexts: + token = str( + _serializer().dumps( + { + "job_id": job_id, + "actor_id": actor_id, + "action": "restore_all_verified", + "affected_count": len(contexts), + "unavailable_count": unavailable_count, + "scope_digest": _restore_scope_digest(contexts), + } + ) + ) + return MisplacedSourceCleanupBulkPreview( + job_id=job_id, + affected_count=len(contexts), + unavailable_count=unavailable_count, + examples=tuple(context.imported_file.file_name for context in contexts[:3]), + preview_token=token, + ) + + +async def apply_verified_misplaced_source_cleanup( + session: AsyncSession, + job_id: int, + *, + actor_id: int, + preview_token: str, + actor_username: str | None = None, + source_ip: str | None = None, +) -> MisplacedSourceCleanupBulkResult: + """Move every file covered by a signed exact-scope preview.""" + payload = _load_token(preview_token) + contexts, unavailable_count = await _load_verified_restore_contexts(session, job_id) + if ( + payload.get("job_id") != job_id + or payload.get("actor_id") != actor_id + or payload.get("action") != "restore_all_verified" + or payload.get("affected_count") != len(contexts) + or payload.get("unavailable_count") != unavailable_count + or payload.get("scope_digest") != _restore_scope_digest(contexts) + ): + raise ValidationError("The verified-file cleanup scope changed. Preview it again.") + if not contexts: + raise ValidationError("No verified misplaced files are currently available to move.") + + moved: list[tuple[Path, Path]] = [] + try: + for context in contexts: + destination = context.destination + if destination is None: + raise ValidationError("A verified misplaced file is already at its proposed path.") + final_path = await _move_restore_source(context) + moved.append((context.source, final_path)) + await _update_restore_registration(session, context, final_path) + await AuditService.log_event( + session, + AuditEventType.IMPORT_MISPLACED_SOURCE_CLEANUP, + source_ip=source_ip, + user_id=actor_id, + username=actor_username, + detail=f"{len(moved)} verified misplaced Mylar sources were organized.", + metadata={ + "job_id": job_id, + "moved_count": len(moved), + "skipped_count": unavailable_count, + }, + ) + await session.commit() + except Exception: + await session.rollback() + await _restore_source_moves(moved) + raise + return MisplacedSourceCleanupBulkResult( + moved_count=len(moved), + skipped_count=unavailable_count, + ) + + +def _load_token(token: str) -> Mapping[str, object]: + try: + payload = _serializer().loads(token, max_age=_TOKEN_MAX_AGE_SECONDS) + except SignatureExpired as exc: + raise ValidationError("The Mylar cleanup preview expired. Preview it again.") from exc + except BadSignature as exc: + raise ValidationError("The Mylar cleanup preview is invalid. Preview it again.") from exc + if not isinstance(payload, Mapping): + raise ValidationError("The Mylar cleanup preview is invalid. Preview it again.") + return payload + + +async def _load_registration_action( + session: AsyncSession, + *, + job_id: int, + imported_file_id: int, +) -> ImportJobAction | None: + return cast( + "ImportJobAction | None", + await session.scalar( + select(ImportJobAction) + .where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.action_type == "library_file_registered", + ImportJobAction.payload["imported_file_id"].as_integer() == imported_file_id, + ) + .order_by(ImportJobAction.id.desc()) + .limit(1) + ), + ) + + +async def apply_misplaced_source_cleanup( + session: AsyncSession, + job_id: int, + file_id: int, + action: MisplacedSourceCleanupAction, + *, + actor_id: int, + preview_token: str, + actor_username: str | None = None, + source_ip: str | None = None, +) -> MisplacedSourceCleanupResult: + """Apply one exact cross-folder cleanup after revalidation.""" + if action is MisplacedSourceCleanupAction.TRASH_IDENTICAL_DUPLICATE: + duplicate = await _load_duplicate_context(session, job_id, file_id) + if duplicate.unavailable_reason or duplicate.trash_dir is None: + raise ValidationError(duplicate.unavailable_reason or "Trash is not configured.") + payload = _load_token(preview_token) + expected_signature = payload.get("signature") + expected_canonical_signature = payload.get("canonical_signature") + if ( + payload.get("job_id") != job_id + or payload.get("file_id") != file_id + or payload.get("actor_id") != actor_id + or payload.get("action") != action.value + or payload.get("source") != str(duplicate.source) + or payload.get("trash_dir") != str(duplicate.trash_dir) + or payload.get("content_hash") != duplicate.content_hash + or not isinstance(expected_signature, Mapping) + or not isinstance(expected_canonical_signature, Mapping) + ): + raise ValidationError("The Mylar cleanup preview does not match this file.") + validate_file_identity_signature(dict(expected_signature), duplicate.signature) + validate_file_identity_signature( + dict(expected_canonical_signature), duplicate.canonical_signature + ) + trash_path = await asyncio.to_thread( + move_file_to_utility_trash, + duplicate.source, + duplicate.trash_dir, + relative_path=Path("import-results") / str(job_id) / duplicate.source.name, + ) + try: + diagnostics = dict(duplicate.imported_file.diagnostics or {}) + diagnostics["misplaced_source_cleanup"] = { + "action": action.value, + "completed_at": datetime.now(UTC).isoformat(), + "trash_path": str(trash_path), + } + duplicate.imported_file.diagnostics = diagnostics + await AuditService.log_event( + session, + AuditEventType.IMPORT_MISPLACED_SOURCE_CLEANUP, + source_ip=source_ip, + user_id=actor_id, + username=actor_username, + detail="One hash-confirmed misplaced Mylar duplicate was moved to Trash.", + metadata={"job_id": job_id, "file_id": file_id}, + ) + await session.commit() + except Exception: + await session.rollback() + await asyncio.to_thread( + restore_file_from_utility_trash, + trash_path, + duplicate.source, + ) + raise + return MisplacedSourceCleanupResult(final_path=trash_path) + if action is not MisplacedSourceCleanupAction.RESTORE_RECORDED_PATH: + raise ValidationError("This misplaced source cleanup action is not supported.") + context = await _load_restore_context(session, job_id, file_id) + if context.unavailable_reason: + raise ValidationError(context.unavailable_reason) + if context.destination is None: + raise ValidationError("This file is already at the Mylar-recorded path.") + payload = _load_token(preview_token) + if ( + payload.get("job_id") != job_id + or payload.get("file_id") != file_id + or payload.get("actor_id") != actor_id + or payload.get("action") != action.value + or payload.get("source") != str(context.source) + or payload.get("destination") != str(context.destination) + ): + raise ValidationError("The Mylar cleanup preview does not match this file.") + expected_signature = payload.get("signature") + if not isinstance(expected_signature, Mapping): + raise ValidationError("The Mylar cleanup preview is invalid. Preview it again.") + validate_file_identity_signature(dict(expected_signature), context.signature) + + source = context.source + moved: list[tuple[Path, Path]] = [] + try: + final_path = await _move_restore_source(context) + moved.append((source, final_path)) + await _update_restore_registration(session, context, final_path) + await AuditService.log_event( + session, + AuditEventType.IMPORT_MISPLACED_SOURCE_CLEANUP, + source_ip=source_ip, + user_id=actor_id, + username=actor_username, + detail="One verified misplaced Mylar source was restored to its recorded path.", + metadata={"job_id": job_id, "file_id": file_id}, + ) + await session.commit() + except Exception: + await session.rollback() + await _restore_source_moves(moved) + raise + return MisplacedSourceCleanupResult(final_path=final_path) diff --git a/src/pullbox/services/import_mylar3_path_preflight.py b/src/pullbox/services/import_mylar3_path_preflight.py new file mode 100644 index 00000000..853dbe41 --- /dev/null +++ b/src/pullbox/services/import_mylar3_path_preflight.py @@ -0,0 +1,1260 @@ +"""Complete, read-only Mylar path-mapping analysis for Import Step 1.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import sqlite3 +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Literal + +from sqlalchemy import select + +from pullbox.core.filesystem_policy import ( + is_invalid_path_text, + is_sensitive_path, + resolve_preview_source, +) +from pullbox.core.mylar3_path_mapping import ( + has_conflicting_overlapping_mappings, + normalize_mylar3_path_map, + normalize_mylar3_path_mapping_items, + ordered_mylar3_path_map_items, +) +from pullbox.models.import_job import ImportFileHandlingMode, ImportSourceType +from pullbox.models.library import LibraryRoot +from pullbox.schemas.import_mylar3_path_preflight import ( + MylarIdentityGroupPreview, + MylarPathAttentionAction, + MylarPathAttentionDetails, + MylarPathAttentionItem, + MylarPathExample, + MylarPathException, + MylarPathMappingDraft, + MylarPathMappingPreview, + MylarPathOutcome, + MylarPathPreviewResponse, + MylarPathProblemGroup, + MylarPathResolutionCounts, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + + from sqlalchemy.ext.asyncio import AsyncSession + +_MAX_LOCATIONS = 100_000 +_MAX_EXAMPLES = 3 + + +@dataclass(frozen=True, slots=True) +class _RootBoundary: + root_id: int + name: str + lexical: Path + resolved: Path + device: int + inode: int + + +@dataclass(slots=True) +class _MutableCounts: + locations: int = 0 + identity_resolved: int = 0 + mapped_existing: int = 0 + mapped_missing: int = 0 + missing: int = 0 + unmapped: int = 0 + outside_root: int = 0 + unreadable: int = 0 + ambiguous: int = 0 + invalid: int = 0 + + def response(self) -> MylarPathResolutionCounts: + return MylarPathResolutionCounts(**asdict(self)) + + +@dataclass(slots=True) +class _MappingState: + stored_prefix: str + pullbox_prefix: str + provenance: Literal["automatic", "manual"] + root: _RootBoundary | None = None + counts: _MutableCounts = field(default_factory=_MutableCounts) + examples: list[MylarPathExample] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + blockers: list[str] = field(default_factory=list) + + +@dataclass(slots=True) +class _IdentityState: + root: _RootBoundary + counts: _MutableCounts = field(default_factory=_MutableCounts) + examples: list[MylarPathExample] = field(default_factory=list) + + +@dataclass(frozen=True, slots=True) +class _Location: + path: str + series_id: str + series_name: str + + +class Mylar3PathPreflightAnalyzer: + """Analyze all bounded Mylar series locations without creating a job.""" + + async def analyze( + self, + session: AsyncSession, + source_path: str | Path, + *, + auto_detect: bool, + mappings: list[MylarPathMappingDraft], + file_handling_mode: ImportFileHandlingMode = ImportFileHandlingMode.MANAGED_COPY, + ) -> MylarPathPreviewResponse: + selected = resolve_preview_source(source_path) + database = selected / "mylar.db" if selected.is_dir() else selected + database = resolve_preview_source(database) + if not database.is_file(): + raise ValueError("Mylar path preview requires a mylar.db file") + + root_query = select(LibraryRoot).where(LibraryRoot.enabled.is_(True)) + if file_handling_mode == ImportFileHandlingMode.IN_PLACE: + root_query = root_query.where(LibraryRoot.allow_referenced_registrations.is_(True)) + root_models = list( + (await session.execute(root_query.order_by(LibraryRoot.id.asc()))).scalars().all() + ) + return await asyncio.to_thread( + self._analyze, + database, + root_models, + auto_detect, + mappings, + file_handling_mode, + ) + + def _analyze( + self, + database: Path, + root_models: list[LibraryRoot], + auto_detect: bool, + mappings: list[MylarPathMappingDraft], + file_handling_mode: ImportFileHandlingMode, + ) -> MylarPathPreviewResponse: + roots, root_warnings = self._snapshot_roots(root_models) + inventory, partial = self._read_locations(database) + locations = [entry.path for entry in inventory] + + supplied_map = normalize_mylar3_path_mapping_items( + (mapping.stored_prefix, mapping.pullbox_prefix) for mapping in mappings + ) + ambiguous_auto_locations: set[str] = set() + if auto_detect and not supplied_map: + supplied_map, ambiguous_auto_locations = self._auto_detect_map(locations, roots) + provenance: Literal["automatic", "manual"] = "automatic" if auto_detect else "manual" + + total = _MutableCounts(locations=len(locations)) + require_library_root = file_handling_mode == ImportFileHandlingMode.IN_PLACE + mapping_states = self._mapping_states( + supplied_map, + provenance, + roots, + require_library_root=require_library_root, + ) + identity_states: dict[int, _IdentityState] = {} + ordered_map = ordered_mylar3_path_map_items(supplied_map) + exceptions: list[MylarPathException] = [] + + for entry in inventory: + location = entry.path + if location in ambiguous_auto_locations: + total.ambiguous += 1 + exceptions.append(_path_exception(entry, "ambiguous", location)) + continue + outcome, root, mapping_state, relative = self._resolve_location( + location, + roots, + mapping_states, + ordered_map, + require_library_root=require_library_root, + ) + setattr(total, _count_field(outcome), getattr(total, _count_field(outcome)) + 1) + if outcome not in {"identity", "mapped"}: + attempted = ( + str(Path(mapping_state.pullbox_prefix) / relative) + if mapping_state + else location + ) + exceptions.append(_path_exception(entry, outcome, attempted)) + if outcome == "identity" and root is not None: + identity = identity_states.setdefault(root.root_id, _IdentityState(root=root)) + identity.counts.locations += 1 + identity.counts.identity_resolved += 1 + _append_example(identity.examples, relative, outcome) + elif mapping_state is not None: + mapping_state.counts.locations += 1 + count_field = _count_field(outcome) + setattr( + mapping_state.counts, + count_field, + getattr(mapping_state.counts, count_field) + 1, + ) + _append_example(mapping_state.examples, relative, outcome) + + warnings = list(root_warnings) + if not roots and require_library_root: + warnings.append("no_enabled_library_roots") + if total.unmapped: + warnings.append("unmapped_locations") + if total.missing: + warnings.append("missing_locations") + if ambiguous_auto_locations: + warnings.append("ambiguous_mapping_candidates") + if partial: + warnings.append("partial_preview") + identity_group_total = sum(state.counts.locations for state in identity_states.values()) + if total.identity_resolved > identity_group_total: + warnings.append("external_identity_sources") + + for state in mapping_states.values(): + if state.counts.locations == 0: + state.blockers.append("mapping_does_not_improve_coverage") + blocking_counts = total.outside_root + total.unreadable + total.ambiguous + total.invalid + mapping_blocked = any(state.blockers for state in mapping_states.values()) + root_contract_blocked = (require_library_root or provenance == "automatic") and any( + warning in {"library_root_alias", "nested_library_roots_ambiguous"} + for warning in root_warnings + ) + unresolved = total.missing + total.mapped_missing + total.unmapped + unavailable_roots = any( + warning in {"library_root_unavailable", "library_root_unreadable"} + for warning in root_warnings + ) + automatic_evidence_blocked = provenance == "automatic" and bool( + unresolved or unavailable_roots + ) + if automatic_evidence_blocked: + warnings.append("automatic_mapping_incomplete") + has_available = total.identity_resolved + total.mapped_existing > 0 + safe = has_available and not any( + (partial, blocking_counts, mapping_blocked, root_contract_blocked, unavailable_roots) + ) + can_confirm = safe and not any( + ( + partial, + blocking_counts, + mapping_blocked, + root_contract_blocked, + automatic_evidence_blocked, + ) + ) + can_continue_with_unresolved = bool(safe and unresolved) + problem_groups = _group_path_problems(exceptions) + unavailable_root_models = [ + root for root in root_models if root.id not in {snapshot.root_id for snapshot in roots} + ] + blocking_reasons = [ + f"Library root '{root.name}' ({root.path}) is unavailable or unreadable. " + "Check its mount and permissions." + for root in unavailable_root_models + ] + [ + f"Mapping {state.stored_prefix} to {state.pullbox_prefix}: {reason.replace('_', ' ')}." + for state in mapping_states.values() + for reason in state.blockers + ] + attention_items = _build_attention_items( + exceptions=exceptions, + can_continue_with_unresolved=can_continue_with_unresolved, + unavailable_roots=unavailable_root_models, + available_roots=roots, + mapping_states=list(mapping_states.values()), + warnings=warnings, + partial=partial, + ) + return MylarPathPreviewResponse( + source_type=ImportSourceType.MYLAR3, + resolution=total.response(), + identity_groups=[ + MylarIdentityGroupPreview( + stored_prefix=str(state.root.lexical), + library_root_id=state.root.root_id, + library_root_name=state.root.name, + resolution=state.counts.response(), + examples=state.examples, + ) + for state in identity_states.values() + ], + mappings=[self._mapping_response(state) for state in mapping_states.values()], + path_map=supplied_map, + requires_confirmation=bool(supplied_map), + can_confirm=can_confirm, + can_continue_with_unresolved=can_continue_with_unresolved, + requires_unresolved_acknowledgement=bool(unresolved), + unresolved_fingerprint=_exception_fingerprint(exceptions) if unresolved else None, + exceptions=exceptions, + exception_count=len(exceptions), + problem_groups=problem_groups, + attention_items=attention_items, + attention_fingerprint=_attention_fingerprint(attention_items), + blocking_reasons=blocking_reasons, + partial=partial, + warnings=list(dict.fromkeys(warnings)), + ) + + @staticmethod + def _read_locations(database: Path) -> tuple[list[_Location], bool]: + uri = f"{database.resolve().as_uri()}?mode=ro" + try: + connection = sqlite3.connect(uri, uri=True) + try: + connection.execute("PRAGMA query_only = ON") + comics_columns = _sqlite_table_columns(connection, "comics") + if "ComicLocation" not in comics_columns: + raise sqlite3.DatabaseError("comics.ComicLocation is unavailable") + # Only these fixed SQL expressions are interpolated, never source column text. + name_column = "comic.ComicName" if "ComicName" in comics_columns else "''" + id_column = "comic.ComicID" if "ComicID" in comics_columns else "''" + inventory_queries = [ + "SELECT ComicLocation AS stored_location, " # nosec B608 + f"{id_column} AS series_id, {name_column} AS series_name, " + "0 AS source_kind, comic.rowid AS source_rowid FROM comics AS comic " + "WHERE ComicLocation IS NOT NULL AND ComicLocation != ''" + ] + issues_columns = _sqlite_table_columns(connection, "issues") + if {"ComicID", "Location"}.issubset(issues_columns) and "ComicID" in ( + comics_columns + ): + inventory_queries.append( + "SELECT issue.Location AS stored_location, " # nosec B608 + f"{id_column} AS series_id, {name_column} AS series_name, " + "1 AS source_kind, issue.rowid AS source_rowid FROM issues AS issue " + "JOIN comics AS comic ON comic.ComicID = issue.ComicID " + "WHERE issue.Location IS NOT NULL AND issue.Location != '' " + "AND substr(issue.Location, 1, 1) = '/' " + "AND EXISTS (" + "SELECT 1 FROM comics AS comic " + "WHERE comic.ComicID = issue.ComicID" + ")" + ) + query = ( + "SELECT stored_location, series_id, series_name FROM (" + + " UNION ALL ".join(inventory_queries) + + ") ORDER BY stored_location, source_kind, source_rowid LIMIT ?" + ) + cursor = connection.execute(query, (_MAX_LOCATIONS + 1,)) + rows = cursor.fetchmany(_MAX_LOCATIONS + 1) + finally: + connection.close() + except sqlite3.DatabaseError as exc: + raise ValueError("Mylar database does not expose ComicLocation values") from exc + partial = len(rows) > _MAX_LOCATIONS + return [ + _Location(str(row[0]), str(row[1] or ""), str(row[2] or "")) + for row in rows[:_MAX_LOCATIONS] + ], partial + + @staticmethod + def _snapshot_roots( + roots: list[LibraryRoot], + ) -> tuple[list[_RootBoundary], list[str]]: + snapshots: list[_RootBoundary] = [] + warnings: list[str] = [] + for root in roots: + try: + lexical = Path(root.path).expanduser().absolute() + resolved = Path(root.path).expanduser().resolve(strict=True) + stat_result = resolved.stat() + except (OSError, RuntimeError, ValueError): + warnings.append("library_root_unavailable") + continue + if not resolved.is_dir() or not os.access(resolved, os.R_OK | os.X_OK): + warnings.append("library_root_unreadable") + continue + snapshots.append( + _RootBoundary( + root_id=root.id, + name=root.name, + lexical=lexical, + resolved=resolved, + device=stat_result.st_dev, + inode=stat_result.st_ino, + ) + ) + + for index, left in enumerate(snapshots): + for right in snapshots[index + 1 :]: + if (left.device, left.inode) == (right.device, right.inode): + warnings.append("library_root_alias") + if _paths_nested(left.lexical, right.lexical) or _paths_nested( + left.resolved, right.resolved + ): + warnings.append("nested_library_roots_ambiguous") + return snapshots, list(dict.fromkeys(warnings)) + + def _mapping_states( + self, + path_map: dict[str, str], + provenance: Literal["automatic", "manual"], + roots: list[_RootBoundary], + *, + require_library_root: bool, + ) -> dict[str, _MappingState]: + states: dict[str, _MappingState] = {} + for stored_prefix, pullbox_prefix in path_map.items(): + state = _MappingState( + stored_prefix=stored_prefix, + pullbox_prefix=pullbox_prefix, + provenance=provenance, + ) + try: + target = Path(pullbox_prefix) + resolved = target.resolve(strict=True) + containing = _containing_roots(target.absolute(), resolved, roots) + if not target.is_absolute() or is_sensitive_path(resolved): + state.blockers.append("mapping_target_unsafe") + elif not resolved.is_dir(): + state.blockers.append("mapping_target_unavailable") + elif not os.access(resolved, os.R_OK | os.X_OK): + state.blockers.append("mapping_target_unreadable") + elif len(containing) == 1: + state.root = containing[0] + elif not containing and require_library_root: + state.blockers.append("mapping_target_outside_enabled_root") + elif len(containing) > 1: + state.blockers.append("mapping_target_root_ambiguous") + except (OSError, RuntimeError, ValueError): + state.blockers.append("mapping_target_unavailable") + states[stored_prefix] = state + return states + + def _resolve_location( + self, + raw_location: str, + roots: list[_RootBoundary], + mapping_states: dict[str, _MappingState], + ordered_map: list[tuple[str, str]], + *, + require_library_root: bool, + ) -> tuple[ + MylarPathOutcome, + _RootBoundary | None, + _MappingState | None, + str, + ]: + if is_invalid_path_text(raw_location) or ".." in Path(raw_location).parts: + return "invalid", None, None, "Unavailable example" + location = Path(raw_location) + if not location.is_absolute(): + return "invalid", None, None, "Unavailable example" + + identity_outside_root = False + try: + resolved_identity = location.resolve(strict=True) + except PermissionError: + return "unreadable", None, None, location.name + except FileNotFoundError: + resolved_identity = None + except (OSError, RuntimeError, ValueError): + return "invalid", None, None, location.name + if resolved_identity is not None and _supported_location_kind(resolved_identity): + if is_sensitive_path(resolved_identity): + return "invalid", None, None, location.name + if not _location_is_readable(resolved_identity): + return "unreadable", None, None, location.name + containing = _containing_roots(location.absolute(), resolved_identity, roots) + if len(containing) == 1: + root = containing[0] + return "identity", root, None, _safe_relative(location, root.lexical) + if len(containing) > 1: + return "ambiguous", None, None, location.name + if require_library_root: + identity_outside_root = True + else: + return "identity", None, None, location.name + + for stored_prefix, pullbox_prefix in ordered_map: + stored_root = Path(stored_prefix) + try: + relative = location.relative_to(stored_root) + except ValueError: + continue + state = mapping_states[stored_prefix] + mapped = Path(pullbox_prefix) / relative + try: + mapped_root = Path(pullbox_prefix).resolve(strict=True) + resolved_mapped = mapped.resolve(strict=False) + except PermissionError: + return "unreadable", state.root, state, str(relative) + except (OSError, RuntimeError, ValueError): + return "mapped_missing", state.root, state, str(relative) + if not resolved_mapped.is_relative_to(mapped_root): + return "outside_root", None, state, str(relative) + if not _supported_location_kind(resolved_mapped): + return "mapped_missing", state.root, state, str(relative) + if not _location_is_readable(resolved_mapped): + return "unreadable", state.root, state, str(relative) + containing = _containing_roots(mapped.absolute(), resolved_mapped, roots) + if len(containing) == 1 and state.root is not None: + return "mapped", containing[0], state, str(relative) + if len(containing) > 1: + return "ambiguous", None, state, str(relative) + if require_library_root: + return "outside_root", None, state, str(relative) + return "mapped", None, state, str(relative) + if identity_outside_root: + return "outside_root", None, None, location.name + try: + resolved_missing = location.resolve(strict=False) + if is_sensitive_path(resolved_missing): + return "invalid", None, None, location.name + containing = _containing_roots(location.absolute(), resolved_missing, roots) + if len(containing) == 1: + return ( + "missing", + containing[0], + None, + _safe_relative(location, containing[0].lexical), + ) + if len(containing) > 1: + return "ambiguous", None, None, location.name + if any(location.is_relative_to(root.lexical) for root in roots): + return "outside_root", None, None, location.name + except PermissionError: + return "unreadable", None, None, location.name + except (OSError, RuntimeError, ValueError): + return "invalid", None, None, location.name + return "unmapped", None, None, location.name + + def _auto_detect_map( + self, + locations: list[str], + roots: list[_RootBoundary], + ) -> tuple[dict[str, str], set[str]]: + candidates_by_location: dict[str, set[tuple[str, str]]] = {} + support: dict[tuple[str, str], int] = {} + for raw_location in locations: + location = Path(raw_location) + if not location.is_absolute() or ".." in location.parts: + continue + try: + identity = location.resolve(strict=True) + except (OSError, RuntimeError, ValueError): + identity = None + if identity is not None and _supported_location_kind(identity): + continue + location_candidates: set[tuple[str, str]] = set() + for root in roots: + for candidate_stored_prefix in _stored_prefixes(location): + relative = location.relative_to(candidate_stored_prefix) + translated = root.lexical / relative + try: + resolved = translated.resolve(strict=True) + except (OSError, RuntimeError, ValueError): + continue + if ( + _supported_location_kind(resolved) + and _location_is_readable(resolved) + and resolved.is_relative_to(root.resolved) + ): + pair = (str(candidate_stored_prefix), str(root.lexical)) + location_candidates.add(pair) + break + if location_candidates: + candidates_by_location[raw_location] = location_candidates + for candidate in location_candidates: + support[candidate] = support.get(candidate, 0) + 1 + + selected: dict[str, str] = {} + ambiguous: set[str] = set() + for raw_location, candidates in candidates_by_location.items(): + best_score = max(support[candidate] for candidate in candidates) + best = sorted(candidate for candidate in candidates if support[candidate] == best_score) + if len(best) != 1: + ambiguous.add(raw_location) + continue + stored_prefix, pullbox_prefix = best[0] + existing = selected.get(stored_prefix) + if existing is not None and existing != pullbox_prefix: + ambiguous.add(raw_location) + continue + selected[stored_prefix] = pullbox_prefix + + if has_conflicting_overlapping_mappings(selected): + return {}, set(candidates_by_location) + return normalize_mylar3_path_map(selected), ambiguous + + @staticmethod + def _mapping_response(state: _MappingState) -> MylarPathMappingPreview: + blocked = bool(state.blockers) + needs_review = state.counts.mapped_missing > 0 or state.counts.unmapped > 0 + return MylarPathMappingPreview( + stored_prefix=state.stored_prefix, + pullbox_prefix=state.pullbox_prefix, + library_root_id=state.root.root_id if state.root is not None else None, + library_root_name=state.root.name if state.root is not None else None, + provenance=state.provenance, + status="blocked" if blocked else "review" if needs_review else "ready", + resolution=state.counts.response(), + examples=state.examples, + warnings=state.warnings, + blocking_reasons=state.blockers, + ) + + +def _count_field(outcome: MylarPathOutcome) -> str: + return { + "identity": "identity_resolved", + "mapped": "mapped_existing", + "mapped_missing": "mapped_missing", + "missing": "missing", + "unmapped": "unmapped", + "outside_root": "outside_root", + "unreadable": "unreadable", + "ambiguous": "ambiguous", + "invalid": "invalid", + }[outcome] + + +def _exception_fingerprint(exceptions: list[MylarPathException]) -> str: + """Bind acknowledgement to paths and outcomes, not just an unchanged count.""" + entries = sorted( + (item.series_id, item.stored_path, item.attempted_path, item.outcome) for item in exceptions + ) + return hashlib.sha256(json.dumps(entries, ensure_ascii=True).encode("utf-8")).hexdigest() + + +def _problem_root(exception: MylarPathException) -> Path: + """Return the smallest useful root that explains this stored location.""" + path = Path(exception.stored_path) + try: + resolved = path.resolve(strict=True) + except (OSError, RuntimeError, ValueError): + resolved = path + if resolved.is_file(): + return resolved.parent.parent + return resolved.parent + + +def _problem_directory(exception: MylarPathException) -> Path: + """Return the existing directory represented by one stored location.""" + path = Path(exception.stored_path) + try: + resolved = path.resolve(strict=True) + except (OSError, RuntimeError, ValueError): + return path.parent + return resolved.parent if resolved.is_file() else resolved + + +def _nearest_safe_mount(path: Path) -> Path | None: + """Return the nearest registerable mount boundary containing ``path``.""" + for candidate in (path, *path.parents): + if candidate == candidate.parent: + break + try: + if candidate.is_mount() and not is_sensitive_path(candidate): + return candidate + except (OSError, RuntimeError, ValueError): + continue + return None + + +def _outside_root_buckets( + exceptions: list[MylarPathException], +) -> list[tuple[Path, list[MylarPathException]]]: + """Group repeated series locations by their actual source boundary.""" + mounted: dict[Path, list[MylarPathException]] = {} + unmounted: list[MylarPathException] = [] + for exception in exceptions: + directory = _problem_directory(exception) + mount = _nearest_safe_mount(directory) + if mount is None: + unmounted.append(exception) + else: + mounted.setdefault(mount, []).append(exception) + + buckets = list(mounted.items()) + if not unmounted: + return buckets + + directories = [_problem_directory(exception) for exception in unmounted] + try: + common = Path(os.path.commonpath([str(path) for path in directories])) + except ValueError: + common = Path() + series_count = len({exception.series_id for exception in unmounted}) + if series_count == 1 and common in directories: + common = common.parent + if common.is_dir() and common != common.parent and not is_sensitive_path(common): + buckets.append((common, unmounted)) + return buckets + + fallback: dict[Path, list[MylarPathException]] = {} + for exception in unmounted: + fallback.setdefault(_problem_root(exception), []).append(exception) + buckets.extend(fallback.items()) + return buckets + + +def _group_path_problems( + exceptions: list[MylarPathException], +) -> list[MylarPathProblemGroup]: + grouped: dict[tuple[MylarPathOutcome, str], list[MylarPathException]] = {} + outside_root: list[MylarPathException] = [] + for exception in exceptions: + if exception.outcome == "outside_root": + outside_root.append(exception) + continue + root = _problem_root(exception) + grouped.setdefault((exception.outcome, str(root)), []).append(exception) + for root, entries in _outside_root_buckets(outside_root): + grouped.setdefault(("outside_root", str(root)), []).extend(entries) + + results: list[MylarPathProblemGroup] = [] + for (outcome, root_path), entries in sorted(grouped.items(), key=lambda item: item[0]): + series_count = len({entry.series_id for entry in entries}) + location_count = len(entries) + if outcome == "outside_root": + reason = ( + f"{series_count} series use {root_path}, which isn't registered as a library root." + ) + suggested_action = ( + "Register this path for existing files, then Pullbox will analyze the paths again." + ) + root = Path(root_path) + can_register = ( + root.is_dir() and _location_is_readable(root) and not is_sensitive_path(root) + ) + else: + reason = entries[0].reason + suggested_action = entries[0].suggested_action + can_register = False + results.append( + MylarPathProblemGroup( + root_path=root_path, + outcome=outcome, + series_count=series_count, + location_count=location_count, + reason=reason, + suggested_action=suggested_action, + can_register_reference_root=can_register, + ) + ) + return results + + +def _attention_key(*parts: object) -> str: + """Return a stable opaque identity for one preview finding or action.""" + payload = json.dumps(parts, ensure_ascii=True, sort_keys=True, default=str) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _attention_fingerprint(items: list[MylarPathAttentionItem]) -> str | None: + if not items: + return None + return _attention_key("mylar_path_attention", [item.key for item in items]) + + +def add_report_unavailable_attention( + preview: MylarPathPreviewResponse, + report_directory: str | Path, +) -> MylarPathPreviewResponse: + """Explain a nonblocking report-storage failure through the Step 1 action contract.""" + path = str(report_directory) + key = _attention_key("report_unavailable", path) + item = MylarPathAttentionItem( + key=key, + code="report_unavailable", + blocks_import=False, + reason="Pullbox could not save the full Mylar path report.", + suggested_action=( + "You can continue, but only the first 25 path exceptions are available until " + "report storage is restored." + ), + root_path=path, + details=MylarPathAttentionDetails( + title="Restore the full path report", + series_count=0, + location_count=preview.exception_count, + known_paths=[path], + steps=[ + f"Confirm that the Pullbox data volume containing {path} has free space.", + f"Give Pullbox permission to create and replace files inside {path}.", + "If storage is healthy and the report is still too large, split the migration " + "into smaller source databases.", + "Return to Pullbox and recheck the import issues.", + ], + ), + ) + items = [*preview.attention_items, item] + return preview.model_copy( + update={ + "attention_items": items, + "attention_fingerprint": _attention_fingerprint(items), + } + ) + + +def _path_is_under(path: str, root: str) -> bool: + try: + return Path(path).absolute().is_relative_to(Path(root).expanduser().absolute()) + except (OSError, RuntimeError, ValueError): + return False + + +def _known_paths(root_path: str, entries: list[MylarPathException]) -> list[str]: + paths = [root_path] + for entry in entries: + paths.extend((entry.stored_path, entry.attempted_path)) + return list(dict.fromkeys(path for path in paths if path))[:6] + + +def _path_attention_steps( + outcome: MylarPathOutcome, + root_path: str, + *, + can_continue: bool, +) -> list[str]: + if outcome in {"missing", "mapped_missing"}: + return [ + f"Check whether {root_path} was renamed, moved, or deleted after Mylar recorded it.", + "If the content still exists, correct its recorded Mylar location or path mapping.", + "Return to Pullbox and recheck the import issues.", + ] + if outcome == "unmapped": + return [ + f"Make the source available inside the Pullbox container at {root_path}, " + "or map its Mylar prefix to the correct container-visible path.", + "Choose the host-side source folder in your container configuration; " + "Pullbox cannot infer that host path.", + "Register the container-visible path for existing files, then recheck " + "the import issues.", + ] + if outcome == "unreadable": + return [ + f"Give the Pullbox container read and directory-traverse access to {root_path}.", + "If this is a Docker mount, confirm the mount is not hidden by another " + "overlapping mount.", + "Recheck the import issues after access is restored.", + ] + if outcome == "ambiguous": + return [ + "Review the configured library roots and Mylar mappings that can both " + f"claim {root_path}.", + "Keep one unambiguous container-visible root or add one explicit " + "non-overlapping mapping.", + "Recheck the import issues after removing the overlap.", + ] + if outcome == "outside_root": + return [ + f"Confirm that {root_path} is the intended container-visible source root.", + "Register only that source root for existing files; do not expose the " + "container filesystem root.", + "Recheck the import issues after the root is registered.", + ] + if outcome == "invalid": + return [ + f"Correct the unsafe or malformed path recorded beneath {root_path} in Mylar.", + "Remove traversal components, invalid path text, or symlink loops instead " + "of broadening Pullbox access.", + "Recheck the import issues after the stored path is corrected.", + ] + if can_continue: + return ["Resolve this finding for the current import, then continue to the scan."] + return ["Correct the source location, then recheck the import issues."] + + +def _path_attention_items( + exceptions: list[MylarPathException], + problem_groups: list[MylarPathProblemGroup], + *, + can_continue_with_unresolved: bool, +) -> list[MylarPathAttentionItem]: + items: list[MylarPathAttentionItem] = [] + for group in problem_groups: + entries = [ + entry + for entry in exceptions + if entry.outcome == group.outcome + and ( + ( + group.outcome == "outside_root" + and _path_is_under(entry.stored_path, group.root_path) + ) + or ( + group.outcome != "outside_root" and str(_problem_root(entry)) == group.root_path + ) + ) + ] + unresolved = group.outcome in {"missing", "mapped_missing", "unmapped"} + blocks_import = not (unresolved and can_continue_with_unresolved) + action_kind: Literal["acknowledge_unavailable", "register_reference_root"] | None = None + if group.can_register_reference_root: + action_kind = "register_reference_root" + elif unresolved and can_continue_with_unresolved: + action_kind = "acknowledge_unavailable" + key = _attention_key( + "path_group", + group.outcome, + group.root_path, + sorted((entry.series_id, entry.stored_path, entry.attempted_path) for entry in entries), + ) + action = ( + MylarPathAttentionAction( + kind=action_kind, + fingerprint=_attention_key("action", action_kind, key), + root_path=group.root_path, + ) + if action_kind is not None + else None + ) + title = { + "missing": "Review missing Mylar locations", + "mapped_missing": "Review missing mapped locations", + "unmapped": "Connect an unavailable Mylar source", + "unreadable": "Restore read access", + "ambiguous": "Choose one source path", + "outside_root": "Register the existing library source", + "invalid": "Correct unsafe Mylar paths", + }.get(group.outcome, "Review this Mylar path issue") + items.append( + MylarPathAttentionItem( + key=key, + code=group.outcome, + blocks_import=blocks_import, + reason=group.reason, + suggested_action=group.suggested_action, + root_path=group.root_path, + action=action, + details=MylarPathAttentionDetails( + title=title, + series_count=group.series_count, + location_count=group.location_count, + known_paths=_known_paths(group.root_path, entries), + steps=_path_attention_steps( + group.outcome, + group.root_path, + can_continue=can_continue_with_unresolved, + ), + ), + ) + ) + return items + + +def _mapping_attention_item( + state: _MappingState, + reason: str, +) -> MylarPathAttentionItem: + code = reason + key = _attention_key("mapping", state.stored_prefix, state.pullbox_prefix, reason) + action: MylarPathAttentionAction | None = None + if reason == "mapping_does_not_improve_coverage": + action = MylarPathAttentionAction( + kind="remove_ineffective_mapping", + fingerprint=_attention_key("action", "remove_ineffective_mapping", key), + stored_prefix=state.stored_prefix, + pullbox_prefix=state.pullbox_prefix, + ) + elif reason == "mapping_target_outside_enabled_root": + target = Path(state.pullbox_prefix) + if target.is_dir() and _location_is_readable(target) and not is_sensitive_path(target): + action = MylarPathAttentionAction( + kind="register_reference_root", + fingerprint=_attention_key("action", "register_reference_root", key), + root_path=state.pullbox_prefix, + ) + labels = { + "mapping_does_not_improve_coverage": ( + "This path mapping does not match any Mylar locations.", + "Remove the unused mapping, then let Pullbox analyze the remaining paths again.", + "Remove an unused path mapping", + ), + "mapping_target_outside_enabled_root": ( + "This mapping points outside every enabled existing-file library root.", + "Register the mapped container path for existing files or choose another mapped path.", + "Register the mapped source", + ), + "mapping_target_unsafe": ( + "This mapping points to a protected or unsafe location.", + "Choose a narrower container-visible comic folder instead of a system location.", + "Choose a safe mapped folder", + ), + "mapping_target_unavailable": ( + "This mapping points to a location Pullbox cannot find.", + "Correct the container-visible target or add the missing Docker mount.", + "Make the mapped folder available", + ), + "mapping_target_unreadable": ( + "This mapping points to a folder Pullbox cannot read.", + "Restore read and directory-traverse access, then recheck the import issues.", + "Restore mapped-folder access", + ), + "mapping_target_root_ambiguous": ( + "More than one configured root can claim this mapping target.", + "Remove the overlapping root or map to one unambiguous container path.", + "Remove the root overlap", + ), + } + reason_text, suggested_action, title = labels.get( + reason, + ( + "This Mylar path mapping needs attention.", + "Review the mapping and recheck the import issues.", + "Review this path mapping", + ), + ) + return MylarPathAttentionItem( + key=key, + code=code, + blocks_import=True, + reason=reason_text, + suggested_action=suggested_action, + root_path=state.pullbox_prefix, + action=action, + details=MylarPathAttentionDetails( + title=title, + series_count=0, + location_count=state.counts.locations, + known_paths=[state.stored_prefix, state.pullbox_prefix], + steps=[ + f"Review the Mylar prefix {state.stored_prefix}.", + f"Verify that its Pullbox container path is {state.pullbox_prefix}.", + "Correct or remove the mapping, then recheck the import issues.", + ], + ), + ) + + +def _build_attention_items( + *, + exceptions: list[MylarPathException], + can_continue_with_unresolved: bool, + unavailable_roots: list[LibraryRoot], + available_roots: list[_RootBoundary], + mapping_states: list[_MappingState], + warnings: list[str], + partial: bool, +) -> list[MylarPathAttentionItem]: + root_exceptions = { + id(entry) + for root in unavailable_roots + for entry in exceptions + if _path_is_under(entry.stored_path, root.path) + } + remaining_exceptions = [entry for entry in exceptions if id(entry) not in root_exceptions] + items = _path_attention_items( + remaining_exceptions, + _group_path_problems(remaining_exceptions), + can_continue_with_unresolved=can_continue_with_unresolved, + ) + for root in unavailable_roots: + related = [entry for entry in exceptions if _path_is_under(entry.stored_path, root.path)] + key = _attention_key("library_root_unavailable", root.id, root.path) + items.insert( + 0, + MylarPathAttentionItem( + key=key, + code="library_root_unavailable", + blocks_import=True, + reason=f"The configured library root {root.path} is unavailable or unreadable.", + suggested_action=( + "Restore its container mount or read access, then recheck the import issues." + ), + root_path=root.path, + details=MylarPathAttentionDetails( + title=f"Restore access to {root.name}", + series_count=len({entry.series_id for entry in related}), + location_count=len(related), + known_paths=[root.path], + steps=[ + f"Confirm that {root.path} exists inside the Pullbox container.", + "If it is a Docker mount, choose the correct host-side source " + "folder; Pullbox cannot infer that host path.", + f"Give Pullbox read and directory-traverse access to {root.path}, " + "then recheck the import issues.", + ], + ), + ), + ) + for state in mapping_states: + items.extend(_mapping_attention_item(state, reason) for reason in state.blockers) + root_paths = [str(root.lexical) for root in available_roots] + aggregate_findings = { + "library_root_alias": ( + "Two library roots point to the same physical folder.", + "Disable or remove the duplicate root before continuing.", + "Remove a duplicate library root", + ), + "nested_library_roots_ambiguous": ( + "Configured library roots overlap or are nested.", + "Keep one clear root boundary for these files before continuing.", + "Remove overlapping library roots", + ), + } + for code, (reason, suggested_action, title) in aggregate_findings.items(): + if code not in warnings: + continue + key = _attention_key(code, root_paths) + items.append( + MylarPathAttentionItem( + key=key, + code=code, + blocks_import=True, + reason=reason, + suggested_action=suggested_action, + details=MylarPathAttentionDetails( + title=title, + series_count=0, + location_count=0, + known_paths=root_paths[:6], + steps=[ + "Open Media Management and compare the configured library-root " + "paths shown here.", + "Disable or remove the duplicate or nested root without changing " + "the source files.", + "Return here and recheck the import issues.", + ], + ), + ) + ) + if partial: + key = _attention_key("partial_preview", _MAX_LOCATIONS) + items.append( + MylarPathAttentionItem( + key=key, + code="partial_preview", + blocks_import=True, + reason="The Mylar path check reached its safe inspection limit.", + suggested_action="Reduce the selected database scope before starting this import.", + details=MylarPathAttentionDetails( + title="Path analysis is incomplete", + series_count=0, + location_count=_MAX_LOCATIONS, + steps=[ + f"This preview inspected the first {_MAX_LOCATIONS:,} stored locations.", + "Use a smaller Mylar database or split the migration into bounded groups.", + "Recheck the import issues after reducing the source scope.", + ], + ), + ) + ) + return list({item.key: item for item in items}.values()) + + +def _path_exception( + entry: _Location, outcome: MylarPathOutcome, attempted: str +) -> MylarPathException: + explanations = { + "missing": ( + "The stored location is missing inside a visible library root.", + "Check for a renamed or deleted folder. Correct the Mylar path or continue " + "and review this unavailable source later.", + ), + "mapped_missing": ( + "The mapping target is visible, but this translated location is missing.", + "Check the translated path and mapping prefix. Correct it or acknowledge " + "the unavailable source.", + ), + "unmapped": ( + "The stored location is unavailable and no mapping resolves it.", + "Check the Docker mount. Map the Mylar prefix to the verified container path, " + "not the host path.", + ), + "unreadable": ( + "Pullbox cannot read or traverse this location.", + "Grant the container user read access to the source and traverse access " + "to its parent directories, then analyze again.", + ), + "ambiguous": ( + "More than one root or automatic mapping could claim this location.", + "Choose an explicit mapping and remove overlapping or duplicate roots, " + "then analyze again.", + ), + "outside_root": ( + "This location escapes its mapped root or is outside an allowed reference root.", + "Check symlinks and root boundaries. Configure the correct permitted root; " + "do not map the whole filesystem.", + ), + "invalid": ( + "This location is not a safe absolute path or could not be resolved safely.", + "Correct the stored path, including traversal components, invalid characters, " + "or symlink loops, then analyze again.", + ), + } + reason, action = explanations[outcome] + return MylarPathException( + series_id=entry.series_id[:255], + series_name=entry.series_name[:1000], + stored_path=entry.path[:4096], + attempted_path=attempted[:4096], + outcome=outcome, + reason=reason, + suggested_action=action, + ) + + +def _append_example( + examples: list[MylarPathExample], + relative: str, + outcome: MylarPathOutcome, +) -> None: + if len(examples) >= _MAX_EXAMPLES: + return + examples.append(MylarPathExample(relative_path=relative, outcome=outcome)) + + +def _safe_relative(path: Path, root: Path) -> str: + try: + return str(path.relative_to(root)) + except ValueError: + return path.name + + +def _containing_roots( + lexical_path: Path, + resolved_path: Path, + roots: Iterable[_RootBoundary], +) -> list[_RootBoundary]: + return [ + root + for root in roots + if lexical_path.is_relative_to(root.lexical) and resolved_path.is_relative_to(root.resolved) + ] + + +def _paths_nested(left: Path, right: Path) -> bool: + return left != right and (left.is_relative_to(right) or right.is_relative_to(left)) + + +def _stored_prefixes(location: Path) -> list[Path]: + parts = location.parts + if len(parts) < 3 or parts[0] != "/": + return [] + segments = parts[1:] + return [Path("/", *segments[:index]) for index in range(len(segments) - 1, 0, -1)] + + +def _sqlite_table_columns(connection: sqlite3.Connection, table: str) -> set[str]: + """Return columns for one fixed Mylar table without interpolating user input.""" + statements = { + "comics": "PRAGMA table_info(comics)", + "issues": "PRAGMA table_info(issues)", + } + statement = statements.get(table) + if statement is None: + raise ValueError("Unsupported Mylar path-inventory table") + return {str(row[1]) for row in connection.execute(statement)} + + +def _supported_location_kind(path: Path) -> bool: + """Return whether a Mylar location resolves to a series directory or issue file.""" + return path.is_dir() or path.is_file() + + +def _location_is_readable(path: Path) -> bool: + """Require read access for files and read/traverse access for directories.""" + required_access = os.R_OK | os.X_OK if path.is_dir() else os.R_OK + return os.access(path, required_access) diff --git a/src/pullbox/services/import_mylar3_path_reports.py b/src/pullbox/services/import_mylar3_path_reports.py new file mode 100644 index 00000000..4b2b7a79 --- /dev/null +++ b/src/pullbox/services/import_mylar3_path_reports.py @@ -0,0 +1,128 @@ +"""Bounded, private preflight reports, independent of import-job creation.""" + +from __future__ import annotations + +import json +import os +import re +import tempfile +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from pullbox.schemas.import_mylar3_path_preflight import MylarPathPreviewResponse + +PAGE_SIZE = 25 +_MAX_REPORT_BYTES = 32 * 1024 * 1024 +_MAX_REPORTS = 5 +_MAX_AGE_SECONDS = 24 * 60 * 60 +_REPORT_ID = re.compile(r"[a-f0-9]{32}") + + +def _directory() -> Path: + from pullbox.config import get_settings + + return get_settings().data_dir / "diagnostics" / "mylar-preflight" + + +def report_directory() -> Path: + """Return the private directory used for retained Mylar preflight reports.""" + return _directory() + + +def save_report(preview: MylarPathPreviewResponse, source_path: str) -> str: + """Atomically retain recent reports, never opening the source database for writing.""" + report_id = uuid4().hex + report = preview.model_dump(mode="json") + report.update( + report_id=report_id, source_path=source_path, captured_at=datetime.now(UTC).isoformat() + ) + content = json.dumps(report, ensure_ascii=True).encode("utf-8") + if len(content) > _MAX_REPORT_BYTES: + raise ValueError("Mylar path report exceeds the diagnostic size limit") + directory = report_directory() + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile(dir=directory, suffix=".tmp", delete=False) as output: + temporary = Path(output.name) + output.write(content) + os.replace(temporary, directory / f"{report_id}.json") + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + reports = _report_paths() + for old in reports[_MAX_REPORTS:]: + old.unlink(missing_ok=True) + return report_id + + +def _report_paths() -> list[Path]: + return sorted( + ( + path + for path in report_directory().glob("*.json") + if _REPORT_ID.fullmatch(path.stem) and not path.is_symlink() + ), + key=lambda path: path.stat().st_mtime, + reverse=True, + ) + + +def load_report(report_id: str) -> dict[str, Any]: + """Load only an application-generated report, never a caller-supplied path.""" + if not _REPORT_ID.fullmatch(report_id): + raise FileNotFoundError("Invalid preflight report") + path = report_directory() / f"{report_id}.json" + if path.is_symlink(): + raise FileNotFoundError("Invalid preflight report") + stat = path.stat() + if stat.st_size > _MAX_REPORT_BYTES or time.time() - stat.st_mtime > _MAX_AGE_SECONDS: + raise FileNotFoundError("Preflight report expired") + with path.open("rb") as source: + content = source.read(_MAX_REPORT_BYTES + 1) + if len(content) > _MAX_REPORT_BYTES: + raise ValueError("Preflight report is too large") + report: dict[str, Any] = json.loads(content) + if not isinstance(report, dict) or report.get("report_id") != report_id: + raise ValueError("Preflight report identity changed") + if not isinstance(report.get("exceptions"), list): + raise ValueError("Preflight report exceptions are invalid") + MylarPathPreviewResponse.model_validate(report) + return report + + +def report_page(report_id: str, page: int, search: str) -> dict[str, Any]: + report = load_report(report_id) + query = search.strip().casefold() + items = report["exceptions"] + if query: + items = [ + item for item in items if any(query in str(value).casefold() for value in item.values()) + ] + total = len(items) + page = min(page, max(1, (total + PAGE_SIZE - 1) // PAGE_SIZE)) + return { + "items": items[(page - 1) * PAGE_SIZE : page * PAGE_SIZE], + "total": total, + "page": page, + "page_size": PAGE_SIZE, + } + + +def latest_report() -> dict[str, Any]: + """Include the most recent pre-job evidence in an operator-requested diagnostic ZIP.""" + try: + for path in _report_paths(): + try: + return load_report(path.stem) + except (OSError, ValueError): + continue + except OSError: + pass + return { + "status": "not_available", + "message": "No recent Mylar path preflight report is available.", + } diff --git a/src/pullbox/services/import_mylar3_path_validation.py b/src/pullbox/services/import_mylar3_path_validation.py new file mode 100644 index 00000000..1849dbef --- /dev/null +++ b/src/pullbox/services/import_mylar3_path_validation.py @@ -0,0 +1,92 @@ +"""Server-side validation for confirmed Mylar path mappings.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING + +from sqlalchemy import select + +from pullbox.core.exceptions import ValidationError +from pullbox.core.filesystem_policy import is_sensitive_path +from pullbox.models.import_job import ImportFileHandlingMode +from pullbox.models.library import LibraryRoot + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +async def validate_mylar3_path_map_targets( + session: AsyncSession, + path_map: dict[str, str], + *, + file_handling_mode: ImportFileHandlingMode, +) -> None: + """Validate readable Mylar source mappings for the selected ownership mode.""" + if not path_map: + return + + available_roots: list[tuple[Path, Path]] = [] + if file_handling_mode == ImportFileHandlingMode.IN_PLACE: + roots = list( + ( + await session.execute( + select(LibraryRoot) + .where( + LibraryRoot.enabled.is_(True), + LibraryRoot.allow_referenced_registrations.is_(True), + ) + .order_by(LibraryRoot.id.asc()) + ) + ) + .scalars() + .all() + ) + available_roots = _available_root_boundaries(roots) + for visible_prefix in path_map.values(): + target = Path(visible_prefix) + if not target.is_absolute() or ".." in target.parts or target == Path("/"): + raise ValidationError( + "Each Pullbox-visible Mylar mapping path must be a safe absolute directory." + ) + try: + lexical_target = target + # This operator-confirmed source is bounded above, resolved before + # containment checks, and screened against sensitive paths below. + # Managed-copy imports intentionally allow external source roots. + # codeql[py/path-injection] + resolved_target = target.resolve(strict=True) + except (OSError, RuntimeError, ValueError) as exc: + raise ValidationError( + "Each Pullbox-visible Mylar mapping path must be an available directory." + ) from exc + if is_sensitive_path(resolved_target): + raise ValidationError( + "Each Pullbox-visible Mylar mapping path must be a safe source directory." + ) + if not resolved_target.is_dir() or not os.access(resolved_target, os.R_OK | os.X_OK): + raise ValidationError( + "Each Pullbox-visible Mylar mapping path must be a readable directory." + ) + if file_handling_mode == ImportFileHandlingMode.IN_PLACE and not any( + lexical_target.is_relative_to(lexical_root) + and resolved_target.is_relative_to(resolved_root) + for lexical_root, resolved_root in available_roots + ): + raise ValidationError( + "Each Pullbox-visible Mylar mapping path must be inside an enabled library root." + ) + + +def _available_root_boundaries(roots: list[LibraryRoot]) -> list[tuple[Path, Path]]: + boundaries: list[tuple[Path, Path]] = [] + for root in roots: + try: + lexical_root = Path(root.path).expanduser().absolute() + resolved_root = Path(root.path).expanduser().resolve(strict=True) + except (OSError, RuntimeError, ValueError): + continue + if resolved_root.is_dir() and os.access(resolved_root, os.R_OK): + boundaries.append((lexical_root, resolved_root)) + return boundaries diff --git a/src/pullbox/services/import_mylar3_paths.py b/src/pullbox/services/import_mylar3_paths.py index 73622fd9..22cc4c7c 100644 --- a/src/pullbox/services/import_mylar3_paths.py +++ b/src/pullbox/services/import_mylar3_paths.py @@ -29,6 +29,7 @@ def auto_detect_mylar3_path_map( rows = conn.execute( "SELECT ComicLocation FROM comics " "WHERE ComicLocation IS NOT NULL AND ComicLocation != '' " + "ORDER BY ComicLocation " "LIMIT 50" ).fetchall() conn.close() @@ -38,22 +39,88 @@ def auto_detect_mylar3_path_map( if not rows: return None + proposals: dict[str, str] = {} + proposal_locations: dict[str, str] = {} for row in rows: comic_location: str = row[0] + if _identity_location_is_available(comic_location): + continue detected = _detect_path_map_for_location(db_path, comic_location) if detected is None: continue container_prefix, host_path = detected - path_map = {container_prefix: host_path} + existing_host_path = proposals.get(container_prefix) + if existing_host_path is not None and not _same_path( + Path(existing_host_path), Path(host_path) + ): + log.debug( + "mylar3_path_map_auto_detection_ambiguous", + container_prefix=container_prefix, + first_host_path=existing_host_path, + second_host_path=host_path, + ) + return None + proposals[container_prefix] = host_path + proposal_locations.setdefault(container_prefix, comic_location) + + if _has_conflicting_overlapping_mappings(proposals): + log.debug("mylar3_path_map_auto_detection_overlapping") + return None + + for container_prefix, host_path in sorted(proposals.items()): log.debug( "mylar3_path_map_auto_detected", container_prefix=container_prefix, host_path=host_path, - comic_location=comic_location, + comic_location=proposal_locations[container_prefix], ) - return path_map + return proposals or None - return None + +def _identity_location_is_available(comic_location: str) -> bool: + """Return whether Mylar's stored directory is already usable unchanged.""" + location_path = Path(comic_location) + if not location_path.is_absolute() or ".." in location_path.parts: + return False + try: + return location_path.resolve(strict=True).is_dir() + except (OSError, RuntimeError, ValueError): + return False + + +def _has_conflicting_overlapping_mappings(path_map: dict[str, str]) -> bool: + """Reject nested stored prefixes whose translated roots disagree.""" + mappings = [(Path(source), Path(target)) for source, target in path_map.items()] + for index, (left_source, left_target) in enumerate(mappings): + for right_source, right_target in mappings[index + 1 :]: + if _nested_mapping_conflicts( + left_source, + left_target, + right_source, + right_target, + ) or _nested_mapping_conflicts( + right_source, + right_target, + left_source, + left_target, + ): + return True + return False + + +def _nested_mapping_conflicts( + parent_source: Path, + parent_target: Path, + child_source: Path, + child_target: Path, +) -> bool: + try: + relative = child_source.relative_to(parent_source) + except ValueError: + return False + if not relative.parts: + return not _same_path(parent_target, child_target) + return not _same_path(parent_target / relative, child_target) def _detect_path_map_for_location( @@ -68,6 +135,7 @@ def _detect_path_map_for_location( for container_prefix in _container_prefixes(location_path): search_name = Path(container_prefix).name relative_location = location_path.relative_to(container_prefix) + candidates: dict[str, Path] = {} for search_dir in _path_map_search_dirs(db_path): candidate_root = search_dir / search_name if not candidate_root.is_dir(): @@ -77,7 +145,12 @@ def _detect_path_map_for_location( translated_location = candidate_root / relative_location if not translated_location.is_dir(): continue + candidates[str(candidate_root.resolve(strict=False))] = candidate_root + if len(candidates) == 1: + candidate_root = next(iter(candidates.values())) return container_prefix, str(candidate_root) + if len(candidates) > 1: + return None return None diff --git a/src/pullbox/services/import_mylar_scan_progress.py b/src/pullbox/services/import_mylar_scan_progress.py new file mode 100644 index 00000000..55692f29 --- /dev/null +++ b/src/pullbox/services/import_mylar_scan_progress.py @@ -0,0 +1,110 @@ +"""Bounded, durable progress while preparing Mylar source pages.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +from pullbox.models.import_job import ImportJobStatus +from pullbox.schemas.import_job import ImportProgressEvent +from pullbox.services import import_progress_runtime +from pullbox.services.import_counters import job_stats +from pullbox.services.import_workflow_state import ( + SCAN_PROGRESS_MATERIALIZE_END, + SCAN_PROGRESS_MATERIALIZE_START, + emit_progress, +) + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.models.import_job import ImportJob + + +@dataclass +class MylarScanProgress: + session: AsyncSession + job: ImportJob + source_total: int + cancellation_check: Callable[[], Awaitable[None]] + callback: Callable[[ImportProgressEvent], Awaitable[None]] | None = None + source_completed: int = 0 + source_page_end: int = 0 + last_report_at: float = 0.0 + checked_files: int = 0 + batch_files: int = 0 + work_started_at: datetime = field(default_factory=lambda: datetime.now(UTC), repr=False) + estimated_seconds_remaining: int | None = field(default=None, init=False) + + async def report_safety(self, completed: int, total: int, file_path: str) -> None: + self.checked_files = completed + self.batch_files = total + now = time.monotonic() + if completed not in {0, total} and now - self.last_report_at < 1.0: + return + self.last_report_at = now + await self.cancellation_check() + message = f"Checking Mylar files: {completed}/{total} in this batch." + await self._publish(completed, total, file_path, message) + + async def _publish(self, completed: int, total: int, file_path: str, message: str) -> None: + fraction = completed / total if total else 0.0 + source_done = ( + self.source_completed + (self.source_page_end - self.source_completed) * fraction + ) + progress = SCAN_PROGRESS_MATERIALIZE_START + if self.source_total: + progress += int( + (SCAN_PROGRESS_MATERIALIZE_END - SCAN_PROGRESS_MATERIALIZE_START) + * min(source_done / self.source_total, 1.0) + ) + if self.source_total: + message += f" {self.source_completed}/{self.source_total} source series saved." + await emit_progress( + self.session, + self.job, + ImportProgressEvent( + job_id=self.job.id, + status=ImportJobStatus.SCANNING, + phase="scanning", + progress=progress, + message=message, + estimated_seconds_remaining=self.estimated_seconds_remaining, + current_item_kind="scan", + current_item_stage="scanning", + current_item_stage_label="Checking Mylar file batch", + current_item_progress_pct=int(fraction * 100), + current_item_detail=Path(file_path).name if file_path else message, + **job_stats(self.job), + ), + self.callback, + ) + + async def checkpoint_page(self) -> None: + self.source_completed = self.source_page_end + if self.source_total > 0 and self.source_completed >= self.source_total: + self.estimated_seconds_remaining = 0 + else: + self.estimated_seconds_remaining = ( + import_progress_runtime.estimate_remaining_work_seconds( + self.work_started_at, + completed_units=self.source_completed, + total_units=self.source_total, + ) + ) + # A single source page can emit extra Annual cohorts. Measure overall + # progress using source rows, not the number of resulting review groups. + await self._publish( + self.checked_files, + self.batch_files, + "", + ( + f"Prepared {self.job.series_found} series and " + f"{self.job.scan_total_files} file records." + ), + ) diff --git a/src/pullbox/services/import_operation_progress.py b/src/pullbox/services/import_operation_progress.py index 518fe06c..dbd0c5b3 100644 --- a/src/pullbox/services/import_operation_progress.py +++ b/src/pullbox/services/import_operation_progress.py @@ -140,7 +140,12 @@ def build_import_operation_update( event: ImportProgressEvent, ) -> OperationProgressUpdate: """Map an import event into the shared durable projection contract.""" - source_label = "Mylar import" if job.source_type is ImportSourceType.MYLAR3 else "Folder import" + if dict(job.progress_snapshot or {}).get("clean_library_adoption") is True: + source_label = "Library organization" + else: + source_label = ( + "Mylar import" if job.source_type is ImportSourceType.MYLAR3 else "Folder import" + ) has_failures = bool((event.series_failed or job.series_failed or 0) > 0) attention_required = event.status in { ImportJobStatus.FAILED, diff --git a/src/pullbox/services/import_orphan_recovery_context.py b/src/pullbox/services/import_orphan_recovery_context.py index 8c4d8ae5..2aa39d54 100644 --- a/src/pullbox/services/import_orphan_recovery_context.py +++ b/src/pullbox/services/import_orphan_recovery_context.py @@ -8,6 +8,7 @@ from sqlalchemy.orm import joinedload from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.issue_numbers import format_issue_number from pullbox.models.import_job import ( ImportedFile, ImportedFileStatus, @@ -18,7 +19,7 @@ ) from pullbox.models.issue import Issue from pullbox.models.library import LibraryRoot -from pullbox.services.import_orphans import is_active_orphan_row +from pullbox.services.import_orphans import is_active_orphan_row, requires_orphan_issue_decision if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -39,7 +40,7 @@ async def load_orphan_recovery_item( if job is None: raise NotFoundError("ImportJob", item.import_job_id) if job.status != ImportJobStatus.COMPLETED: - raise ValidationError("Unmatched recovery is only available for completed imports.") + raise ValidationError("Import follow-up is only available for completed imports.") has_live_issue_recovery = False if item.status == ImportSeriesStatus.IMPORTED: has_live_issue_recovery = bool( @@ -160,7 +161,7 @@ def _build_recovery_file_rows( for option in issue_options: issue_cv_id = int(option["issue_cv_id"]) issue_number = float(option["issue_number"]) - label = f"#{issue_number:g}" + label = f"#{format_issue_number(issue_number)}" if option.get("title"): label = f"{label} - {option['title']}" issue_label_by_cv_id[issue_cv_id] = label @@ -185,10 +186,7 @@ def _build_recovery_file_rows( ): suggested_issue_cv_id = issue_number_to_cv_ids[imp_file.parsed_issue_number][0] - decision_locked = imp_file.status in { - ImportedFileStatus.IMPORTED, - ImportedFileStatus.SKIPPED, - } + decision_locked = not requires_orphan_issue_decision(imp_file) if decision_locked: files_completed += 1 else: diff --git a/src/pullbox/services/import_orphans.py b/src/pullbox/services/import_orphans.py index a0dfeaa1..9dce5308 100644 --- a/src/pullbox/services/import_orphans.py +++ b/src/pullbox/services/import_orphans.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import UTC, datetime from typing import TYPE_CHECKING, Any, Protocol from sqlalchemy import and_, or_ @@ -9,6 +10,7 @@ from sqlalchemy import select as sa_select from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.release_parser import parse_release_title from pullbox.models.import_job import ( ImportedFile, ImportedFileStatus, @@ -17,18 +19,30 @@ ImportJobStatus, ImportSeriesStatus, ) +from pullbox.models.issue import Issue, IssueType from pullbox.models.series import Series from pullbox.services.import_counters import recompute_file_counters, recompute_series_counters +from pullbox.services.import_file_issue_signals import ( + candidate_issue_number, + candidate_issue_number_text, + comicinfo_issue_number, + filename_issue_number, + volume_issue_number, +) from pullbox.services.import_file_resolution import load_issue_lookup_for_series +from pullbox.services.import_job_actions import build_series_created_action_payload +from pullbox.services.import_job_execution_items import ensure_target_issue_summary_for_import_file +from pullbox.services.import_retry_helpers import require_retained_import_destination +from pullbox.services.import_review_recheck import prepare_retryable_failed_sources_for_retry +from pullbox.services.import_terminal_recovery import allows_terminal_import_recovery if TYPE_CHECKING: - from collections.abc import Awaitable, Callable + from collections.abc import Awaitable, Callable, Sequence from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql import Select from sqlalchemy.sql.elements import ColumnElement - from pullbox.models.issue import Issue from pullbox.providers.base import SeriesMetadata from pullbox.schemas.import_job import OrphanRecoveryDecision, RecoverOrphanRequest @@ -79,6 +93,231 @@ async def add_from_comicvine( ImportSeriesStatus.NO_MATCH, ImportSeriesStatus.RECOVERY_PENDING, ) +_UNRESOLVED_TARGET_ERROR = "Could not resolve to a library issue" + + +def requires_orphan_issue_decision(file: ImportedFile) -> bool: + """Return whether Follow-up should ask for an issue assignment or skip.""" + return file.status in { + ImportedFileStatus.PENDING, + ImportedFileStatus.MATCHED, + ImportedFileStatus.CONFIRMED, + ImportedFileStatus.CONFLICT, + ImportedFileStatus.NO_MATCH, + } + + +def _saved_target_provider_ids(file: ImportedFile) -> set[int]: + diagnostics = dict(file.diagnostics or {}) + raw_summary = diagnostics.get("target_issue_summary") + summary = raw_summary if isinstance(raw_summary, dict) else {} + raw_values: tuple[object, ...] = ( + file.matched_issue_cv_id, + file.comicvine_issue_id, + summary.get("provider_id"), + ) + provider_ids: set[int] = set() + for raw_value in raw_values: + if raw_value is None: + continue + try: + provider_ids.add(int(str(raw_value))) + except ValueError: + provider_ids.add(-1) + return provider_ids + + +def _source_issue_type_matches(file: ImportedFile, issue: Issue) -> bool: + diagnostics = dict(file.diagnostics or {}) + source_metadata = diagnostics.get("source_metadata") + filename_parse = ( + source_metadata.get("filename_parse") if isinstance(source_metadata, dict) else None + ) + raw_issue_type = diagnostics.get("source_issue_type") or ( + filename_parse.get("issue_type") if isinstance(filename_parse, dict) else None + ) + if raw_issue_type: + try: + return IssueType(str(raw_issue_type)) is issue.issue_type + except ValueError: + return False + parsed = parse_release_title(file.file_name or "") + return parsed is None or parsed.issue_type is issue.issue_type + + +def _failed_target_issue_number(file: ImportedFile) -> float | None: + """Return affirmative issue-number evidence without treating volume as issue.""" + exact_number = candidate_issue_number_text(file) + if exact_number is not None: + return candidate_issue_number(file) + filename_number = filename_issue_number(file) + if filename_number is not None: + return filename_number + if file.parsed_issue_number is not None and volume_issue_number(file) is None: + return file.parsed_issue_number + return comicinfo_issue_number(file) + + +async def _resolve_proven_failed_target( + session: AsyncSession, + item: ImportedSeries, + file: ImportedFile, + *, + cv_id_to_issue: dict[int, Issue], + exact_number_to_issue: dict[str, Issue], + number_to_issue: dict[float, Issue], +) -> Issue | None: + """Resolve only an exact target inside the already identified local series.""" + if item.series_id is None: + return None + if file.matched_issue_id is not None: + saved_issue = await session.get(Issue, file.matched_issue_id) + if saved_issue is not None and saved_issue.series_id == item.series_id: + return saved_issue + return None + + provider_ids = _saved_target_provider_ids(file) + if provider_ids: + if len(provider_ids) != 1: + return None + return cv_id_to_issue.get(next(iter(provider_ids))) + + exact_number = candidate_issue_number_text(file) + if exact_number is not None: + issue = exact_number_to_issue.get(exact_number) + else: + issue_number = _failed_target_issue_number(file) + issue = number_to_issue.get(issue_number) if issue_number is not None else None + return issue if issue is not None and _source_issue_type_matches(file, issue) else None + + +def _prepare_exact_target_retry(file: ImportedFile, issue: Issue) -> None: + diagnostics = dict(file.diagnostics or {}) + diagnostics["previous_import_error"] = file.error_message + diagnostics["completed_import_follow_up"] = { + "resolution": "exact_issue_target", + "resolved_at": datetime.now(UTC).isoformat(), + "source_preserved": True, + } + file.status = ImportedFileStatus.CONFIRMED + file.include_in_import = True + file.matched_issue_id = issue.id + file.matched_issue_cv_id = issue.comicvine_id + file.match_confidence = "high" + file.match_method = "completed_import_exact_target" + file.error_message = None + file.diagnostics = diagnostics + + +def _defer_target_to_follow_up(file: ImportedFile) -> None: + diagnostics = dict(file.diagnostics or {}) + diagnostics["previous_import_error"] = file.error_message + diagnostics["completed_import_follow_up"] = { + "resolution": "issue_decision_required", + "resolved_at": datetime.now(UTC).isoformat(), + "source_preserved": True, + } + file.status = ImportedFileStatus.NO_MATCH + file.include_in_import = False + file.matched_issue_id = None + file.error_message = "Choose the correct issue in Follow-up." + file.diagnostics = diagnostics + + +async def _prepare_terminal_follow_up( + session: AsyncSession, + job: ImportJob, +) -> tuple[set[int], set[int], int]: + """Repair legacy terminal outcomes before retrying only proven work.""" + affected_series_ids: set[int] = set() + retry_series_ids: set[int] = set() + target_rows = list( + ( + await session.execute( + sa_select(ImportedSeries, ImportedFile) + .join(ImportedFile, ImportedFile.import_series_id == ImportedSeries.id) + .where( + ImportedSeries.import_job_id == job.id, + ImportedFile.status == ImportedFileStatus.FAILED, + ImportedFile.error_message == _UNRESOLVED_TARGET_ERROR, + ) + .order_by(ImportedFile.id) + ) + ).all() + ) + issue_lookups: dict[int, tuple[dict[int, Issue], dict[str, Issue], dict[float, Issue]]] = {} + for item, file in target_rows: + if item.series_id is not None and item.series_id not in issue_lookups: + issue_lookups[item.series_id] = await load_issue_lookup_for_series( + session, + item.series_id, + ) + lookup = issue_lookups.get(item.series_id or -1, ({}, {}, {})) + issue = await _resolve_proven_failed_target( + session, + item, + file, + cv_id_to_issue=lookup[0], + exact_number_to_issue=lookup[1], + number_to_issue=lookup[2], + ) + if issue is None: + _defer_target_to_follow_up(file) + else: + _prepare_exact_target_retry(file, issue) + retry_series_ids.add(item.id) + affected_series_ids.add(item.id) + + imported_series_ids = set( + await session.scalars( + sa_select(ImportedFile.import_series_id) + .where( + ImportedFile.import_job_id == job.id, + ImportedFile.status == ImportedFileStatus.IMPORTED, + ) + .distinct() + ) + ) + failed_series = list( + await session.scalars( + sa_select(ImportedSeries).where( + ImportedSeries.import_job_id == job.id, + ImportedSeries.status == ImportSeriesStatus.FAILED, + ) + ) + ) + normalized_series_count = 0 + for item in failed_series: + previous_error = item.error_message + if item.id in imported_series_ids: + item.status = ImportSeriesStatus.IMPORTED + elif item.error_message == "No ComicVine ID available": + item.status = ImportSeriesStatus.NO_MATCH + item.selected_for_import = False + elif item.error_message == "No eligible files available for import" or ( + item.id in affected_series_ids + ): + item.status = ImportSeriesStatus.RECOVERY_PENDING + item.selected_for_import = False + else: + continue + item.error_message = None + item.diagnostics = { + **dict(item.diagnostics or {}), + "previous_series_error": previous_error, + "completed_import_follow_up": { + "resolution": "series_status_repaired", + "resolved_at": datetime.now(UTC).isoformat(), + "source_preserved": True, + }, + } + normalized_series_count += 1 + affected_series_ids.add(item.id) + + if affected_series_ids: + await recompute_file_counters(session, job, series_ids=sorted(affected_series_ids)) + await recompute_series_counters(session, job) + return affected_series_ids, retry_series_ids, normalized_series_count def _active_issue_recovery_clause() -> ColumnElement[bool]: @@ -103,7 +342,7 @@ def _active_orphan_clause() -> ColumnElement[bool]: def is_active_orphan_row(item: ImportedSeries | None) -> bool: - """Return True when a row should appear in the active Unmatched queue.""" + """Return True when a row should appear in active import Follow-up.""" return bool( item is not None and ( @@ -208,6 +447,7 @@ async def recover_orphan( progress_callback: ProgressCallback | None = None, ) -> dict[str, Any]: """Create/reuse the local series and import selected files for delayed recovery.""" + require_retained_import_destination(job) job_id = job.id item_id = item.id has_identified_series = ( @@ -222,11 +462,7 @@ async def recover_orphan( .order_by(ImportedFile.id.asc()) ) files = list(files_result.scalars().all()) - active_files = [ - imp_file - for imp_file in files - if imp_file.status not in {ImportedFileStatus.IMPORTED, ImportedFileStatus.SKIPPED} - ] + active_files = [imp_file for imp_file in files if requires_orphan_issue_decision(imp_file)] decision_by_id = {decision.imported_file_id: decision for decision in request.decisions} missing_ids = [imp_file.id for imp_file in active_files if imp_file.id not in decision_by_id] @@ -268,10 +504,14 @@ async def recover_orphan( job, phase="import", action_type="series_created", - payload={"series_id": series.id, "import_series_id": item.id}, + payload=await build_series_created_action_payload( + session, + series_id=series.id, + import_series_id=item.id, + ), ) - cv_id_to_issue, _ = await load_issue_lookup_for_series(session, series.id) + cv_id_to_issue, _, _ = await load_issue_lookup_for_series(session, series.id) apply_orphan_recovery_decisions( item=item, files=files, @@ -320,9 +560,9 @@ async def recover_orphan( return recovery_summary -def _orphaned_series_query() -> Select[tuple[ImportedSeries]]: +def _orphaned_series_query(*, job_id: int | None = None) -> Select[tuple[ImportedSeries]]: """Build the shared unresolved-orphan filter used by list and count queries.""" - return ( + query = ( sa_select(ImportedSeries) .join(ImportJob, ImportedSeries.import_job_id == ImportJob.id) .where( @@ -330,6 +570,9 @@ def _orphaned_series_query() -> Select[tuple[ImportedSeries]]: ImportJob.status == ImportJobStatus.COMPLETED, ) ) + if job_id is not None: + query = query.where(ImportedSeries.import_job_id == job_id) + return query async def get_orphaned_series( @@ -338,21 +581,16 @@ async def get_orphaned_series( page: int = 1, page_size: int = 25, sort: str = "file_count_desc", + job_id: int | None = None, ) -> tuple[list[ImportedSeries], int]: """Return paginated active orphaned import series from completed jobs.""" - count_q = ( - sa_select(sa_func.count()) - .select_from(ImportedSeries) - .join(ImportJob, ImportedSeries.import_job_id == ImportJob.id) - .where( - _active_orphan_clause(), - ImportJob.status == ImportJobStatus.COMPLETED, - ) + count_q = sa_select(sa_func.count()).select_from( + _orphaned_series_query(job_id=job_id).subquery() ) total_result = await session.execute(count_q) total = total_result.scalar() or 0 - query = _orphaned_series_query() + query = _orphaned_series_query(job_id=job_id) if sort == "series_name_asc": query = query.order_by(ImportedSeries.raw_series_name.asc()) elif sort == "date_found_desc": @@ -368,16 +606,10 @@ async def get_orphaned_series( return list(result.scalars().all()), total -async def get_orphaned_count(session: AsyncSession) -> int: +async def get_orphaned_count(session: AsyncSession, *, job_id: int | None = None) -> int: """Return total count of active orphaned series from completed jobs.""" - count_q = ( - sa_select(sa_func.count()) - .select_from(ImportedSeries) - .join(ImportJob, ImportedSeries.import_job_id == ImportJob.id) - .where( - _active_orphan_clause(), - ImportJob.status == ImportJobStatus.COMPLETED, - ) + count_q = sa_select(sa_func.count()).select_from( + _orphaned_series_query(job_id=job_id).subquery() ) result = await session.execute(count_q) return result.scalar() or 0 @@ -516,63 +748,186 @@ async def retry_failed_series( job_id: int, *, log_event: ImportEventLogger, + file_ids: Sequence[int] | None = None, ) -> tuple[ImportJob, int]: """Reset failed import rows/files for re-execution.""" job = await session.get(ImportJob, job_id) if job is None: raise NotFoundError("ImportJob", job_id) - if job.status != ImportJobStatus.COMPLETED: - raise ValidationError(f"Job must be in COMPLETED state to retry (current: {job.status})") - - result = await session.execute( - sa_select(ImportedSeries).where( - ImportedSeries.import_job_id == job_id, - ImportedSeries.status == ImportSeriesStatus.FAILED, + if not allows_terminal_import_recovery(job): + raise ValidationError( + "Job must have a COMPLETED canonical import with no pending control or rollback " + f"work to retry (current: {job.status})" ) + + require_retained_import_destination(job) + + normalized_file_ids = ( + tuple(sorted({int(file_id) for file_id in file_ids})) if file_ids is not None else None + ) + source_recheck = await prepare_retryable_failed_sources_for_retry( + session, + job, + file_ids=normalized_file_ids, ) - failed_items = list(result.scalars().all()) + repaired_series_ids: set[int] = set() + prepared_target_series_ids: set[int] = set() + normalized_series_count = 0 + if normalized_file_ids is None: + ( + repaired_series_ids, + prepared_target_series_ids, + normalized_series_count, + ) = await _prepare_terminal_follow_up( + session, + job, + ) + if repaired_series_ids: + await log_event( + session, + job_id, + "INFO", + "import_terminal_follow_up_prepared", + message=( + "Restored successful outcomes and moved unresolved identities to Follow-up." + ), + repaired_series_count=normalized_series_count, + affected_series_count=len(repaired_series_ids), + retry_series_count=len(prepared_target_series_ids), + ) - failed_file_result = await session.execute( - sa_select(ImportedSeries) - .join(ImportedFile, ImportedFile.import_series_id == ImportedSeries.id) - .where( - ImportedSeries.import_job_id == job_id, - ImportedSeries.status.in_( - [ - ImportSeriesStatus.DUPLICATE, - ImportSeriesStatus.IMPORTED, - ] - ), - ImportedFile.status == ImportedFileStatus.FAILED, + identity_blocked_count = 0 + if normalized_file_ids is None: + result = await session.execute( + sa_select(ImportedSeries).where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status == ImportSeriesStatus.FAILED, + ) ) - .distinct() - ) - failed_file_items = list(failed_file_result.scalars().all()) - retry_items_by_id = {item.id: item for item in [*failed_items, *failed_file_items]} + failed_items = list(result.scalars().all()) + eligible_failed_items = [ + item + for item in failed_items + if not ( + item.cv_id is None + and item.user_selected_cv_id is None + and dict(item.diagnostics or {}).get("reason") == "trusted_source_identity_conflict" + ) + ] + identity_blocked_count = len(failed_items) - len(eligible_failed_items) + failed_items = eligible_failed_items + + failed_file_result = await session.execute( + sa_select(ImportedSeries, ImportedFile) + .join(ImportedFile, ImportedFile.import_series_id == ImportedSeries.id) + .where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status.in_( + [ + ImportSeriesStatus.DUPLICATE, + ImportSeriesStatus.IMPORTED, + ] + ), + ImportedFile.status == ImportedFileStatus.FAILED, + ) + ) + failed_file_items = [ + item + for item, imp_file in failed_file_result.all() + if not isinstance(dict(imp_file.diagnostics or {}).get("source_revalidation"), dict) + ] + retry_items_by_id = {item.id: item for item in [*failed_items, *failed_file_items]} + if prepared_target_series_ids: + prepared_items = await session.scalars( + sa_select(ImportedSeries).where(ImportedSeries.id.in_(prepared_target_series_ids)) + ) + retry_items_by_id.update({item.id: item for item in prepared_items}) + else: + scoped_items = await session.execute( + sa_select(ImportedSeries, ImportedFile) + .join(ImportedFile, ImportedFile.import_series_id == ImportedSeries.id) + .where( + ImportedSeries.import_job_id == job_id, + ImportedFile.id.in_(normalized_file_ids), + ImportedFile.status == ImportedFileStatus.FAILED, + ImportedFile.diagnostics["source_recheck"]["ready_for_retry"] + .as_boolean() + .is_(True), + ) + ) + retry_items_by_id = {item.id: item for item, _imp_file in scoped_items.all()} retry_items = list(retry_items_by_id.values()) if not retry_items: + if repaired_series_ids: + await session.flush() + return job, 0 + if identity_blocked_count: + raise ValidationError( + "These series still need identity review. Open Follow-up and use " + "Recover known series when available, or review their remaining conflicts." + ) + if source_recheck["files_checked"] > 0: + await session.flush() + await log_event( + session, + job_id, + "WARNING", + "import_retry_failed_source_revalidation_blocked", + message="No failed files passed source revalidation", + source_files_checked=source_recheck["files_checked"], + source_files_blocked=source_recheck["blocked_files"], + ) + return job, 0 raise ValidationError("No failed series or files to retry") retry_series_ids = [item.id for item in retry_items] for item in retry_items: - if item.status in {ImportSeriesStatus.FAILED, ImportSeriesStatus.IMPORTED}: + if item.status in { + ImportSeriesStatus.FAILED, + ImportSeriesStatus.IMPORTED, + ImportSeriesStatus.RECOVERY_PENDING, + }: item.status = ImportSeriesStatus.CONFIRMED item.error_message = None - failed_files_result = await session.execute( - sa_select(ImportedFile).where( - ImportedFile.import_series_id.in_(retry_series_ids), - ImportedFile.status == ImportedFileStatus.FAILED, - ) + failed_files_query = sa_select(ImportedFile).where( + ImportedFile.import_series_id.in_(retry_series_ids), + ImportedFile.status == ImportedFileStatus.FAILED, ) - failed_files = list(failed_files_result.scalars().all()) + if normalized_file_ids is not None: + failed_files_query = failed_files_query.where(ImportedFile.id.in_(normalized_file_ids)) + failed_files_result = await session.execute(failed_files_query) + failed_files = [ + imp_file + for imp_file in failed_files_result.scalars().all() + if not isinstance(dict(imp_file.diagnostics or {}).get("source_revalidation"), dict) + ] for imp_file in failed_files: imp_file.status = ImportedFileStatus.CONFIRMED imp_file.include_in_import = True imp_file.error_message = None + retry_file_query = sa_select(ImportedFile).where( + ImportedFile.import_series_id.in_(retry_series_ids), + ImportedFile.status.in_([ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED]), + ) + if normalized_file_ids is not None: + retry_file_query = retry_file_query.where(ImportedFile.id.in_(normalized_file_ids)) + retry_file_result = await session.execute(retry_file_query) + repaired_target_count = 0 + for imp_file in retry_file_result.scalars().all(): + had_summary = isinstance(dict(imp_file.diagnostics or {}).get("target_issue_summary"), dict) + if ensure_target_issue_summary_for_import_file(imp_file): + if not had_summary and isinstance( + dict(imp_file.diagnostics or {}).get("target_issue_summary"), dict + ): + repaired_target_count += 1 + continue + imp_file.status = ImportedFileStatus.NO_MATCH + imp_file.include_in_import = False + count = len(retry_items) job.status = ImportJobStatus.IMPORTING await recompute_file_counters(session, job, series_ids=retry_series_ids) @@ -587,6 +942,7 @@ async def retry_failed_series( message=f"Retrying {count} failed import item{'s' if count != 1 else ''}", retry_count=count, retry_file_count=len(failed_files), + repaired_target_count=repaired_target_count, ) return job, count diff --git a/src/pullbox/services/import_path_identity.py b/src/pullbox/services/import_path_identity.py new file mode 100644 index 00000000..8c575934 --- /dev/null +++ b/src/pullbox/services/import_path_identity.py @@ -0,0 +1,113 @@ +"""Shared evidence and filesystem boundaries for stale source reconciliation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from pullbox.core.exceptions import ConfigurationError +from pullbox.core.filesystem_policy import is_invalid_path_text, resolve_preview_source +from pullbox.core.library_file_ownership import ( + build_file_identity_signature, + validate_file_identity_signature, +) +from pullbox.core.name_matcher import NameMatcher +from pullbox.core.source_metadata import MetadataSignal, SourceMetadata + +if TYPE_CHECKING: + from pathlib import Path + + +def same_trusted_issue(recorded: SourceMetadata, actual: SourceMetadata) -> bool: + """Require independent exact IDs; folder names and issue numbers are not proof.""" + if ( + not recorded.comicvine_issue_id + or recorded.comicvine_issue_id != actual.comicvine_issue_id + or recorded.signals.get("comicvine_issue_id") != MetadataSignal.MYLAR3 + or actual.signals.get("comicvine_issue_id") != MetadataSignal.COMICINFO + or recorded.diagnostics.get("identity_conflicts") + or actual.diagnostics.get("identity_conflicts") + or recorded.issue_type != actual.issue_type + ): + return False + if ( + recorded.comicvine_series_id is not None + and actual.comicvine_series_id is not None + and recorded.comicvine_series_id != actual.comicvine_series_id + ): + return False + exact_series_identity = ( + recorded.comicvine_series_id is not None + and actual.comicvine_series_id is not None + and recorded.comicvine_series_id == actual.comicvine_series_id + ) + series_names_match = bool( + recorded.series_name + and actual.series_name + and NameMatcher.normalize(recorded.series_name) == NameMatcher.normalize(actual.series_name) + ) + if (not exact_series_identity and not series_names_match) or ( + recorded.issue_number != actual.issue_number + ): + return False + recorded_issue = recorded.diagnostics.get("mylar3_issue") + date = recorded_issue.get("release_date") if isinstance(recorded_issue, dict) else None + return not ( + isinstance(date, str) + and date[:4].isdigit() + and actual.year is not None + and int(date[:4]) != actual.year + ) + + +def unchanged_same_folder_pair(recorded: Path, actual: Path, signature: dict[str, Any]) -> bool: + """A missing reference must really be absent, never unreadable or a dangling link.""" + if recorded.parent != actual.parent: + return False + if any( + not path.is_absolute() or ".." in path.parts or is_invalid_path_text(str(path)) + for path in (recorded, actual) + ): + return False + try: + recorded.lstat() + except FileNotFoundError: + pass + except OSError: + return False + else: + return False + try: + if actual.is_symlink(): + return False + resolve_preview_source(actual) + validate_file_identity_signature(signature, build_file_identity_signature(actual)) + return True + except (OSError, RuntimeError, ValueError, ConfigurationError): + return False + + +def reconciliation_evidence( + recorded: str, + actual: str, + issue_id: int, + *, + recorded_series_name: str | None = None, + actual_series_name: str | None = None, +) -> dict[str, Any]: + evidence: dict[str, Any] = { + "recorded_path": recorded, + "actual_path": actual, + "comicvine_issue_id": issue_id, + "method": "verified_same_folder_issue_identity", + } + if ( + recorded_series_name + and actual_series_name + and NameMatcher.normalize(recorded_series_name) != NameMatcher.normalize(actual_series_name) + ): + evidence["series_name_alias"] = { + "recorded": recorded_series_name, + "actual": actual_series_name, + "accepted_by": "exact_comicvine_series_and_issue_identity", + } + return evidence diff --git a/src/pullbox/services/import_path_reconciliation.py b/src/pullbox/services/import_path_reconciliation.py new file mode 100644 index 00000000..df5085e6 --- /dev/null +++ b/src/pullbox/services/import_path_reconciliation.py @@ -0,0 +1,298 @@ +"""Offline, transactional repair of proven stale Mylar review references.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from sqlalchemy import func, or_, select +from sqlalchemy.orm import aliased + +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.file_safety import ( + get_archive_size_limit_bytes, + is_dangerous_file_blocking_enabled, +) +from pullbox.core.filesystem_policy import resolve_preview_source +from pullbox.models.import_job import ( + ImportControlRequest, + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobLog, + ImportJobStatus, + ImportSourceType, +) +from pullbox.services.import_counters import recompute_file_counters +from pullbox.services.import_path_identity import ( + reconciliation_evidence, + same_trusted_issue, + unchanged_same_folder_pair, +) +from pullbox.services.import_review_recheck import inspect_review_source +from pullbox.services.import_source_metadata import source_metadata_for_import_file + +if TYPE_CHECKING: + from sqlalchemy import Select + from sqlalchemy.ext.asyncio import AsyncSession + + +def _candidate_query( + job_id: int, +) -> tuple[Select[tuple[ImportedFile, ImportedFile, ImportedSeries]], type[ImportedFile]]: + # Count all same-folder copies, including blocked ones, before choosing a match. + folder = func.substr( + ImportedFile.file_path, + 1, + func.length(ImportedFile.file_path) - func.length(ImportedFile.file_name), + ) + groups = select( + ImportedFile.import_series_id.label("series_id"), + ImportedFile.comicvine_issue_id.label("issue_id"), + folder.label("folder"), + func.count().label("n"), + func.min(ImportedFile.id).label("id"), + ).where(ImportedFile.import_job_id == job_id, ImportedFile.comicvine_issue_id.is_not(None)) + group_by = (ImportedFile.import_series_id, ImportedFile.comicvine_issue_id, folder) + missing = ( + groups.where( + ImportedFile.diagnostics["safety_block"]["code"].as_string() == "source_missing" + ) + .group_by(*group_by) + .cte("missing_references") + ) + available = ( + groups.where(ImportedFile.file_size > 0).group_by(*group_by).cte("available_sources") + ) + old, actual = aliased(ImportedFile), aliased(ImportedFile) + return ( + select(old, actual, ImportedSeries) + .join(missing, old.id == missing.c.id) + .join( + available, + (missing.c.series_id == available.c.series_id) + & (missing.c.issue_id == available.c.issue_id) + & (missing.c.folder == available.c.folder), + ) + .join(actual, actual.id == available.c.id) + .join(ImportedSeries, ImportedSeries.id == old.import_series_id) + .where( + missing.c.n == 1, + available.c.n == 1, + actual.status == ImportedFileStatus.MATCHED, + old.status == ImportedFileStatus.SAFETY_BLOCKED, + ), + old, + ) + + +async def _protected_series(session: AsyncSession, ids: list[int]) -> set[int]: + return set( + await session.scalars( + select(ImportedFile.import_series_id) + .where( + ImportedFile.import_series_id.in_(ids), + or_( + ImportedFile.status.not_in( + [ + ImportedFileStatus.MATCHED, + ImportedFileStatus.PENDING, + ImportedFileStatus.NO_MATCH, + ImportedFileStatus.SAFETY_BLOCKED, + ] + ), + ImportedFile.match_method.startswith("manual"), + ImportedFile.include_in_import.is_(True), + ImportedFile.diagnostics["safety_exception"]["allowed_once"] + .as_boolean() + .is_(True), + ), + ) + .distinct() + ) + ) + + +def _retain(report: dict[str, Any], record: ImportedFile, reason: str) -> None: + counts = report["retained_reasons"] + counts[reason] = counts.get(reason, 0) + 1 + if len(report["retained_samples"]) < 20: + report["retained_samples"].append( + { + "recorded_file_id": record.id, + "recorded_path": record.file_path, + "reason": reason, + } + ) + + +async def reconcile_saved_mylar_paths( + session: AsyncSession, + job_id: int, + *, + source_roots: list[Path], + apply: bool = False, + series_ids: list[int] | None = None, +) -> dict[str, Any]: + """Leave the job in REVIEW; caller commits only after an explicit apply.""" + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if job.status != ImportJobStatus.REVIEW or job.control_request != ImportControlRequest.NONE: + raise ValidationError("Job must be idle in REVIEW before offline reconciliation") + if job.source_type != ImportSourceType.MYLAR3: + raise ValidationError("Stale Mylar reference repair requires a Mylar import") + if not source_roots: + raise ValidationError("At least one explicit source root is required") + roots = [(path.expanduser().absolute(), resolve_preview_source(path)) for path in source_roots] + if any(not real.is_dir() or real.parent == real for _, real in roots): + raise ValidationError("Source roots must be specific existing directories") + dangerous = await is_dangerous_file_blocking_enabled(session) + limit = await get_archive_size_limit_bytes(session) + report: dict[str, Any] = { + "candidates_checked": 0, + "references_reconciled": 0, + "candidates_retained": 0, + "samples": [], + "retained_reasons": {}, + "retained_samples": [], + } + count = ( + select(func.count()) + .select_from(ImportedFile) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + ImportedFile.diagnostics["safety_block"]["code"].as_string() == "source_missing", + ) + ) + if series_ids: + count = count.where(ImportedFile.import_series_id.in_(series_ids)) + report["missing_references"] = await session.scalar(count) or 0 + query, old = _candidate_query(job_id) + if series_ids: + query = query.where(ImportedSeries.id.in_(series_ids)) + changed_series: set[int] = set() + # Stream one grouped query rather than re-aggregate the entire library per page. + result = await session.stream(query.order_by(old.id).execution_options(yield_per=250)) + try: + async for rows in result.partitions(250): + protected = await _protected_series(session, list({s.id for _, _, s in rows})) + referenced = set( + await session.scalars( + select(ImportedFile.duplicate_of_file_id).where( + ImportedFile.duplicate_of_file_id.in_([record.id for record, _, _ in rows]) + ) + ) + ) + sidecars: dict[str, dict[str, Any]] = {} + for record, actual, series in rows: + report["candidates_checked"] += 1 + report["candidates_retained"] += 1 + if ( + series.id in protected + or series.user_selected_cv_id is not None + or series.selected_for_import + or record.source_signature + or actual.diagnostics.get("safety_block") + or record.id in referenced + or actual.library_file_id is not None + or actual.matched_issue_cv_id != record.comicvine_issue_id + ): + _retain(report, record, "review_or_source_protected") + continue + if not await asyncio.to_thread( + unchanged_same_folder_pair, + Path(record.file_path), + Path(actual.file_path), + dict(actual.source_signature), + ): + _retain(report, record, "source_check_failed") + continue + base = source_metadata_for_import_file(series, record) + metadata, content, _signature = await asyncio.to_thread( + inspect_review_source, + Path(actual.file_path), + source_metadata_for_import_file(series, actual), + dict(actual.source_signature), + roots=roots, + block_dangerous=dangerous, + max_archive_size=limit, + accept_replaced_files=False, + sidecars=sidecars, + ) + if "file_safety" in content: + _retain(report, record, "file_safety_review") + continue + if not same_trusted_issue(base, metadata): + _retain(report, record, "identity_unconfirmed") + continue + if not await asyncio.to_thread( + unchanged_same_folder_pair, + Path(record.file_path), + Path(actual.file_path), + dict(actual.source_signature), + ): + _retain(report, record, "source_changed_during_check") + continue + evidence = { + **reconciliation_evidence( + record.file_path, + actual.file_path, + record.comicvine_issue_id, + recorded_series_name=base.series_name, + actual_series_name=metadata.series_name, + ), + "recorded_file_id": record.id, + } + report["references_reconciled"] += 1 + report["candidates_retained"] -= 1 + if len(report["samples"]) < 20: + report["samples"].append(evidence) + if apply: + diagnostics = dict(actual.diagnostics or {}) + source = dict(diagnostics.get("source_metadata") or {}) + source["mylar3_path_reconciliation"] = evidence + actual.diagnostics = {**diagnostics, "source_metadata": source} + await session.delete(record) + changed_series.add(series.id) + if apply: + await session.flush() + finally: + await result.close() + unmatched = report["missing_references"] - report["candidates_checked"] + if unmatched: + report["retained_reasons"]["no_unique_matched_counterpart"] = unmatched + report["remaining_missing_references"] = ( + report["missing_references"] - report["references_reconciled"] + ) + if apply and report["references_reconciled"]: + await recompute_file_counters(session, job) + for series_id in changed_series: + series = await session.get(ImportedSeries, series_id) + if series is not None: + series.file_count = series.files_total + series.sample_paths = list( + await session.scalars( + select(ImportedFile.file_path) + .where(ImportedFile.import_series_id == series.id) + .order_by(ImportedFile.id) + .limit(5) + ) + ) + session.add( + ImportJobLog( + import_job_id=job.id, + level="INFO", + event="import_mylar_paths_reconciled", + message=( + f"Reconciled {report['references_reconciled']} stale Mylar references " + "with verified files; source files and user decisions unchanged." + ), + data=report, + ) + ) + await session.flush() + return report diff --git a/src/pullbox/services/import_placement_recovery.py b/src/pullbox/services/import_placement_recovery.py new file mode 100644 index 00000000..e7835f57 --- /dev/null +++ b/src/pullbox/services/import_placement_recovery.py @@ -0,0 +1,165 @@ +"""Narrow same-job recovery for durably published import placements.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from sqlalchemy import select + +from pullbox.core.exceptions import ConfigurationError +from pullbox.core.library_file_ownership import build_managed_placement_signature +from pullbox.models.import_job import ImportJobAction, ImportJobActionStatus +from pullbox.models.library import LibraryFile + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +@dataclass(frozen=True, slots=True) +class CompletedImportPlacementRecovery: + """Validated same-job evidence for a placement awaiting DB registration.""" + + action_id: int + destination_path: Path + destination_signature: dict[str, int | str] + + +async def has_completed_direct_move_placement_record( + session: AsyncSession, + *, + job_id: int, + imported_file_id: int, + source_path: Path, +) -> bool: + """Return whether a missing direct-move source has a durable completion row. + + This only permits execution to reach the full recovery validator. It does + not authorize registration or filesystem mutation. + """ + actions = await _candidate_actions(session, job_id, imported_file_id) + for action in actions: + payload = dict(action.payload or {}) + if payload.get("placement_completed") is not True: + continue + if str(payload.get("transfer_method") or "") != "move": + continue + original_source = _payload_path(payload, "original_source_path") + artifact_source = _payload_path(payload, "artifact_source_path") + if original_source is None or artifact_source is None: + continue + if not _same_unresolved_path(original_source, source_path): + continue + if not _same_unresolved_path(artifact_source, source_path): + continue + destination = _payload_path(payload, "destination_path") + if destination is not None and os.path.lexists(destination): + return True + return False + + +async def load_completed_import_placement_recovery( + session: AsyncSession, + *, + job_id: int, + imported_file_id: int, + issue_id: int, + source_path: Path, + transfer_method: str, +) -> CompletedImportPlacementRecovery | None: + """Validate exact durable provenance and content for same-job recovery.""" + actions = await _candidate_actions(session, job_id, imported_file_id) + for action in actions: + payload = dict(action.payload or {}) + if payload.get("placement_completed") is not True: + continue + if int(payload.get("issue_id") or 0) != issue_id: + continue + if str(payload.get("transfer_method") or "") != transfer_method: + continue + original_source = _payload_path(payload, "original_source_path") + artifact_source = _payload_path(payload, "artifact_source_path") + destination = _payload_path(payload, "destination_path") + if original_source is None or artifact_source is None or destination is None: + continue + if not _same_unresolved_path(original_source, source_path): + continue + if _same_unresolved_path(destination, source_path): + continue + if any( + os.path.lexists(Path(str(temp_path))) + for temp_path in payload.get("temp_paths") or [] + if str(temp_path) + ): + continue + if not os.path.lexists(destination): + continue + if ( + transfer_method == "move" + and _same_unresolved_path(artifact_source, original_source) + and os.path.lexists(original_source) + ): + # A direct-move source reappeared. Preserve both paths for review. + continue + signature = payload.get("destination_signature") + if not isinstance(signature, dict): + continue + if ( + not signature.get("content_digest") + or signature.get("content_digest_algorithm") != "sha256" + ): + continue + try: + current_signature = build_managed_placement_signature(destination) + except (ConfigurationError, OSError, RuntimeError, ValueError): + continue + if current_signature != signature: + continue + existing_library_file = await session.scalar( + select(LibraryFile.id).where(LibraryFile.file_path == str(destination)).limit(1) + ) + if existing_library_file is not None: + # Recovery is only for the crash window before LibraryFile commit. + continue + return CompletedImportPlacementRecovery( + action_id=int(action.id), + destination_path=destination, + destination_signature={str(key): value for key, value in signature.items()}, + ) + return None + + +async def _candidate_actions( + session: AsyncSession, + job_id: int, + imported_file_id: int, +) -> list[ImportJobAction]: + return list( + ( + await session.scalars( + select(ImportJobAction) + .where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.action_type == "library_file_placement_started", + ImportJobAction.status == ImportJobActionStatus.COMPLETED, + ImportJobAction.payload["imported_file_id"].as_integer() == imported_file_id, + ) + .order_by(ImportJobAction.sequence_no.desc()) + .limit(2) + ) + ).all() + ) + + +def _payload_path(payload: dict[str, object], key: str) -> Path | None: + raw = str(payload.get(key) or "").strip() + return Path(raw) if raw else None + + +def _same_unresolved_path(left: Path, right: Path) -> bool: + try: + return left.expanduser().resolve(strict=False) == right.expanduser().resolve(strict=False) + except (OSError, RuntimeError, ValueError): + return False diff --git a/src/pullbox/services/import_policy_snapshot.py b/src/pullbox/services/import_policy_snapshot.py index 87dbf5d5..664e1b8f 100644 --- a/src/pullbox/services/import_policy_snapshot.py +++ b/src/pullbox/services/import_policy_snapshot.py @@ -8,7 +8,7 @@ LibraryIngestPolicy, serialize_library_ingest_policy, ) -from pullbox.models.import_job import ImportSourceType +from pullbox.models.import_job import ImportFileHandlingMode, ImportSourceType if TYPE_CHECKING: from pullbox.models.import_job import ImportJob @@ -26,12 +26,21 @@ def apply_ingest_policy_to_import_job( policy: LibraryIngestPolicy, ) -> None: """Apply and snapshot the ingest policy that this import job should honor.""" - effective_transfer_method = _effective_transfer_method_for_import(job, policy) job.ingest_policy_snapshot = serialize_library_ingest_policy(policy) - job.move_to_library = True job.transfer_method = policy.post_processing_method job.torrent_import_strategy = policy.torrent_import_strategy job.effective_import_strategy = "standard" + handling_mode = job.file_handling_mode or ImportFileHandlingMode.MANAGED_COPY + if handling_mode == ImportFileHandlingMode.IN_PLACE: + job.move_to_library = False + job.effective_transfer_method = "leave_in_place" + job.source_preserved = True + job.convert_to_preferred_format = False + job.update_embedded_comicinfo_from_match = False + return + + effective_transfer_method = _effective_transfer_method_for_import(job, policy) + job.move_to_library = True job.effective_transfer_method = effective_transfer_method job.source_preserved = effective_transfer_method in {"copy", "hardlink", "symlink"} job.convert_to_preferred_format = policy.normalize_imported_archives_to_cbz diff --git a/src/pullbox/services/import_progress_runtime.py b/src/pullbox/services/import_progress_runtime.py index 16a0442c..b7b3c6da 100644 --- a/src/pullbox/services/import_progress_runtime.py +++ b/src/pullbox/services/import_progress_runtime.py @@ -117,6 +117,8 @@ class ScanReviewProgressPlan: analysis_weights: tuple[float, ...] series_match_weights: tuple[float, ...] file_match_weights: tuple[float, ...] + progress_start: int = _SCAN_REVIEW_PROGRESS_START + progress_end: int = _SCAN_REVIEW_PROGRESS_END @property def analysis_weight(self) -> float: @@ -171,6 +173,27 @@ def total_weight(self) -> float: ) +@dataclass(frozen=True, slots=True) +class ImportWorkProgress: + """Keep measured work precise even when its display percentage rounds to zero.""" + + completed_weight: float + total_weight: float + + @property + def progress_pct(self) -> int: + if self.total_weight <= 0: + return 0 + return max(0, min(round(self.completed_weight / self.total_weight * 100), 99)) + + def remaining_seconds(self, started_at: datetime | None) -> int | None: + return estimate_remaining_work_seconds( + started_at, + completed_units=self.completed_weight, + total_units=self.total_weight, + ) + + def phase_range(phase: str) -> tuple[int, int]: """Return the canonical bounded progress range for a workflow phase.""" return _PHASE_RANGES.get(phase, (0, 100)) @@ -276,6 +299,11 @@ def scan_review_progress_plan( ) +def scan_review_analysis_weight(series_count: int) -> float: + """Return aggregate analysis work without allocating one entry per series.""" + return max(series_count, 0) * _ANALYSIS_SERIES_WEIGHT + + def scan_review_series_match_weight(profile: ScanReviewSeriesMatchProfile) -> float: """Estimate relative Step 2 cost for one series-level match.""" base = _SERIES_MATCH_DIRECT_WEIGHT if profile.direct_match else _SERIES_MATCH_SEARCH_WEIGHT @@ -290,6 +318,13 @@ def scan_review_file_target_weight(profile: ScanReviewFileMatchProfile) -> float return max(_FILE_TARGET_BASE_WEIGHT + issue_bonus, 0.1) +def scan_review_file_match_weight(profile: ScanReviewFileMatchProfile) -> float: + """Return aggregate file-match work without allocating one entry per file.""" + return scan_review_file_target_weight(profile) + ( + max(profile.file_count, 0) * _FILE_MATCH_FILE_WEIGHT + ) + + def scan_review_completed_weight( plan: ScanReviewProgressPlan, *, @@ -324,10 +359,8 @@ def scan_review_completed_weight( def scan_review_progress_pct(plan: ScanReviewProgressPlan, *, completed_weight: float) -> int: """Scale weighted Step 2 review-prep work into the canonical 35-99 range.""" fraction = min(max(completed_weight / plan.total_weight, 0.0), 1.0) - progress = _SCAN_REVIEW_PROGRESS_START + ( - (_SCAN_REVIEW_PROGRESS_END - _SCAN_REVIEW_PROGRESS_START) * fraction - ) - return max(_SCAN_REVIEW_PROGRESS_START, min(round(progress), _SCAN_REVIEW_PROGRESS_END)) + progress = plan.progress_start + ((plan.progress_end - plan.progress_start) * fraction) + return max(plan.progress_start, min(round(progress), plan.progress_end)) def _weighted_completed( @@ -387,10 +420,21 @@ def import_group_metadata_progress_pct( metadata_progress_pct: int | float, ) -> int: """Return current group progress while series metadata is being prepared.""" - completed = plan.metadata_weight * (_clamped_pct(metadata_progress_pct) / 100) + completed = import_group_metadata_completed_weight( + plan, metadata_progress_pct=metadata_progress_pct + ) return _clamped_pct((completed / plan.total_weight) * 100) +def import_group_metadata_completed_weight( + plan: ImportGroupProgressPlan, + *, + metadata_progress_pct: int | float, +) -> float: + """Measure metadata work without rounding its share of the group.""" + return plan.metadata_weight * (_clamped_pct(metadata_progress_pct) / 100) + + def import_group_file_progress_pct( plan: ImportGroupProgressPlan, *, @@ -398,6 +442,19 @@ def import_group_file_progress_pct( current_file_pct: int | float, ) -> int: """Return current group progress while an importable file is being processed.""" + completed = import_group_file_completed_weight( + plan, file_index=file_index, current_file_pct=current_file_pct + ) + return _clamped_pct((completed / plan.total_weight) * 100) + + +def import_group_file_completed_weight( + plan: ImportGroupProgressPlan, + *, + file_index: int, + current_file_pct: int | float, +) -> float: + """Measure completed file work without rounding its share of the group.""" completed = plan.metadata_weight safe_index = max(file_index, 1) for idx, (_file_id, weight) in enumerate(plan.file_weights, start=1): @@ -407,7 +464,7 @@ def import_group_file_progress_pct( if idx == safe_index: completed += weight * (_clamped_pct(current_file_pct) / 100) break - return _clamped_pct((completed / plan.total_weight) * 100) + return completed def weighted_import_progress_pct( @@ -417,16 +474,31 @@ def weighted_import_progress_pct( current_group_progress_pct: int | float, ) -> int: """Return whole-job Step 4 progress across weighted review groups.""" - if not group_weights: - return 0 safe_index = max(current_group_index, 0) - total_weight = max(sum(max(weight, 1.0) for weight in group_weights), 1.0) + current_weight = max(group_weights[safe_index], 1.0) if safe_index < len(group_weights) else 0.0 + return import_work_progress( + group_weights, + current_group_index=safe_index, + current_group_completed_weight=current_weight + * (_clamped_pct(current_group_progress_pct) / 100), + ).progress_pct + + +def import_work_progress( + group_weights: list[float] | tuple[float, ...], + *, + current_group_index: int, + current_group_completed_weight: float, +) -> ImportWorkProgress: + """Use one unrounded work position for both the display and Step 4 ETA.""" + safe_index = max(current_group_index, 0) + total_weight = sum(max(weight, 1.0) for weight in group_weights) completed = sum(max(weight, 1.0) for weight in group_weights[:safe_index]) if safe_index < len(group_weights): - completed += max(group_weights[safe_index], 1.0) * ( - _clamped_pct(current_group_progress_pct) / 100 + completed += min( + max(current_group_completed_weight, 0.0), max(group_weights[safe_index], 1.0) ) - return max(0, min(round((completed / total_weight) * 100), 99)) + return ImportWorkProgress(completed_weight=completed, total_weight=total_weight) def _clamped_pct(value: int | float | None) -> int: diff --git a/src/pullbox/services/import_provider_cache.py b/src/pullbox/services/import_provider_cache.py index dd557516..5655e768 100644 --- a/src/pullbox/services/import_provider_cache.py +++ b/src/pullbox/services/import_provider_cache.py @@ -44,8 +44,10 @@ def build_import_scan_metadata_provider( provider: Any, ) -> CachedImportMetadataProvider: """Return the Step 2 provider stack: persistent cache, then per-job cache.""" + from pullbox.services.catalog.lookup import catalog_or_provider + return CachedImportMetadataProvider( - build_persistent_import_metadata_provider(session, provider) + catalog_or_provider(build_persistent_import_metadata_provider(session, provider)) ) diff --git a/src/pullbox/services/import_reconcile_helpers.py b/src/pullbox/services/import_reconcile_helpers.py index 7ea137c6..342b4d8e 100644 --- a/src/pullbox/services/import_reconcile_helpers.py +++ b/src/pullbox/services/import_reconcile_helpers.py @@ -8,6 +8,7 @@ from sqlalchemy.orm import joinedload from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.issue_numbers import format_issue_number from pullbox.core.name_matcher import NameMatcher from pullbox.core.release_parser import parse_release_title from pullbox.models.import_job import ImportedFile, ImportedFileStatus, ImportedSeries @@ -316,7 +317,7 @@ def build_reconcile_file_rows( for option in issue_options: issue_cv_id = int(option["issue_cv_id"]) issue_number = float(option["issue_number"]) - label = f"#{issue_number:g}" + label = f"#{format_issue_number(issue_number)}" if option.get("title"): label = f"{label} - {option['title']}" issue_label_by_cv_id[issue_cv_id] = label diff --git a/src/pullbox/services/import_referenced_sources.py b/src/pullbox/services/import_referenced_sources.py new file mode 100644 index 00000000..ac0ac0c8 --- /dev/null +++ b/src/pullbox/services/import_referenced_sources.py @@ -0,0 +1,244 @@ +"""Read-only scan eligibility for Mylar files adopted across configured roots.""" + +from __future__ import annotations + +import asyncio +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from sqlalchemy import select + +from pullbox.core.exceptions import ConfigurationError, ValidationError +from pullbox.core.library_file_ownership import ( + ReferencedFileValidationError, + build_file_identity_signature, + validate_file_identity_signature, +) +from pullbox.core.library_root_resolution import resolve_library_root +from pullbox.models.library import LibraryRoot +from pullbox.services.import_safety_diagnostics import build_import_safety_diagnostics +from pullbox.services.library_root_management import validate_managed_library_root + +if TYPE_CHECKING: + from collections.abc import Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.core.collection_scanner import DiscoveredFile, DiscoveredSeries + + +MYLAR_REFERENCE_ROOT_ID_SIGNATURE_KEY = "mylar_reference_root_id" +_CONTROL_CHARACTER_RE = re.compile(r"[\x00-\x1f\x7f]") + + +@dataclass(frozen=True, slots=True) +class MylarReferenceRootBoundary: + """Immutable scan-time containment evidence for one reference-capable root.""" + + root_id: int + lexical: Path + resolved: Path + device: int + inode: int + + +async def load_mylar_in_place_root(session: AsyncSession, root_id: int | None) -> LibraryRoot: + """Require the preferred future managed destination for an in-place import.""" + if root_id is None: + raise ValidationError( + "Select an enabled managed-write library root for Mylar in-place import." + ) + try: + root = await resolve_library_root(session, Path(), root_id) + except ConfigurationError as exc: + raise ValidationError(exc.message) from exc + await validate_managed_library_root(root) + return root + + +async def load_mylar_reference_root_boundaries( + session: AsyncSession, +) -> tuple[MylarReferenceRootBoundary, ...]: + """Load the current enabled roots that explicitly allow references.""" + roots = list( + ( + await session.execute( + select(LibraryRoot) + .where( + LibraryRoot.enabled.is_(True), + LibraryRoot.allow_referenced_registrations.is_(True), + ) + .order_by(LibraryRoot.id.asc()) + ) + ) + .scalars() + .all() + ) + root_values = [(int(root.id), str(root.path)) for root in roots] + return await asyncio.to_thread(_snapshot_root_boundaries, root_values) + + +def _snapshot_root_boundaries( + roots: Sequence[tuple[int, str]], +) -> tuple[MylarReferenceRootBoundary, ...]: + boundaries: list[MylarReferenceRootBoundary] = [] + for root_id, raw_path in roots: + path = Path(raw_path) + if not path.is_absolute() or ".." in path.parts or _CONTROL_CHARACTER_RE.search(raw_path): + continue + try: + lexical = path.expanduser().absolute() + resolved = path.expanduser().resolve(strict=True) + stat_result = resolved.stat() + except (OSError, RuntimeError, ValueError): + continue + if not resolved.is_dir() or not os.access(resolved, os.R_OK | os.X_OK): + continue + boundaries.append( + MylarReferenceRootBoundary( + root_id=root_id, + lexical=lexical, + resolved=resolved, + device=stat_result.st_dev, + inode=stat_result.st_ino, + ) + ) + return tuple(boundaries) + + +def _select_file_root( + source: Path, + roots: Sequence[MylarReferenceRootBoundary], +) -> tuple[MylarReferenceRootBoundary, dict[str, int | str]]: + raw_source = str(source) + if not source.is_absolute() or ".." in source.parts or _CONTROL_CHARACTER_RE.search(raw_source): + raise ReferencedFileValidationError( + "source_path_unsafe", "The Mylar comic path contains unsafe path components." + ) + try: + lexical_source = source.expanduser().absolute() + resolved_source = source.expanduser().resolve(strict=True) + except (OSError, RuntimeError, ValueError) as exc: + raise ReferencedFileValidationError( + "source_missing", "The Mylar comic file is missing or unavailable." + ) from exc + if not resolved_source.is_file(): + raise ReferencedFileValidationError( + "source_missing", "The Mylar comic file is missing or unavailable." + ) + if not os.access(resolved_source, os.R_OK): + raise ReferencedFileValidationError( + "source_unreadable", "The Mylar comic file is not readable by Pullbox." + ) + + lexical_claims = [root for root in roots if lexical_source.is_relative_to(root.lexical)] + resolved_claims = [root for root in roots if resolved_source.is_relative_to(root.resolved)] + resolved_claim_ids = {root.root_id for root in resolved_claims} + candidates = [root for root in lexical_claims if root.root_id in resolved_claim_ids] + candidates.sort( + key=lambda root: (len(root.lexical.parts), len(root.resolved.parts), -root.root_id), + reverse=True, + ) + if not candidates: + raise ReferencedFileValidationError( + "source_outside_root", + "The Mylar comic file is outside every enabled reference-capable library root.", + ) + selected = candidates[0] + selected_aliases = [ + root + for root in roots + if root.root_id != selected.root_id + and (root.device, root.inode) == (selected.device, selected.inode) + ] + if ( + len(candidates) != 1 + or len(lexical_claims) != 1 + or len(resolved_claims) != 1 + or selected_aliases + ): + raise ReferencedFileValidationError( + "source_root_ambiguous", + "The Mylar comic file matches ambiguous nested or aliased library roots.", + ) + return selected, build_file_identity_signature(resolved_source) + + +def _validate_file( + discovered_file: DiscoveredFile, + roots: Sequence[MylarReferenceRootBoundary], +) -> None: + source = Path(discovered_file.file_path) + try: + selected, current = _select_file_root(source, roots) + except ReferencedFileValidationError: + raise + except (OSError, RuntimeError, ValueError, ConfigurationError) as exc: + raise ReferencedFileValidationError( + "source_missing", "The Mylar comic file is missing or unavailable." + ) from exc + validate_file_identity_signature(dict(discovered_file.source_signature), current) + signature = dict(discovered_file.source_signature) + signature[MYLAR_REFERENCE_ROOT_ID_SIGNATURE_KEY] = selected.root_id + discovered_file.source_signature = signature + + +def validate_mylar_in_place_files( + discovered: list[DiscoveredSeries], + roots: Sequence[MylarReferenceRootBoundary], +) -> None: + """Mark ineligible files for review without dropping them or copying them.""" + for series in discovered: + failure_count = 0 + first_failure: tuple[str, str] | None = None + for comic in series.files: + try: + _validate_file(comic, roots) + except ReferencedFileValidationError as exc: + failure_count += 1 + if first_failure is None: + first_failure = exc.reason, exc.message + comic.metadata_diagnostics["file_safety"] = build_import_safety_diagnostics( + exc.message, + kind="source_revalidation", + code=exc.reason, + source="source_revalidation", + overrideable_hint=False, + ) + if first_failure is not None and failure_count == len(series.files): + series.diagnostics.update( + { + "kind": "mylar3_path_incompatible", + "reason": first_failure[0], + "rejection_reason": first_failure[1], + } + ) + path_details = series.diagnostics.get("mylar3_path") + if isinstance(path_details, dict): + path_details["status"] = "in_place_incompatible" + + +async def revalidate_mylar_in_place_file_root( + session: AsyncSession, + source_path: Path, + source_signature: dict[str, object], +) -> int: + """Revalidate the exact scan-selected root immediately before registration.""" + raw_root_id = source_signature.get(MYLAR_REFERENCE_ROOT_ID_SIGNATURE_KEY) + if isinstance(raw_root_id, bool) or not isinstance(raw_root_id, int) or raw_root_id <= 0: + raise ReferencedFileValidationError( + "source_root_unconfirmed", + "The Mylar comic file is missing its confirmed library-root selection. Rescan it.", + ) + roots = await load_mylar_reference_root_boundaries(session) + selected, current = await asyncio.to_thread(_select_file_root, source_path, roots) + if selected.root_id != raw_root_id: + raise ReferencedFileValidationError( + "source_root_changed", + "The Mylar comic file no longer resolves inside its scan-selected library root.", + ) + validate_file_identity_signature(source_signature, current) + return selected.root_id diff --git a/src/pullbox/services/import_retry_helpers.py b/src/pullbox/services/import_retry_helpers.py index 112dca6e..2efd0657 100644 --- a/src/pullbox/services/import_retry_helpers.py +++ b/src/pullbox/services/import_retry_helpers.py @@ -4,7 +4,10 @@ from typing import Any +from pullbox.core.exceptions import ValidationError +from pullbox.models.import_job import ImportFileHandlingMode from pullbox.schemas.import_job import ImportJobCreate +from pullbox.schemas.import_layout import SourceLayoutSpecPayload _RETRY_RUNTIME_FIELDS = ( "move_to_library", @@ -21,21 +24,66 @@ def build_retry_import_request(original: Any) -> ImportJobCreate: """Build the creation request for a fresh retry job.""" + require_retained_import_destination(original) + source_layout_snapshot = getattr(original, "source_layout_snapshot", None) or {} + future_root_policy_snapshot = getattr(original, "future_root_policy_snapshot", None) + frozen_path_map = dict(original.mylar3_path_map or {}) return ImportJobCreate( source_path=original.source_path, file_paths=list(original.selected_file_paths or []) or None, source_type=original.source_type, target_library_root_id=original.target_library_root_id, monitored=original.monitored, - mylar3_path_map=dict(original.mylar3_path_map or {}), + mylar3_path_map=frozen_path_map, + mylar3_path_map_confirmed=bool( + frozen_path_map or getattr(original, "mylar3_path_map_confirmed", False) + ), cv_match_threshold=original.cv_match_threshold, min_files_per_series=original.min_files_per_series, file_formats=original.file_formats, + file_handling_mode=getattr( + original, + "file_handling_mode", + ImportFileHandlingMode.MANAGED_COPY, + ), + source_layout=SourceLayoutSpecPayload.model_validate(source_layout_snapshot), + future_layout_requested=bool(getattr(original, "future_layout_requested", False)), + future_root_policy=future_root_policy_snapshot, + story_arc_import_requested=bool(getattr(original, "story_arc_import_requested", False)), + story_arc_materialization_requested=bool( + getattr(original, "story_arc_materialization_requested", False) + ), ) +def require_retained_import_destination(job: Any) -> None: + """Never replace an explicitly removed historical destination with a default.""" + if getattr(job, "removed_library_root_snapshot", None) and job.target_library_root_id is None: + raise ValidationError( + "This import's library root was removed. Start a new import and explicitly " + "select an enabled library root. Existing library files and import history " + "are unchanged." + ) + + def copy_retry_import_settings(original: Any, retry: Any) -> None: """Copy runtime import policy fields onto the fresh retry job.""" for field_name in _RETRY_RUNTIME_FIELDS: setattr(retry, field_name, getattr(original, field_name)) retry.ingest_policy_snapshot = dict(original.ingest_policy_snapshot or {}) + retry.file_handling_mode = getattr( + original, + "file_handling_mode", + ImportFileHandlingMode.MANAGED_COPY, + ) + retry.source_layout_snapshot = dict(getattr(original, "source_layout_snapshot", None) or {}) + retry.future_layout_requested = bool(getattr(original, "future_layout_requested", False)) + future_root_policy_snapshot = getattr(original, "future_root_policy_snapshot", None) + retry.future_root_policy_snapshot = ( + dict(future_root_policy_snapshot) if future_root_policy_snapshot is not None else None + ) + retry.future_root_policy_applied_at = None + retry.story_arc_import_requested = bool(getattr(original, "story_arc_import_requested", False)) + retry.story_arc_materialization_requested = bool( + getattr(original, "story_arc_materialization_requested", False) + ) diff --git a/src/pullbox/services/import_review_actions.py b/src/pullbox/services/import_review_actions.py index 0e3f45fe..d956584d 100644 --- a/src/pullbox/services/import_review_actions.py +++ b/src/pullbox/services/import_review_actions.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from datetime import UTC, datetime from sqlalchemy import or_ @@ -20,11 +20,63 @@ ImportSeriesStatus, ) from pullbox.services.import_duplicates import duplicate_merge_is_actionable, is_duplicate_series +from pullbox.services.import_safety_diagnostics import normalize_import_safety_diagnostics +from pullbox.services.import_story_arc_resolution import ( + refresh_story_arc_entries_for_import_files, +) RecomputeFileCounters = Callable[[AsyncSession, ImportJob, list[int]], Awaitable[None]] RecomputeSeriesCounters = Callable[[AsyncSession, ImportJob], Awaitable[None]] +def prepare_series_for_safety_rematch(imported_series: ImportedSeries) -> bool: + """Restore a proven series target and mark it for safety-file rematching.""" + if imported_series.status == ImportSeriesStatus.SKIPPED: + if imported_series.series_id is not None: + imported_series.status = ImportSeriesStatus.DUPLICATE + elif imported_series.user_selected_cv_id is not None or imported_series.cv_id is not None: + imported_series.status = ImportSeriesStatus.MATCHED + + if imported_series.status not in { + ImportSeriesStatus.MATCHED, + ImportSeriesStatus.DUPLICATE, + }: + return False + + diagnostics = dict(imported_series.diagnostics or {}) + diagnostics["rematch_pending"] = True + imported_series.diagnostics = diagnostics + imported_series.selected_for_import = False + return True + + +def apply_safety_allow_once_to_file( + imp_file: ImportedFile, + *, + retry_import: bool = False, + allowed_at: datetime | None = None, +) -> None: + """Apply the canonical one-job safety exception payload to one staged file.""" + diagnostics = dict(imp_file.diagnostics or {}) + previous_block = diagnostics.pop("safety_block", None) + if not isinstance(previous_block, Mapping): + raise ValidationError("This safety block cannot be overridden.") + normalized_previous_block = normalize_import_safety_diagnostics(previous_block) + if normalized_previous_block["overrideable"] is not True: + raise ValidationError("This safety block cannot be overridden.") + diagnostics["safety_exception"] = { + "allowed_once": True, + "allowed_at": (allowed_at or datetime.now(UTC)).isoformat(), + "previous_block": normalized_previous_block, + } + imp_file.status = ( + ImportedFileStatus.CONFIRMED if retry_import else ImportedFileStatus.SAFETY_APPROVED + ) + imp_file.include_in_import = bool(retry_import) + imp_file.error_message = None + imp_file.diagnostics = diagnostics + + async def resolve_conflict( session: AsyncSession, job_id: int, @@ -304,21 +356,14 @@ async def allow_safety_blocked_file_once( allow_terminal_job=retry_import, ) - diagnostics = dict(imp_file.diagnostics or {}) - previous_block = diagnostics.pop("safety_block", None) - if isinstance(previous_block, dict) and previous_block.get("overrideable") is False: - raise ValidationError("This safety block cannot be overridden.") - diagnostics["safety_exception"] = { - "allowed_once": True, - "allowed_at": datetime.now(UTC).isoformat(), - "previous_block": previous_block, - } - imp_file.status = ( - ImportedFileStatus.CONFIRMED if retry_import else ImportedFileStatus.SAFETY_APPROVED + apply_safety_allow_once_to_file(imp_file, retry_import=retry_import) + if not retry_import: + prepare_series_for_safety_rematch(imported_series) + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job_id, + import_file_ids=[imp_file.id], ) - imp_file.include_in_import = bool(retry_import) - imp_file.error_message = None - imp_file.diagnostics = diagnostics imported_series.selected_for_import = bool(retry_import) if retry_import and imported_series.status in { ImportSeriesStatus.IMPORTED, @@ -348,6 +393,22 @@ async def skip_safety_blocked_file( """Skip a safety-blocked file from the active import review.""" job, imported_series, imp_file = await _load_safety_blocked_file(session, job_id, file_id) + apply_safety_skip_to_file(imp_file) + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job_id, + import_file_ids=[imp_file.id], + ) + imported_series.selected_for_import = False + + await recompute_file_counters(session, job, [imported_series.id]) + await recompute_series_counters(session, job) + await session.flush() + return imp_file + + +def apply_safety_skip_to_file(imp_file: ImportedFile) -> None: + """Apply the source-preserving skip mutation to one reviewed file.""" diagnostics = dict(imp_file.diagnostics or {}) imp_file.status = ImportedFileStatus.SKIPPED imp_file.include_in_import = False @@ -365,12 +426,6 @@ async def skip_safety_blocked_file( "kind": "file_safety_review", "resolution": "skipped", } - imported_series.selected_for_import = False - - await recompute_file_counters(session, job, [imported_series.id]) - await recompute_series_counters(session, job) - await session.flush() - return imp_file def _series_has_match_target(imported_series: ImportedSeries) -> bool: @@ -493,6 +548,7 @@ async def _unmatch_series_target( imp_file.diagnostics = { **existing_diagnostics, "kind": "file_no_match", + "reason": reason, "target_state": "needs_series_match", "rejection_reason": rejection_reason, "previous_match": previous_file_match, diff --git a/src/pullbox/services/import_review_queries.py b/src/pullbox/services/import_review_queries.py index c114642c..49293302 100644 --- a/src/pullbox/services/import_review_queries.py +++ b/src/pullbox/services/import_review_queries.py @@ -2,12 +2,15 @@ from __future__ import annotations +from dataclasses import dataclass from typing import TYPE_CHECKING, Any +from sqlalchemy import Integer, String, and_, case, cast, literal, null, union_all from sqlalchemy import func as sa_func from sqlalchemy import select as sa_select +from sqlalchemy.orm import aliased -from pullbox.core.exceptions import NotFoundError +from pullbox.core.exceptions import NotFoundError, ValidationError from pullbox.models.import_job import ( ImportedFile, ImportedFileStatus, @@ -15,9 +18,40 @@ ImportJob, ImportSeriesStatus, ) +from pullbox.models.issue import Issue if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.selectable import CTE, Subquery + + +CONFLICT_GROUP_COMPATIBILITY_PAGE_SIZE = 500 +MAX_CONFLICT_GROUP_PAGE_SIZE = 100 +MAX_CONFLICT_GROUP_FILES = 100 +CONFLICT_SORT_FIELDS = frozenset({"series", "conflict", "files", "signal", "status"}) + + +@dataclass(frozen=True, slots=True) +class ConflictGroupsPage: + """One bounded, deterministic page of import conflict groups.""" + + items: tuple[dict[str, Any], ...] + total: int + page: int + page_size: int + auto_resolved: int = 0 + needs_decision: int = 0 + series_candidate_conflicts: int = 0 + file_conflict_groups: int = 0 + + +@dataclass(frozen=True, slots=True) +class _ConflictGroupCounts: + total: int + auto_resolved: int + needs_decision: int + series_candidate_conflicts: int + file_conflict_groups: int async def get_files_for_series( @@ -64,72 +98,524 @@ async def get_conflict_groups( session: AsyncSession, job_id: int, ) -> list[dict[str, Any]]: - """Return all file and series conflict groups for a job.""" + """Return all conflict groups while preserving the legacy API contract. + + New request paths should use :func:`get_conflict_groups_page`. This wrapper + walks bounded server-side pages so existing callers keep complete results + without the former per-series N+1 query pattern. + """ job = await session.get(ImportJob, job_id) if job is None: raise NotFoundError("ImportJob", job_id) - query = ( - sa_select(ImportedFile) + group_keys = _conflict_group_keys(job_id) + counts = await _count_conflict_groups(session, group_keys) + groups: list[dict[str, Any]] = [] + offset = 0 + while offset < counts.total: + page_items = await _load_conflict_group_slice( + session, + job_id, + group_keys, + offset=offset, + limit=CONFLICT_GROUP_COMPATIBILITY_PAGE_SIZE, + max_files_per_group=None, + ) + if not page_items: + break + groups.extend(page_items) + offset += CONFLICT_GROUP_COMPATIBILITY_PAGE_SIZE + return groups + + +async def get_conflict_groups_page( + session: AsyncSession, + job_id: int, + *, + page: int = 1, + page_size: int = 25, + sort: str = "legacy", +) -> ConflictGroupsPage: + """Return one server-paginated conflict-group page without N+1 queries.""" + if isinstance(page, bool) or page < 1: + raise ValidationError("Conflict review page must be at least 1") + if isinstance(page_size, bool) or page_size < 1 or page_size > MAX_CONFLICT_GROUP_PAGE_SIZE: + raise ValidationError( + f"Conflict review page_size must be between 1 and {MAX_CONFLICT_GROUP_PAGE_SIZE}" + ) + if sort != "legacy" and sort.removeprefix("-") not in CONFLICT_SORT_FIELDS: + raise ValidationError("Unsupported conflict review sort") + + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + + group_keys = _conflict_group_keys(job_id) + counts = await _count_conflict_groups(session, group_keys) + total_pages = max(1, (counts.total + page_size - 1) // page_size) + current_page = min(page, total_pages) + items = await _load_conflict_group_slice( + session, + job_id, + group_keys, + offset=(current_page - 1) * page_size, + limit=page_size, + sort=sort, + max_files_per_group=MAX_CONFLICT_GROUP_FILES, + ) + return ConflictGroupsPage( + items=tuple(items), + total=counts.total, + page=current_page, + page_size=page_size, + auto_resolved=counts.auto_resolved, + needs_decision=counts.needs_decision, + series_candidate_conflicts=counts.series_candidate_conflicts, + file_conflict_groups=counts.file_conflict_groups, + ) + + +def _conflict_group_keys(job_id: int) -> Subquery: + """Build the portable sort-key relation shared by counts and pages.""" + diagnostics_kind = ImportedSeries.diagnostics["kind"].as_string() + series_file_counts = ( + sa_select( + ImportedFile.import_series_id.label("series_id"), + sa_func.count(ImportedFile.id).label("file_count"), + ) + .where(ImportedFile.import_job_id == job_id) + .group_by(ImportedFile.import_series_id) + .subquery() + ) + series_label = _series_label_expression( + ImportedSeries.raw_series_name, + ImportedSeries.raw_year, + ) + selected_candidate_title = ImportedSeries.diagnostics["selected_candidate"]["title"].as_string() + series_keys = ( + sa_select( + literal(0).label("kind_order"), + literal("series_conflict").label("kind"), + ImportedSeries.id.label("group_key"), + cast(ImportedSeries.id, String).label("sort_key"), + series_label.label("series_sort"), + literal(1).label("issue_null_order"), + literal(0.0).label("issue_sort"), + sa_func.coalesce(series_file_counts.c.file_count, 0).label("file_count"), + literal(0).label("has_preferred"), + sa_func.coalesce(selected_candidate_title, "candidate needs review").label( + "signal_sort" + ), + literal("series match conflict").label("status_sort"), + ImportedSeries.id.label("series_id"), + cast(null(), Integer).label("matched_issue_id"), + ) + .where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status == ImportSeriesStatus.NO_MATCH, + diagnostics_kind == "series_conflict", + ) + .outerjoin( + series_file_counts, + series_file_counts.c.series_id == ImportedSeries.id, + ) + ) + + file_aggregates = ( + sa_select( + ImportedFile.conflict_group_id.label("group_key"), + sa_func.min(ImportedFile.id).label("first_file_id"), + sa_func.count(ImportedFile.id).label("file_count"), + sa_func.max(case((ImportedFile.is_preferred.is_(True), 1), else_=0)).label( + "has_preferred" + ), + sa_func.min(ImportedFile.parsed_issue_number).label("parsed_issue_number"), + ) .where( ImportedFile.import_job_id == job_id, ImportedFile.status == ImportedFileStatus.CONFLICT, ImportedFile.conflict_group_id.is_not(None), ) - .order_by(ImportedFile.conflict_group_id.asc(), ImportedFile.id.asc()) + .group_by(ImportedFile.conflict_group_id) + .subquery() ) - result = await session.execute(query) - conflict_files = list(result.scalars().all()) + first_file = aliased(ImportedFile) + file_series = aliased(ImportedSeries) + + issue_sort = sa_func.coalesce(Issue.issue_number, file_aggregates.c.parsed_issue_number) + file_series_label = _series_label_expression( + file_series.raw_series_name, + file_series.raw_year, + ) + file_keys = ( + sa_select( + literal(1).label("kind_order"), + literal("file_conflict").label("kind"), + file_aggregates.c.group_key, + cast(file_aggregates.c.group_key, String).label("sort_key"), + file_series_label.label("series_sort"), + case((issue_sort.is_(None), 1), else_=0).label("issue_null_order"), + sa_func.coalesce(issue_sort, 0.0).label("issue_sort"), + file_aggregates.c.file_count, + file_aggregates.c.has_preferred, + case( + (file_aggregates.c.has_preferred == 1, "auto-selected"), + else_="needs choice", + ).label("signal_sort"), + case( + (file_aggregates.c.has_preferred == 1, "auto-selected"), + else_="needs choice", + ).label("status_sort"), + first_file.import_series_id.label("series_id"), + first_file.matched_issue_id.label("matched_issue_id"), + ) + .select_from(file_aggregates) + .join(first_file, first_file.id == file_aggregates.c.first_file_id) + .join(file_series, file_series.id == first_file.import_series_id) + .outerjoin(Issue, Issue.id == first_file.matched_issue_id) + ) + return _normalized_sort_keys(union_all(series_keys, file_keys).subquery()) + + +def _series_label_expression(name: Any, year: Any) -> Any: + return name + case( + (year.is_not(None), literal(" (") + cast(year, String) + literal(")")), + else_="", + ) + + +def _sort_key_stage(keys: CTE | Subquery, expressions: dict[str, Any]) -> CTE: + """Name intermediate expressions without nesting their SQL text.""" + return sa_select( + *( + expressions[column.key].label(column.key) if column.key in expressions else column + for column in keys.c + ) + ).cte() - groups: dict[int, list[ImportedFile]] = {} - for imp_file in conflict_files: - group_id = imp_file.conflict_group_id - assert group_id is not None - groups.setdefault(group_id, []).append(imp_file) - file_groups = [ +def _normalized_sort_keys(keys: Subquery) -> Subquery: + """Keep portable, server-side normalization within fixed SQLite parser stacks.""" + names = ("series_sort", "signal_sort") + stage = _sort_key_stage( + keys, + {name: sa_func.lower(sa_func.trim(sa_func.coalesce(keys.c[name], ""))) for name in names}, + ) + stage = _sort_key_stage( + stage, { - "kind": "file_conflict", - "conflict_group_id": group_id, - "matched_issue_id": files[0].matched_issue_id, - "files": files, - } - for group_id, files in groups.items() - ] + name: case( + (stage.c[name].like("the %"), sa_func.substr(stage.c[name], 5)), + (stage.c[name].like("an %"), sa_func.substr(stage.c[name], 4)), + (stage.c[name].like("a %"), sa_func.substr(stage.c[name], 3)), + else_=stage.c[name], + ) + for name in names + }, + ) + replacements = [("&", " and "), ("'s", "s")] + replacements.extend( + (char, " ") for char in ("-", "_", ".", ",", ":", ";", "!", "?", "'", '"', "(", ")") + ) + replacements.extend([(" ", " ")] * 4) + # CTEs are hoisted rather than nesting SELECTs. Four replacements per stage + # leave parser headroom for CASE, JSON extraction, UNION, and page/count SQL. + for offset in range(0, len(replacements), 4): + expressions: dict[str, Any] = {} + for name in names: + value: Any = stage.c[name] + for old, new in replacements[offset : offset + 4]: + value = sa_func.replace(value, old, new) + expressions[name] = value + stage = _sort_key_stage(stage, expressions) + stage = _sort_key_stage( + stage, + { + "series_sort": sa_func.trim(stage.c.series_sort), + # File-conflict signal labels historically equal status_sort and + # are not title-normalized (notably the hyphen in auto-selected). + "signal_sort": case( + (stage.c.kind_order == 1, stage.c.status_sort), + else_=sa_func.trim(stage.c.signal_sort), + ), + }, + ) + return sa_select(stage).subquery() - series_result = await session.execute( - sa_select(ImportedSeries) - .where( - ImportedSeries.import_job_id == job_id, - ImportedSeries.status == ImportSeriesStatus.NO_MATCH, + +async def _count_conflict_groups( + session: AsyncSession, + group_keys: Subquery, +) -> _ConflictGroupCounts: + row = ( + await session.execute( + sa_select( + sa_func.count().label("total"), + sa_func.coalesce( + sa_func.sum(case((group_keys.c.kind_order == 0, 1), else_=0)), 0 + ).label("series_candidate_conflicts"), + sa_func.coalesce( + sa_func.sum(case((group_keys.c.kind_order == 1, 1), else_=0)), 0 + ).label("file_conflict_groups"), + sa_func.coalesce( + sa_func.sum( + case( + ( + and_( + group_keys.c.kind_order == 1, + group_keys.c.has_preferred == 1, + ), + 1, + ), + else_=0, + ) + ), + 0, + ).label("auto_resolved"), + sa_func.coalesce( + sa_func.sum( + case( + ( + and_( + group_keys.c.kind_order == 1, + group_keys.c.has_preferred == 0, + ), + 1, + ), + else_=0, + ) + ), + 0, + ).label("needs_decision"), + ).select_from(group_keys) ) - .order_by(ImportedSeries.id.asc()) + ).one() + return _ConflictGroupCounts( + total=int(row.total or 0), + auto_resolved=int(row.auto_resolved or 0), + needs_decision=int(row.needs_decision or 0), + series_candidate_conflicts=int(row.series_candidate_conflicts or 0), + file_conflict_groups=int(row.file_conflict_groups or 0), ) - series_conflicts: list[dict[str, Any]] = [] - for imp_series in series_result.scalars().all(): - diagnostics = dict(imp_series.diagnostics or {}) - if diagnostics.get("kind") != "series_conflict": + + +async def _load_conflict_group_slice( + session: AsyncSession, + job_id: int, + group_keys: Subquery, + *, + offset: int, + limit: int, + sort: str = "legacy", + max_files_per_group: int | None = MAX_CONFLICT_GROUP_FILES, +) -> list[dict[str, Any]]: + """Load one bounded group-key slice and all files for only those groups.""" + key_rows = ( + await session.execute( + sa_select( + group_keys.c.kind, + group_keys.c.group_key, + group_keys.c.file_count, + group_keys.c.series_id, + group_keys.c.matched_issue_id, + ) + .order_by(*_conflict_group_order(group_keys, sort)) + .offset(offset) + .limit(limit) + ) + ).all() + if not key_rows: + return [] + + ordered_keys = [(str(row.kind), int(row.group_key)) for row in key_rows] + file_count_by_key = { + (str(row.kind), int(row.group_key)): int(row.file_count or 0) for row in key_rows + } + series_ids = [int(row.series_id) for row in key_rows if row.series_id is not None] + conflict_series_ids = [ + group_key for kind, group_key in ordered_keys if kind == "series_conflict" + ] + file_group_ids = [group_key for kind, group_key in ordered_keys if kind == "file_conflict"] + + series_by_id: dict[int, ImportedSeries] = {} + if series_ids: + series_result = await session.execute( + sa_select(ImportedSeries).where(ImportedSeries.id.in_(series_ids)) + ) + series_by_id = {int(item.id): item for item in series_result.scalars().all()} + + files_by_series_id: dict[int, list[ImportedFile]] = {} + files_by_group_id: dict[int, list[ImportedFile]] = {} + if conflict_series_ids or file_group_ids: + file_rows = await _load_bounded_group_files( + session, + job_id, + conflict_series_ids=conflict_series_ids, + file_group_ids=file_group_ids, + max_files_per_group=max_files_per_group, + ) + for imp_file, kind, group_key in file_rows: + if kind == "series_conflict": + files_by_series_id.setdefault(group_key, []).append(imp_file) + group_id = imp_file.conflict_group_id + if ( + kind == "file_conflict" + and group_id is not None + and group_id in file_group_ids + and imp_file.status == ImportedFileStatus.CONFLICT + ): + files_by_group_id.setdefault(group_id, []).append(imp_file) + + groups: list[dict[str, Any]] = [] + for kind, group_key in ordered_keys: + if kind == "series_conflict": + imp_series = series_by_id.get(group_key) + if imp_series is None: + continue + groups.append( + { + "kind": "series_conflict", + "conflict_group_id": f"series-{imp_series.id}", + "series_id": imp_series.id, + "matched_issue_id": None, + "files": files_by_series_id.get(group_key, []), + "file_count": file_count_by_key[(kind, group_key)], + "files_truncated": ( + file_count_by_key[(kind, group_key)] + > len(files_by_series_id.get(group_key, [])) + ), + "series": imp_series, + "diagnostics": dict(imp_series.diagnostics or {}), + } + ) + continue + + files = files_by_group_id.get(group_key, []) + if not files: continue - files_result = await session.execute( - sa_select(ImportedFile) - .where(ImportedFile.import_series_id == imp_series.id) - .order_by(ImportedFile.id.asc()) + key_row = next( + row for row in key_rows if str(row.kind) == kind and int(row.group_key) == group_key ) - series_conflicts.append( + series_id = int(key_row.series_id) + groups.append( { - "kind": "series_conflict", - "conflict_group_id": f"series-{imp_series.id}", - "series_id": imp_series.id, - "matched_issue_id": None, - "files": list(files_result.scalars().all()), - "diagnostics": diagnostics, + "kind": "file_conflict", + "conflict_group_id": group_key, + "matched_issue_id": files[0].matched_issue_id, + "files": files, + "file_count": file_count_by_key[(kind, group_key)], + "files_truncated": file_count_by_key[(kind, group_key)] > len(files), + "series_id": series_id, + "series": series_by_id.get(series_id), } ) + return groups + + +def _conflict_group_order(group_keys: Subquery, sort: str) -> tuple[Any, ...]: + if sort == "legacy": + return (group_keys.c.kind_order.asc(), group_keys.c.sort_key.asc()) - return sorted( - [*series_conflicts, *file_groups], - key=lambda group: ( - 0 if group.get("kind") == "series_conflict" else 1, - str(group.get("conflict_group_id")), - ), + descending = sort.startswith("-") + field = sort.removeprefix("-") + default_columns = ( + group_keys.c.series_sort, + group_keys.c.kind_order, + group_keys.c.issue_null_order, + group_keys.c.issue_sort, + group_keys.c.sort_key, ) + columns: tuple[Any, ...] + match field: + case "conflict": + columns = ( + group_keys.c.kind_order, + group_keys.c.issue_null_order, + group_keys.c.issue_sort, + *default_columns, + ) + case "files": + columns = (group_keys.c.file_count, *default_columns) + case "signal": + columns = (group_keys.c.signal_sort, *default_columns) + case "status": + columns = (group_keys.c.status_sort, *default_columns) + case _: + columns = default_columns + if descending: + return tuple(column.desc() for column in columns) + return tuple(column.asc() for column in columns) + + +async def _load_bounded_group_files( + session: AsyncSession, + job_id: int, + *, + conflict_series_ids: list[int], + file_group_ids: list[int], + max_files_per_group: int | None, +) -> list[tuple[ImportedFile, str, int]]: + candidates: list[Any] = [] + if conflict_series_ids: + candidates.append( + sa_select( + ImportedFile.id.label("file_id"), + literal("series_conflict").label("kind"), + ImportedFile.import_series_id.label("group_key"), + ).where( + ImportedFile.import_job_id == job_id, + ImportedFile.import_series_id.in_(conflict_series_ids), + ) + ) + if file_group_ids: + candidates.append( + sa_select( + ImportedFile.id.label("file_id"), + literal("file_conflict").label("kind"), + ImportedFile.conflict_group_id.label("group_key"), + ).where( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.CONFLICT, + ImportedFile.conflict_group_id.in_(file_group_ids), + ) + ) + if not candidates: + return [] + + candidate_rows = union_all(*candidates).subquery() + if max_files_per_group is None: + selected_rows = candidate_rows + else: + ranked_rows = sa_select( + candidate_rows.c.file_id, + candidate_rows.c.kind, + candidate_rows.c.group_key, + sa_func.row_number() + .over( + partition_by=(candidate_rows.c.kind, candidate_rows.c.group_key), + order_by=candidate_rows.c.file_id.asc(), + ) + .label("group_row_number"), + ).subquery() + selected_rows = ( + sa_select( + ranked_rows.c.file_id, + ranked_rows.c.kind, + ranked_rows.c.group_key, + ) + .where(ranked_rows.c.group_row_number <= max_files_per_group) + .subquery() + ) + + rows = ( + await session.execute( + sa_select(ImportedFile, selected_rows.c.kind, selected_rows.c.group_key) + .join(selected_rows, selected_rows.c.file_id == ImportedFile.id) + .order_by( + selected_rows.c.kind.asc(), + selected_rows.c.group_key.asc(), + ImportedFile.id.asc(), + ) + ) + ).all() + return [(row[0], str(row[1]), int(row[2])) for row in rows] diff --git a/src/pullbox/services/import_review_recheck.py b/src/pullbox/services/import_review_recheck.py new file mode 100644 index 00000000..29f2d86e --- /dev/null +++ b/src/pullbox/services/import_review_recheck.py @@ -0,0 +1,847 @@ +"""Targeted offline recheck of staged import evidence, never source mutations.""" + +from __future__ import annotations + +import asyncio +from dataclasses import asdict +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from sqlalchemy import and_, exists, func, or_, select + +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.file_safety import ( + FileSafetyError, + get_archive_size_limit_bytes, + is_dangerous_file_blocking_enabled, + run_safety_checks, +) +from pullbox.core.filesystem_policy import is_invalid_path_text, resolve_preview_source +from pullbox.core.library_file_ownership import ( + ReferencedFileValidationError, + build_file_identity_signature, + validate_file_identity_signature, +) +from pullbox.core.source_metadata import MetadataSignal, SourceMetadata, SourceMetadataExtractor +from pullbox.models.import_job import ( + ImportControlRequest, + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportFileHandlingMode, + ImportJob, + ImportJobLog, + ImportJobStatus, + ImportSeriesStatus, + ImportSourceType, +) +from pullbox.models.library import LibraryRoot +from pullbox.services.import_content_inspection import inspect_import_content +from pullbox.services.import_safety_diagnostics import ( + ImportSafetyCategory, + build_import_safety_diagnostics, +) +from pullbox.services.import_series_match_state import clear_auto_cv_match_fields +from pullbox.services.import_source_metadata import source_metadata_for_import_file +from pullbox.services.import_terminal_recovery import allows_terminal_import_recovery + +if TYPE_CHECKING: + from collections.abc import Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + + +_TRANSIENT_SOURCE_RECHECK_CATEGORIES = ( + ImportSafetyCategory.PERMISSION_UNREADABLE.value, + ImportSafetyCategory.ARCHIVE_INSPECTION_FAILED.value, +) +_TRANSIENT_SOURCE_RECHECK_CODES = ( + "permission_unreadable", + "permission_denied", + "source_unreadable", + "unreadable", + "archive_inspection_failed", + "corrupt_archive", + "inspection_failed", + "source_changed", + "source_signature_missing", + "source_signature_unsupported", + "source_unavailable", +) + + +def retryable_failed_source_filters(job_id: int) -> tuple[Any, ...]: + """Select only transient source failures that another inspection can resolve.""" + category = ImportedFile.diagnostics["source_revalidation"]["category"].as_string() + code = ImportedFile.diagnostics["source_revalidation"]["code"].as_string() + return ( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.FAILED, + ImportedFile.diagnostics["source_revalidation"]["retryable"].as_boolean().is_(True), + or_( + category.in_(_TRANSIENT_SOURCE_RECHECK_CATEGORIES), + code.in_(_TRANSIENT_SOURCE_RECHECK_CODES), + and_( + category == ImportSafetyCategory.SOURCE_CHANGED.value, + code.is_(None), + ), + ), + ) + + +def _preserve_mylar_identity(base: SourceMetadata, fresh: SourceMetadata) -> SourceMetadata: + updates: dict[str, Any] = {} + signals = dict(fresh.signals) + diagnostics = dict(fresh.diagnostics) + raw_conflicts = diagnostics.get("identity_conflicts") + conflicts = list(raw_conflicts) if isinstance(raw_conflicts, list) else [] + for field in ("comicvine_series_id", "comicvine_issue_id", "issue_type"): + value = getattr(base, field) + if base.signals.get(field) != MetadataSignal.MYLAR3 or value is None: + continue + current = getattr(fresh, field) + if field != "issue_type" and current is not None and current != value: + conflicts.append({"field": field, "mylar3": value, "source": current}) + updates[field] = value + signals[field] = MetadataSignal.MYLAR3 + for key in ("mylar3_issue", "mylar3_path_reconciliation", "source_layout"): + if key in base.diagnostics: + diagnostics[key] = base.diagnostics[key] + if conflicts: + diagnostics["identity_conflicts"] = conflicts + return fresh.model_copy(update={**updates, "signals": signals, "diagnostics": diagnostics}) + + +def inspect_review_source( + path: Path, + base: SourceMetadata, + signature: dict[str, Any], + *, + roots: list[tuple[Path, Path]], + block_dangerous: bool, + max_archive_size: int, + accept_replaced_files: bool, + sidecars: dict[str, dict[str, Any]], +) -> tuple[SourceMetadata, dict[str, Any], dict[str, int | str]]: + """Inspect only explicitly permitted paths; no providers or page extraction.""" + current_signature: dict[str, int | str] = {} + fresh = base + try: + if is_invalid_path_text(str(path)) or ".." in path.parts: + raise ReferencedFileValidationError("source_path_unsafe", "Unsafe source path") + lexical = path.expanduser().absolute() + resolved = path.expanduser().resolve(strict=False) + if not any( + lexical.is_relative_to(root) and resolved.is_relative_to(real) for root, real in roots + ): + raise ReferencedFileValidationError("source_outside_root", "Outside approved root") + resolve_preview_source(path) + current_signature = build_file_identity_signature(path) + if not accept_replaced_files: + validate_file_identity_signature(signature, current_signature) + extractor = SourceMetadataExtractor() + folder = str(path.parent) + if folder not in sidecars: + sidecars[folder] = extractor.read_sidecars(path.parent) + fresh = _preserve_mylar_identity( + base, + extractor.from_path( + path, + include_archive_comicinfo=False, + include_archive_entry_issue_hint=False, + sidecar_data=sidecars[folder], + ), + ) + inspection = run_safety_checks( + path, block_dangerous=block_dangerous, max_archive_size=max_archive_size + ) + content = inspect_import_content(path, inspection) + report = next((r for r in inspection.archives if r.archive_path == path), None) + evidence = ( + None + if report is None + else { + "member_index_scanned": True, + "comicinfo_entry_count": report.comicinfo_entry_count, + "comicinfo": asdict(report.comicinfo) if report.comicinfo is not None else None, + "comicinfo_entry": report.comicinfo_entry, + "comicinfo_error": report.comicinfo_error, + } + ) + fresh = _preserve_mylar_identity( + base, + extractor.from_path( + path, + sidecar_data=sidecars[folder], + archive_member_evidence=evidence, + ), + ) + # Do not accept evidence from a file replaced during the inspection. + validate_file_identity_signature( + dict(current_signature), build_file_identity_signature(path) + ) + return fresh, content, current_signature + except (ReferencedFileValidationError, FileSafetyError, OSError, ValueError) as exc: + code = exc.reason if isinstance(exc, ReferencedFileValidationError) else None + if isinstance(exc, OSError): + code = "source_unreadable" if isinstance(exc, PermissionError) else "source_missing" + reason = exc.reason if isinstance(exc, FileSafetyError) else str(code or "source_missing") + return ( + fresh, + { + "file_safety": build_import_safety_diagnostics( + reason, code=code, source="review_recheck" + ) + }, + {}, + ) + + +def _apply_file( + file: ImportedFile, + metadata: SourceMetadata, + content: dict[str, Any], + signature: dict[str, int | str], +) -> None: + diagnostics: dict[str, Any] = {} + source = {**metadata.diagnostics, **content} + block = source.pop("file_safety", None) + diagnostics.update( + { + "source_metadata": source, + "source_issue_type": metadata.issue_type.value, + "comicvine_series_id": metadata.comicvine_series_id, + "metadata_signals": {key: value.value for key, value in metadata.signals.items()}, + } + ) + file.parsed_series = metadata.series_name + file.parsed_issue_number = metadata.issue_number + file.parsed_year = metadata.year + file.comicvine_issue_id = metadata.comicvine_issue_id + file.has_comicinfo = bool(source.get("has_comicinfo")) + file.include_in_import = False + file.matched_issue_id = None + file.matched_issue_cv_id = None + file.match_confidence = None + file.match_method = None + file.error_message = None + file.status = ImportedFileStatus.PENDING + if signature: + file.source_signature = signature + file.file_size = int(signature["size"]) + if isinstance(block, dict): + diagnostics["safety_block"] = block + file.status = ImportedFileStatus.SAFETY_BLOCKED + file.error_message = str(block["reason"]) + file.diagnostics = diagnostics + + +def _apply_completed_file_recheck( + file: ImportedFile, + metadata: SourceMetadata, + content: dict[str, Any], + signature: dict[str, int | str], + *, + reviewed_series_cv_id: int | None, +) -> bool: + """Refresh source evidence without changing the saved import decision.""" + diagnostics = dict(file.diagnostics or {}) + previous_signature = dict(file.source_signature or {}) + source = {**metadata.diagnostics, **content} + block = _completed_file_recheck_block( + file, + metadata, + source, + reviewed_series_cv_id=reviewed_series_cv_id, + ) + source.pop("file_safety", None) + checked_at = datetime.now(UTC).isoformat() + if block is not None: + diagnostics["source_revalidation"] = { + **block, + "kind": "source_revalidation", + "source": "completed_import_recheck", + } + diagnostics["source_recheck"] = { + "checked_at": checked_at, + "ready_for_retry": False, + } + file.error_message = str(block.get("sanitized_reason") or block.get("reason")) + file.diagnostics = diagnostics + return False + + diagnostics.pop("source_revalidation", None) + diagnostics.update( + { + "source_metadata": source, + "source_issue_type": metadata.issue_type.value, + "comicvine_series_id": metadata.comicvine_series_id, + "metadata_signals": {key: value.value for key, value in metadata.signals.items()}, + "source_recheck": { + "checked_at": checked_at, + "ready_for_retry": True, + }, + } + ) + file.parsed_series = metadata.series_name + file.parsed_issue_number = metadata.issue_number + file.parsed_year = metadata.year + file.comicvine_issue_id = metadata.comicvine_issue_id + file.has_comicinfo = bool(source.get("has_comicinfo")) + refreshed_signature = dict(signature) + reference_root_id = previous_signature.get("mylar_reference_root_id") + if ( + isinstance(reference_root_id, int) + and not isinstance(reference_root_id, bool) + and reference_root_id > 0 + ): + refreshed_signature["mylar_reference_root_id"] = reference_root_id + file.source_signature = refreshed_signature + file.file_size = int(signature["size"]) + file.error_message = "Source rechecked and ready to retry." + file.diagnostics = diagnostics + return True + + +def _completed_file_recheck_block( + file: ImportedFile, + metadata: SourceMetadata, + source: dict[str, Any], + *, + reviewed_series_cv_id: int | None, +) -> dict[str, Any] | None: + """Return the final safety block after archive and identity checks.""" + raw_block = source.get("file_safety") + block = dict(raw_block) if isinstance(raw_block, dict) else None + identity_conflicts = source.get("identity_conflicts") + saved_target_conflicts = _saved_target_identity_conflicts( + file, + metadata, + reviewed_series_cv_id=reviewed_series_cv_id, + ) + if block is None and isinstance(identity_conflicts, list) and identity_conflicts: + block = build_import_safety_diagnostics( + "The current source identity conflicts with the file reviewed during import.", + kind="source_revalidation", + code="source_identity_changed", + source="completed_import_recheck", + overrideable_hint=False, + ) + if block is None and saved_target_conflicts: + block = build_import_safety_diagnostics( + "The replacement source does not match the issue reviewed during import.", + kind="source_revalidation", + code="source_identity_changed", + source="completed_import_recheck", + overrideable_hint=False, + ) + block["identity_conflicts"] = saved_target_conflicts + return block + + +def _saved_target_identity_conflicts( + file: ImportedFile, + metadata: SourceMetadata, + *, + reviewed_series_cv_id: int | None, +) -> list[dict[str, int | float | str]]: + """Compare fresh replacement evidence with the immutable reviewed target.""" + diagnostics = dict(file.diagnostics or {}) + raw_summary = diagnostics.get("target_issue_summary") + summary = raw_summary if isinstance(raw_summary, dict) else {} + conflicts: list[dict[str, int | float | str]] = [] + + target_issue_cv_id = file.matched_issue_cv_id + if target_issue_cv_id is None: + try: + target_issue_cv_id = int(str(summary.get("provider_id") or "")) + except ValueError: + target_issue_cv_id = None + if ( + target_issue_cv_id is not None + and metadata.comicvine_issue_id is not None + and metadata.comicvine_issue_id != target_issue_cv_id + ): + conflicts.append( + { + "field": "comicvine_issue_id", + "reviewed": target_issue_cv_id, + "source": metadata.comicvine_issue_id, + } + ) + + raw_target_number = summary.get("issue_number", file.parsed_issue_number) + try: + target_issue_number = float(str(raw_target_number)) + except (TypeError, ValueError): + target_issue_number = None + if ( + target_issue_number is not None + and metadata.issue_number is not None + and abs(metadata.issue_number - target_issue_number) >= 0.0001 + ): + conflicts.append( + { + "field": "issue_number", + "reviewed": target_issue_number, + "source": metadata.issue_number, + } + ) + + if ( + reviewed_series_cv_id is not None + and metadata.comicvine_series_id is not None + and metadata.comicvine_series_id != reviewed_series_cv_id + ): + conflicts.append( + { + "field": "comicvine_series_id", + "reviewed": reviewed_series_cv_id, + "source": metadata.comicvine_series_id, + } + ) + return conflicts + + +async def _retry_source_roots( + session: AsyncSession, + job: ImportJob, + *, + file_ids: Sequence[int] | None = None, +) -> list[Path]: + """Recover only source boundaries already approved by the saved import.""" + candidates: list[Path] = [] + if job.source_type is ImportSourceType.FILESYSTEM: + candidates.append(Path(job.source_path)) + else: + candidates.extend(Path(value) for value in dict(job.mylar3_path_map or {}).values()) + signature_query = select(ImportedFile.source_signature).where( + *retryable_failed_source_filters(job.id) + ) + if file_ids is not None: + signature_query = signature_query.where(ImportedFile.id.in_(file_ids)) + signature_rows = await session.scalars(signature_query) + root_ids = { + root_id + for signature in signature_rows.all() + if isinstance(signature, dict) + and isinstance( + root_id := signature.get("mylar_reference_root_id"), + int, + ) + and not isinstance(root_id, bool) + and root_id > 0 + } + if root_ids: + root_query = select(LibraryRoot).where( + LibraryRoot.id.in_(root_ids), + LibraryRoot.enabled.is_(True), + ) + if job.file_handling_mode is ImportFileHandlingMode.IN_PLACE: + root_query = root_query.where(LibraryRoot.allow_referenced_registrations.is_(True)) + candidates.extend(Path(root.path) for root in (await session.scalars(root_query)).all()) + + roots: list[Path] = [] + seen: set[tuple[str, str]] = set() + for candidate in candidates: + try: + lexical = candidate.expanduser().absolute() + resolved = resolve_preview_source(candidate) + except (OSError, RuntimeError, ValueError): + continue + key = (str(lexical), str(resolved)) + if resolved.parent == resolved or key in seen: + continue + seen.add(key) + roots.append(lexical) + return roots + + +async def prepare_retryable_failed_sources_for_retry( + session: AsyncSession, + job: ImportJob, + *, + file_ids: Sequence[int] | None = None, +) -> dict[str, int]: + """Revalidate changed failed sources as part of the in-app retry action.""" + retryable_filters = list(retryable_failed_source_filters(job.id)) + if file_ids is not None: + retryable_filters.append(ImportedFile.id.in_(file_ids)) + retryable_count = int( + await session.scalar(select(func.count(ImportedFile.id)).where(*retryable_filters)) or 0 + ) + if retryable_count == 0: + return { + "files_checked": 0, + "files_prepared": 0, + "blocked_files": 0, + "skipped_files": 0, + } + + roots = await _retry_source_roots(session, job, file_ids=file_ids) + if not roots: + raise ValidationError( + "No failed files are safe to retry because their approved source root is unavailable." + ) + return await prepare_completed_import_file_recheck( + session, + job.id, + source_roots=roots, + apply=True, + file_ids=list(file_ids) if file_ids is not None else None, + accept_replaced_files=True, + ) + + +async def prepare_completed_import_file_recheck( + session: AsyncSession, + job_id: int, + *, + source_roots: list[Path], + apply: bool = False, + series_ids: list[int] | None = None, + file_ids: list[int] | None = None, + accept_replaced_files: bool = False, +) -> dict[str, int]: + """Recheck only retryable failed sources from an otherwise completed import.""" + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if not allows_terminal_import_recovery(job): + raise ValidationError( + "Job must have a COMPLETED canonical import with no pending control or rollback " + "work before a failed-file recheck" + ) + if not source_roots: + raise ValidationError("At least one explicit source root is required") + roots = [(path.expanduser().absolute(), resolve_preview_source(path)) for path in source_roots] + if any(not real.is_dir() or real.parent == real for _, real in roots): + raise ValidationError("Source roots must be specific existing directories") + + block_dangerous = await is_dangerous_file_blocking_enabled(session) + max_archive_size = await get_archive_size_limit_bytes(session) + report = { + "files_checked": 0, + "files_prepared": 0, + "blocked_files": 0, + "skipped_files": 0, + } + sidecars: dict[str, dict[str, Any]] = {} + cursor = 0 + while True: + query = ( + select(ImportedFile, ImportedSeries) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .where( + *retryable_failed_source_filters(job_id), + ImportedFile.id > cursor, + ) + .order_by(ImportedFile.id) + .limit(250) + ) + if series_ids: + query = query.where(ImportedFile.import_series_id.in_(series_ids)) + if file_ids is not None: + query = query.where(ImportedFile.id.in_(file_ids)) + rows = list((await session.execute(query)).all()) + if not rows: + break + inspected: list[ + tuple[ + int, + SourceMetadata, + dict[str, Any], + dict[str, int | str], + ] + ] = [] + for imported_file, imported_series in rows: + cursor = int(imported_file.id) + metadata, content, signature = await asyncio.to_thread( + inspect_review_source, + Path(imported_file.file_path), + source_metadata_for_import_file(imported_series, imported_file), + dict(imported_file.source_signature or {}), + roots=roots, + block_dangerous=block_dangerous, + max_archive_size=max_archive_size, + accept_replaced_files=accept_replaced_files, + sidecars=sidecars, + ) + report["files_checked"] += 1 + if apply: + inspected.append((int(imported_file.id), metadata, content, signature)) + else: + source = {**metadata.diagnostics, **content} + ready_for_retry = ( + _completed_file_recheck_block( + imported_file, + metadata, + source, + reviewed_series_cv_id=imported_series.cv_id, + ) + is None + ) + blocked = not ready_for_retry + report["blocked_files"] += int(blocked) + report["files_prepared"] += int(not blocked) + + if apply: + # Inspect the complete bounded page before taking SQLite's writer + # lock. Reload and lock the job and eligible rows so a concurrent + # retry cannot have its newer import evidence overwritten. + current_job = await session.scalar( + select(ImportJob) + .where(ImportJob.id == job_id) + .with_for_update() + .execution_options(populate_existing=True) + ) + if current_job is None: + raise NotFoundError("ImportJob", job_id) + if not allows_terminal_import_recovery(current_job): + raise ValidationError( + "Job changed while failed sources were being inspected; retry the recheck" + ) + + inspected_by_id = { + file_id: (metadata, content, signature) + for file_id, metadata, content, signature in inspected + } + apply_query = ( + select(ImportedFile, ImportedSeries) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .where( + *retryable_failed_source_filters(job_id), + ImportedFile.id.in_(inspected_by_id), + ) + .order_by(ImportedFile.id) + .with_for_update() + .execution_options(populate_existing=True) + ) + current_rows = list((await session.execute(apply_query)).all()) + report["skipped_files"] += len(inspected) - len(current_rows) + for imported_file, imported_series in current_rows: + metadata, content, signature = inspected_by_id[int(imported_file.id)] + ready_for_retry = _apply_completed_file_recheck( + imported_file, + metadata, + content, + signature, + reviewed_series_cv_id=imported_series.cv_id, + ) + blocked = not ready_for_retry + report["blocked_files"] += int(blocked) + report["files_prepared"] += int(not blocked) + # Commit before reading the next page of source files. + await session.commit() + rows.clear() + current_rows.clear() + inspected.clear() + inspected_by_id.clear() + + if apply and report["files_checked"]: + session.add( + ImportJobLog( + import_job_id=job.id, + level="INFO", + event="import_failed_sources_rechecked", + message=( + f"Rechecked {report['files_checked']} failed source file(s); " + f"{report['files_prepared']} are ready for retry." + ), + data={**report, "prepared_at": datetime.now(UTC).isoformat()}, + ) + ) + await session.flush() + return report + + +async def prepare_import_recheck( + session: AsyncSession, + job_id: int, + *, + source_roots: list[Path], + apply: bool = False, + series_ids: list[int] | None = None, + accept_replaced_files: bool = False, +) -> dict[str, int]: + """Dispatch a source recheck without widening the saved job's safe scope.""" + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if job.status == ImportJobStatus.COMPLETED: + return await prepare_completed_import_file_recheck( + session, + job_id, + source_roots=source_roots, + apply=apply, + series_ids=series_ids, + accept_replaced_files=accept_replaced_files, + ) + return await prepare_review_recheck( + session, + job_id, + source_roots=source_roots, + apply=apply, + series_ids=series_ids, + accept_replaced_files=accept_replaced_files, + ) + + +async def prepare_review_recheck( + session: AsyncSession, + job_id: int, + *, + source_roots: list[Path], + apply: bool = False, + series_ids: list[int] | None = None, + accept_replaced_files: bool = False, +) -> dict[str, int]: + """Stage a bounded recheck; caller commits once and restarts the offline app. + + Default scope is automatic identity conflicts. Any existing manual decision + excludes its entire series. All matching resumes from persisted evidence, not + a new directory inventory. Dry runs do not dirty the session. + """ + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if job.status != ImportJobStatus.REVIEW or job.control_request != ImportControlRequest.NONE: + raise ValidationError("Job must be idle in REVIEW before an offline recheck") + if not source_roots: + raise ValidationError("At least one explicit source root is required") + roots = [(path.expanduser().absolute(), resolve_preview_source(path)) for path in source_roots] + if any(not real.is_dir() or real.parent == real for _, real in roots): + raise ValidationError("Source roots must be specific existing directories") + block_dangerous = await is_dangerous_file_blocking_enabled(session) + max_archive_size = await get_archive_size_limit_bytes(session) + report = {"files_checked": 0, "blocked_files": 0, "skipped_series": 0, "series_prepared": 0} + cursor = 0 + while True: + query = select(ImportedSeries).where( + ImportedSeries.import_job_id == job_id, ImportedSeries.id > cursor + ) + query = ( + query.where(ImportedSeries.id.in_(series_ids)) + if series_ids + else query.where( + ImportedSeries.status == ImportSeriesStatus.NO_MATCH, + ImportedSeries.diagnostics["reason"].as_string() + == "trusted_source_identity_conflict", + ) + ) + item = await session.scalar(query.order_by(ImportedSeries.id).limit(1)) + if item is None: + break + cursor = item.id + protected = await session.scalar( + select( + exists().where( + ImportedFile.import_series_id == item.id, + or_( + ImportedFile.status.not_in( + [ + ImportedFileStatus.NO_MATCH, + ImportedFileStatus.PENDING, + ImportedFileStatus.SAFETY_BLOCKED, + ] + ), + ImportedFile.match_method.startswith("manual"), + ImportedFile.include_in_import.is_(True), + ImportedFile.diagnostics["safety_exception"]["allowed_once"] + .as_boolean() + .is_(True), + ), + ) + ) + ) + if ( + item.user_selected_cv_id is not None + or item.selected_for_import + or protected + or item.status not in {ImportSeriesStatus.NO_MATCH, ImportSeriesStatus.MATCHED} + ): + report["skipped_series"] += 1 + continue + sidecars: dict[str, dict[str, Any]] = {} + identities: set[tuple[int, MetadataSignal]] = set() + after = 0 + checked = 0 + while True: + files = list( + ( + await session.scalars( + select(ImportedFile) + .where( + ImportedFile.import_series_id == item.id, + ImportedFile.id > after, + ) + .order_by(ImportedFile.id) + .limit(250) + ) + ).all() + ) + if not files: + break + for file in files: + metadata, content, signature = await asyncio.to_thread( + inspect_review_source, + Path(file.file_path), + source_metadata_for_import_file(item, file), + dict(file.source_signature or {}), + roots=roots, + block_dangerous=block_dangerous, + max_archive_size=max_archive_size, + accept_replaced_files=accept_replaced_files, + sidecars=sidecars, + ) + checked += 1 + signal = metadata.signals.get("comicvine_series_id") + if metadata.comicvine_series_id is not None and signal is not None: + identities.add((metadata.comicvine_series_id, signal)) + report["files_checked"] += 1 + report["blocked_files"] += int("file_safety" in content) + if apply: + _apply_file(file, metadata, content, signature) + after = files[-1].id + if apply: + await session.flush() + if checked: + report["series_prepared"] += 1 + if apply: + clear_auto_cv_match_fields(item) + if len({identity for identity, _signal in identities}) == 1: + item.cv_id = next(iter(identities))[0] + item.cv_match_method = ( + "mylar3_cv_id" + if any(signal is MetadataSignal.MYLAR3 for _, signal in identities) + else "comicinfo_cv_id" + ) + item.status = ImportSeriesStatus.PENDING + item.diagnostics = { + "reason": "source_recheck_prepared", + "previous_reason": "trusted_source_identity_conflict", + } + if apply and report["series_prepared"]: + job.status = ImportJobStatus.MATCHING + job.match_completed_at = None + job.progress_snapshot = { + **(job.progress_snapshot or {}), + "review_recheck": report, + "phase": "matching", + "progress": 0, + } + session.add( + ImportJobLog( + import_job_id=job.id, + level="INFO", + event="import_review_recheck_prepared", + message=( + "Offline source recheck prepared; local matching resumes on startup " + "without a new scan." + ), + data={**report, "prepared_at": datetime.now(UTC).isoformat()}, + ) + ) + await session.flush() + return report diff --git a/src/pullbox/services/import_rollback_execution.py b/src/pullbox/services/import_rollback_execution.py index ad730c5c..9bed442a 100644 --- a/src/pullbox/services/import_rollback_execution.py +++ b/src/pullbox/services/import_rollback_execution.py @@ -2,10 +2,12 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import dataclass +from time import monotonic from typing import TYPE_CHECKING +from sqlalchemy import and_, func, or_ from sqlalchemy import select as sa_select from pullbox.core.exceptions import NotFoundError @@ -17,6 +19,11 @@ ImportJobStatus, ) from pullbox.schemas.import_job import ImportProgressEvent +from pullbox.services.import_comicinfo_enrichment import comicinfo_enrichment_gate +from pullbox.services.import_job_actions import ( + StoryArcManagedPlacementRollbackDeferredError, +) +from pullbox.services.import_workflow_state import sync_progress_snapshot_state if TYPE_CHECKING: from datetime import datetime @@ -42,12 +49,133 @@ class RollbackActionPlan: RecomputeSeriesCounters = Callable[["AsyncSession", ImportJob], Awaitable[None]] LogImportEvent = Callable[..., Awaitable[None]] EmitProgress = Callable[ - ["AsyncSession", ImportJob, ImportProgressEvent, ProgressCallback], + ["AsyncSession", ImportJob, ImportProgressEvent, ProgressCallback | None], Awaitable[None], ] EstimateRemainingSeconds = Callable[["datetime | None", int], int | None] JobStats = Callable[[ImportJob], dict[str, int]] +ROLLBACK_ACTION_PAGE_SIZE = 500 +ROLLBACK_ACTION_CHECKPOINT_SIZE = 25 +ROLLBACK_PROGRESS_MIN_INTERVAL_SECONDS = 1.0 +_RETRIABLE_FAILED_ACTION_TYPES = ("library_file_placement_started",) + + +async def _count_completed_rollback_actions(session: AsyncSession, job_id: int) -> int: + count = await session.scalar( + sa_select(func.count(ImportJobAction.id)).where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.status == ImportJobActionStatus.COMPLETED, + ) + ) + return int(count or 0) + + +async def _load_rollback_action_status_counts( + session: AsyncSession, + job_id: int, +) -> dict[ImportJobActionStatus, int]: + """Return the durable whole-journal state, including prior attempts.""" + result = await session.execute( + sa_select(ImportJobAction.status, func.count(ImportJobAction.id)) + .where(ImportJobAction.import_job_id == job_id) + .group_by(ImportJobAction.status) + ) + return {status: int(count or 0) for status, count in result.all()} + + +async def _iter_completed_rollback_actions( + session: AsyncSession, + job_id: int, +) -> AsyncIterator[RollbackActionPlan]: + """Yield bounded reverse-order journal pages with a stable keyset cursor.""" + cursor: tuple[int, int] | None = None + while True: + statement = sa_select(ImportJobAction).where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.status == ImportJobActionStatus.COMPLETED, + ) + if cursor is not None: + sequence_no, action_id = cursor + statement = statement.where( + or_( + ImportJobAction.sequence_no < sequence_no, + and_( + ImportJobAction.sequence_no == sequence_no, + ImportJobAction.id < action_id, + ), + ) + ) + result = await session.execute( + statement.order_by( + ImportJobAction.sequence_no.desc(), + ImportJobAction.id.desc(), + ).limit(ROLLBACK_ACTION_PAGE_SIZE) + ) + page = [ + RollbackActionPlan( + action_id=action.id, + sequence_no=action.sequence_no, + action_type=action.action_type, + payload=dict(action.payload or {}), + ) + for action in result.scalars().all() + ] + if not page: + return + cursor = (page[-1].sequence_no, page[-1].action_id) + for action in page: + yield action + if len(page) < ROLLBACK_ACTION_PAGE_SIZE: + return + + +async def _iter_retriable_failed_rollback_actions( + session: AsyncSession, + job_id: int, +) -> AsyncIterator[RollbackActionPlan]: + """Replay collision starts once their earlier destination owner is removed.""" + cursor: tuple[int, int] | None = None + while True: + statement = sa_select(ImportJobAction).where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.status == ImportJobActionStatus.ROLLBACK_FAILED, + ImportJobAction.action_type.in_(_RETRIABLE_FAILED_ACTION_TYPES), + ) + if cursor is not None: + sequence_no, action_id = cursor + statement = statement.where( + or_( + ImportJobAction.sequence_no < sequence_no, + and_( + ImportJobAction.sequence_no == sequence_no, + ImportJobAction.id < action_id, + ), + ) + ) + result = await session.execute( + statement.order_by( + ImportJobAction.sequence_no.desc(), + ImportJobAction.id.desc(), + ).limit(ROLLBACK_ACTION_PAGE_SIZE) + ) + page = [ + RollbackActionPlan( + action_id=action.id, + sequence_no=action.sequence_no, + action_type=action.action_type, + payload=dict(action.payload or {}), + ) + for action in result.scalars().all() + ] + if not page: + return + cursor = (page[-1].sequence_no, page[-1].action_id) + for action in page: + yield action + if len(page) < ROLLBACK_ACTION_PAGE_SIZE: + return + async def rollback_import_job( session: AsyncSession, @@ -62,31 +190,44 @@ async def rollback_import_job( estimate_remaining_seconds: EstimateRemainingSeconds, job_stats: JobStats, progress_callback: ProgressCallback | None = None, -) -> None: +) -> bool: """Rollback durable import actions in reverse execution order.""" + async with comicinfo_enrichment_gate(): + return await _rollback_import_job_while_enrichment_fenced( + session, + job_id, + rollback_action=rollback_action, + restore_review_state=restore_review_state, + recompute_series_counters=recompute_series_counters, + recompute_file_counters=recompute_file_counters, + log_event=log_event, + emit_progress=emit_progress, + estimate_remaining_seconds=estimate_remaining_seconds, + job_stats=job_stats, + progress_callback=progress_callback, + ) + + +async def _rollback_import_job_while_enrichment_fenced( + session: AsyncSession, + job_id: int, + *, + rollback_action: RollbackAction, + restore_review_state: RestoreReviewState, + recompute_series_counters: RecomputeSeriesCounters, + recompute_file_counters: RecomputeFileCounters, + log_event: LogImportEvent, + emit_progress: EmitProgress, + estimate_remaining_seconds: EstimateRemainingSeconds, + job_stats: JobStats, + progress_callback: ProgressCallback | None = None, +) -> bool: + """Execute rollback while holding the process-local filesystem fence.""" job = await session.get(ImportJob, job_id) if job is None: raise NotFoundError("ImportJob", job_id) - actions_result = await session.execute( - sa_select(ImportJobAction) - .where( - ImportJobAction.import_job_id == job_id, - ImportJobAction.status == ImportJobActionStatus.COMPLETED, - ) - .order_by(ImportJobAction.sequence_no.desc()) - ) - actions = [ - RollbackActionPlan( - action_id=action.id, - sequence_no=action.sequence_no, - action_type=action.action_type, - payload=dict(action.payload or {}), - ) - for action in actions_result.scalars().all() - ] - - total = len(actions) + total = await _count_completed_rollback_actions(session, job_id) await log_event( session, job_id, @@ -95,7 +236,23 @@ async def rollback_import_job( message=f"Rolling back {total} recorded import actions.", action_count=total, ) - for idx, action in enumerate(actions): + await emit_progress( + session, + job, + ImportProgressEvent( + job_id=job.id, + status=ImportJobStatus.ROLLING_BACK, + phase="rollback", + progress=0, + message=f"Rolling back 0/{total} actions...", + estimated_seconds_remaining=None, + **job_stats(job), + ), + progress_callback, + ) + idx = 0 + last_progress_emit_at = monotonic() + async for action in _iter_completed_rollback_actions(session, job_id): await log_event( session, job_id, @@ -110,9 +267,60 @@ async def rollback_import_job( action_type=action.action_type, sequence_no=action.sequence_no, ) + deferred_error: StoryArcManagedPlacementRollbackDeferredError | None = None + action_error: Exception | None = None try: - await rollback_action(session, action) + async with session.begin_nested(): + try: + await rollback_action(session, action) + except StoryArcManagedPlacementRollbackDeferredError as exc: + # Cooperative placement cancellation persists its request before + # yielding. Swallow only this control-flow exception inside the + # savepoint so the request remains durable. + deferred_error = exc except Exception as exc: + # The savepoint rolls back partial database mutations from this action + # without discarding earlier successful rollback work. Filesystem changes + # cannot be inferred safely, so the journal row becomes manual recovery. + action_error = exc + + if deferred_error is not None: + progress = int((idx / max(total, 1)) * 100) + sync_progress_snapshot_state( + job, + status=ImportJobStatus.ROLLING_BACK, + mode="rollback", + phase="story_arc_placements", + progress=progress, + message="Waiting for an in-progress story-arc placement to stop safely...", + ) + snapshot = dict(job.progress_snapshot or {}) + snapshot["story_arc_rollback_waiting_work_id"] = deferred_error.work_id + job.progress_snapshot = snapshot + job.story_arc_rollback_waiting_work_id = deferred_error.work_id + await session.flush() + await log_event( + session, + job_id, + "INFO", + "import_rollback_waiting_for_story_arc_placement", + message="Rollback is waiting for in-progress story-arc placement work.", + action_index=idx + 1, + action_count=total, + action_type=action.action_type, + sequence_no=action.sequence_no, + sync_work_id=deferred_error.work_id, + ) + await session.commit() + return False + + current_action = await session.get(ImportJobAction, action.action_id) + if action_error is not None: + if current_action is not None: + current_action.status = ImportJobActionStatus.ROLLBACK_FAILED + current_action.error_message = str(action_error) or type(action_error).__name__ + current_action.rolled_back_at = None + await session.flush() await log_event( session, job_id, @@ -123,24 +331,49 @@ async def rollback_import_job( action_count=total, action_type=action.action_type, sequence_no=action.sequence_no, - error=str(exc), + error=str(action_error), ) - raise - await log_event( - session, - job_id, - "DEBUG", - "import_rollback_action_completed", - message=( - f"Rolled back action {idx + 1}/{total}: {action.action_type} #{action.sequence_no}." - ), - action_index=idx + 1, - action_count=total, - action_type=action.action_type, - sequence_no=action.sequence_no, + elif ( + current_action is not None + and current_action.status == ImportJobActionStatus.ROLLBACK_FAILED + ): + await log_event( + session, + job_id, + "ERROR", + "import_rollback_action_failed", + message=(f"Rollback action failed: {action.action_type} #{action.sequence_no}."), + action_index=idx + 1, + action_count=total, + action_type=action.action_type, + sequence_no=action.sequence_no, + error=current_action.error_message or "Manual recovery is required.", + ) + else: + await log_event( + session, + job_id, + "DEBUG", + "import_rollback_action_completed", + message=( + f"Rolled back action {idx + 1}/{total}: " + f"{action.action_type} #{action.sequence_no}." + ), + action_index=idx + 1, + action_count=total, + action_type=action.action_type, + sequence_no=action.sequence_no, + ) + idx += 1 + page_checkpoint = idx % ROLLBACK_ACTION_CHECKPOINT_SIZE == 0 or idx == total + if not page_checkpoint: + continue + now = monotonic() + should_emit_progress = ( + idx == total or now - last_progress_emit_at >= ROLLBACK_PROGRESS_MIN_INTERVAL_SECONDS ) - if progress_callback: - progress = int(((idx + 1) / max(total, 1)) * 100) + if should_emit_progress: + progress = int((idx / max(total, 1)) * 100) await emit_progress( session, job, @@ -149,7 +382,7 @@ async def rollback_import_job( status=ImportJobStatus.ROLLING_BACK, phase="rollback", progress=progress, - message=f"Rolling back {idx + 1}/{total} actions...", + message=f"Rolling back {idx}/{total} actions...", estimated_seconds_remaining=estimate_remaining_seconds( job.import_started_at, progress, @@ -158,9 +391,99 @@ async def rollback_import_job( ), progress_callback, ) + last_progress_emit_at = now + else: + await session.commit() - await restore_review_state(session, job_id) + # A failed placement start can point at a destination still owned by an + # earlier completed action. The first reverse pass must preserve that path; + # after the owner action is reversed, one bounded replay can prove the + # destination is gone and close the start record as a safe no-op. Changed or + # otherwise unproven destinations remain failed after this replay. + async for action in _iter_retriable_failed_rollback_actions(session, job_id): + retry_action_error: Exception | None = None + try: + async with session.begin_nested(): + await rollback_action(session, action) + except Exception as exc: + retry_action_error = exc + current_action = await session.get(ImportJobAction, action.action_id) + if retry_action_error is not None and current_action is not None: + current_action.status = ImportJobActionStatus.ROLLBACK_FAILED + current_action.error_message = ( + str(retry_action_error) or type(retry_action_error).__name__ + ) + current_action.rolled_back_at = None + await session.flush() + if ( + current_action is not None + and current_action.status is ImportJobActionStatus.ROLLED_BACK + ): + await log_event( + session, + job_id, + "DEBUG", + "import_rollback_action_retry_completed", + message=( + "Rolled back a deferred placement-start action after its earlier " + "destination owner was removed." + ), + action_type=action.action_type, + sequence_no=action.sequence_no, + ) + await session.commit() + + action_status_counts = await _load_rollback_action_status_counts(session, job_id) + rollback_action_count = sum(action_status_counts.values()) + rolled_back_count = action_status_counts.get(ImportJobActionStatus.ROLLED_BACK, 0) + manual_recovery_count = action_status_counts.get(ImportJobActionStatus.ROLLBACK_FAILED, 0) + if manual_recovery_count > 0: + await recompute_series_counters(session, job) + await recompute_file_counters(session, job) + message = _incomplete_rollback_message( + rolled_back_count=rolled_back_count, + manual_recovery_count=manual_recovery_count, + ) + progress = round((rolled_back_count / max(rollback_action_count, 1)) * 100) + job.status = ImportJobStatus.FAILED + job.control_request = ImportControlRequest.NONE + job.error_message = message + job.story_arc_placement_followup_pending = False + job.story_arc_rollback_waiting_work_id = None + sync_progress_snapshot_state( + job, + status=ImportJobStatus.FAILED, + mode="rollback", + phase="rollback_incomplete", + progress=progress, + message=message, + ) + snapshot = dict(job.progress_snapshot or {}) + snapshot.update( + { + "rollback_action_count": rollback_action_count, + "rollback_actions_rolled_back": rolled_back_count, + "rollback_manual_recovery_count": manual_recovery_count, + } + ) + job.progress_snapshot = snapshot + await session.flush() + await log_event( + session, + job_id, + "ERROR", + "import_rollback_incomplete", + message=message, + action_count=rollback_action_count, + rolled_back_count=rolled_back_count, + manual_recovery_count=manual_recovery_count, + ) + # True means the reverse walk reached a terminal state. The FAILED job + # and durable snapshot distinguish incomplete rollback from success. + return True + + await restore_review_state(session, job_id) await recompute_series_counters(session, job) await recompute_file_counters(session, job) cancelled_during_rollback = job.control_request == ImportControlRequest.CANCEL @@ -170,6 +493,8 @@ async def rollback_import_job( job.control_request = ImportControlRequest.NONE job.error_message = "Import cancelled by user." if cancelled_during_rollback else None job.progress_snapshot = {} + job.story_arc_placement_followup_pending = False + job.story_arc_rollback_waiting_work_id = None await session.flush() await log_event( session, @@ -185,3 +510,25 @@ async def rollback_import_job( ), action_count=total, ) + return True + + +def _incomplete_rollback_message( + *, + rolled_back_count: int, + manual_recovery_count: int, +) -> str: + if rolled_back_count <= 0: + action_word = "action" if manual_recovery_count == 1 else "actions" + require_word = "requires" if manual_recovery_count == 1 else "require" + return ( + f"Rollback incomplete: {manual_recovery_count} {action_word} {require_word} " + "manual recovery. Pullbox preserved the affected data." + ) + rolled_back_word = "action was" if rolled_back_count == 1 else "actions were" + require_word = "requires" if manual_recovery_count == 1 else "require" + return ( + f"Rollback incomplete: {rolled_back_count} {rolled_back_word} rolled back; " + f"{manual_recovery_count} {require_word} manual recovery. " + "Pullbox preserved the affected data." + ) diff --git a/src/pullbox/services/import_rollback_state.py b/src/pullbox/services/import_rollback_state.py index 2c820d54..ac0d4df8 100644 --- a/src/pullbox/services/import_rollback_state.py +++ b/src/pullbox/services/import_rollback_state.py @@ -12,76 +12,189 @@ ImportedSeries, ImportSeriesStatus, ) +from pullbox.models.story_arc import ImportedStoryArcStatus +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession -async def restore_review_state_after_rollback(session: AsyncSession, job_id: int) -> None: - """Restore imported rows to their pre-import review states after rollback.""" - series_result = await session.execute( - sa_select(ImportedSeries).where(ImportedSeries.import_job_id == job_id) - ) - series_items = list(series_result.scalars().all()) +async def restore_review_state_after_rollback( + session: AsyncSession, + job_id: int, + *, + batch_size: int = 500, +) -> None: + """Restore pre-import review state using bounded keyset pages. - files_result = await session.execute( - sa_select(ImportedFile).where(ImportedFile.import_job_id == job_id) - ) - imported_files = list(files_result.scalars().all()) - orphan_recovery_series_ids = { - imp_file.import_series_id - for imp_file in imported_files - if dict(imp_file.diagnostics or {}).get("kind") == "orphan_recovery" - } + Rollback can cover hundreds of thousands of staging rows. Flush each page + so SQLAlchemy can release clean ORM identities before the next keyset page; + the caller still owns the transaction and commit boundary. + """ + if isinstance(batch_size, bool) or batch_size <= 0: + raise ValueError("Rollback review-state batch size must be positive") - for series_item in series_items: - if series_item.status in { - ImportSeriesStatus.IMPORTED, - ImportSeriesStatus.FAILED, - ImportSeriesStatus.CONFIRMED, - ImportSeriesStatus.IMPORTING, - }: - series_item.status = ( - ImportSeriesStatus.RECOVERY_PENDING - if series_item.id in orphan_recovery_series_ids - else ImportSeriesStatus.MATCHED - ) - series_item.series_id = None - series_item.error_message = None - series_item.files_imported = 0 - series_item.files_failed = 0 + orphan_recovery_series_ids: set[int] = set() + last_file_id = 0 + while True: + imported_files = list( + ( + await session.scalars( + sa_select(ImportedFile) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.id > last_file_id, + ) + .order_by(ImportedFile.id) + .limit(batch_size) + ) + ).all() + ) + if not imported_files: + break + for imp_file in imported_files: + diagnostics = dict(imp_file.diagnostics or {}) + if diagnostics.get("kind") == "orphan_recovery": + orphan_recovery_series_ids.add(int(imp_file.import_series_id)) + _restore_imported_file_state(imp_file, diagnostics=diagnostics) + await session.flush() + last_file_id = int(imported_files[-1].id) - for imp_file in imported_files: - diagnostics = dict(imp_file.diagnostics or {}) - if imp_file.status not in { - ImportedFileStatus.IMPORTED, - ImportedFileStatus.FAILED, - ImportedFileStatus.CONFIRMED, - ImportedFileStatus.SKIPPED, - }: - continue + last_series_id = 0 + while True: + series_items = list( + ( + await session.scalars( + sa_select(ImportedSeries) + .where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.id > last_series_id, + ) + .order_by(ImportedSeries.id) + .limit(batch_size) + ) + ).all() + ) + if not series_items: + break + for series_item in series_items: + if series_item.status in { + ImportSeriesStatus.IMPORTED, + ImportSeriesStatus.FAILED, + ImportSeriesStatus.CONFIRMED, + ImportSeriesStatus.IMPORTING, + }: + series_item.status = ( + ImportSeriesStatus.RECOVERY_PENDING + if series_item.id in orphan_recovery_series_ids + else ImportSeriesStatus.MATCHED + ) + series_item.series_id = None + series_item.error_message = None + series_item.files_imported = 0 + series_item.files_failed = 0 + await session.flush() + last_series_id = int(series_items[-1].id) - if diagnostics.get("kind") == "orphan_recovery": - if diagnostics.get("resolution") == "skipped": - imp_file.status = ImportedFileStatus.SKIPPED - imp_file.include_in_import = False - elif imp_file.matched_issue_id is not None or imp_file.matched_issue_cv_id is not None: - imp_file.status = ImportedFileStatus.MATCHED - imp_file.include_in_import = False - else: - imp_file.status = ImportedFileStatus.NO_MATCH - imp_file.include_in_import = False - elif imp_file.conflict_group_id is not None: - imp_file.status = ImportedFileStatus.CONFLICT - imp_file.include_in_import = False - elif diagnostics.get("target_state") == "already_owned": - imp_file.status = ImportedFileStatus.ALREADY_OWNED - imp_file.include_in_import = False + last_arc_id = 0 + while True: + story_arcs = list( + ( + await session.scalars( + sa_select(ImportedStoryArc) + .where( + ImportedStoryArc.import_job_id == job_id, + ImportedStoryArc.id > last_arc_id, + ) + .order_by(ImportedStoryArc.id) + .limit(batch_size) + ) + ).all() + ) + if not story_arcs: + break + for story_arc in story_arcs: + if story_arc.status in { + ImportedStoryArcStatus.IMPORTED, + ImportedStoryArcStatus.FAILED, + }: + story_arc.status = ( + ImportedStoryArcStatus.CONFIRMED + if story_arc.selected_for_import + else ImportedStoryArcStatus.SKIPPED + ) + story_arc.materialized_story_arc_id = None + diagnostics = dict(story_arc.diagnostics or {}) + diagnostics.pop("materialization", None) + story_arc.diagnostics = diagnostics + await session.flush() + last_arc_id = int(story_arcs[-1].id) + + last_entry_id = 0 + while True: + story_arc_entries = list( + ( + await session.scalars( + sa_select(ImportedStoryArcEntry) + .join( + ImportedStoryArc, + ImportedStoryArcEntry.imported_story_arc_id == ImportedStoryArc.id, + ) + .where( + ImportedStoryArc.import_job_id == job_id, + ImportedStoryArcEntry.id > last_entry_id, + ) + .order_by(ImportedStoryArcEntry.id) + .limit(batch_size) + ) + ).all() + ) + if not story_arc_entries: + break + for entry in story_arc_entries: + entry.materialized_membership_id = None + diagnostics = dict(entry.diagnostics or {}) + diagnostics.pop("materialization", None) + entry.diagnostics = diagnostics + await session.flush() + last_entry_id = int(story_arc_entries[-1].id) + + +def _restore_imported_file_state( + imp_file: ImportedFile, + *, + diagnostics: dict[str, object], +) -> None: + if imp_file.status not in { + ImportedFileStatus.IMPORTED, + ImportedFileStatus.FAILED, + ImportedFileStatus.CONFIRMED, + ImportedFileStatus.SKIPPED, + }: + return + + if diagnostics.get("kind") == "orphan_recovery": + if diagnostics.get("resolution") == "skipped": + imp_file.status = ImportedFileStatus.SKIPPED elif imp_file.matched_issue_id is not None or imp_file.matched_issue_cv_id is not None: imp_file.status = ImportedFileStatus.MATCHED - imp_file.include_in_import = False else: imp_file.status = ImportedFileStatus.NO_MATCH - imp_file.include_in_import = False - imp_file.library_file_id = None - imp_file.error_message = None + elif imp_file.conflict_group_id is not None: + imp_file.status = ImportedFileStatus.CONFLICT + elif diagnostics.get("target_state") == "already_owned": + imp_file.status = ImportedFileStatus.ALREADY_OWNED + elif imp_file.matched_issue_id is not None or imp_file.matched_issue_cv_id is not None: + imp_file.status = ImportedFileStatus.MATCHED + else: + imp_file.status = ImportedFileStatus.NO_MATCH + if imp_file.status == ImportedFileStatus.NO_MATCH: + diagnostics.setdefault("reason", "rollback_restored_unmatched") + diagnostics.setdefault( + "rejection_reason", + "This file returned to unresolved review after the import was rolled back.", + ) + imp_file.diagnostics = diagnostics + imp_file.include_in_import = False + imp_file.library_file_id = None + imp_file.error_message = None diff --git a/src/pullbox/services/import_root_policy_activation.py b/src/pullbox/services/import_root_policy_activation.py new file mode 100644 index 00000000..b1945ddb --- /dev/null +++ b/src/pullbox/services/import_root_policy_activation.py @@ -0,0 +1,409 @@ +"""Transactional activation and rollback for import-adopted root policies.""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import UTC, datetime +from typing import TYPE_CHECKING, cast + +from sqlalchemy import select + +from pullbox.core.exceptions import PullboxError, ValidationError +from pullbox.core.library_naming import ( + validate_library_file_template, + validate_series_path_template, +) +from pullbox.models.import_job import ( + ImportJob, + ImportJobAction, + ImportJobActionStatus, +) +from pullbox.models.library import ( + LibraryRoot, + LibraryRootPolicy, + LibraryRootPolicySource, +) +from pullbox.services.import_job_actions import record_action + +if TYPE_CHECKING: + from collections.abc import Mapping + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.core.library_policy import LibraryIngestPolicy + +_POLICY_ACTION_TYPE = "library_root_policy_applied" +_TEMPLATE_KEYS = ( + "series_path_template", + "comic_file_template", + "annual_file_template", + "non_standard_file_template", + "single_non_standard_file_template", +) +_COLON_REPLACEMENTS = frozenset({"dash", "space", "empty", "smart"}) + + +class RootPolicyActivationConflictError(PullboxError): + """A newer root-policy edit won an optimistic comparison.""" + + def __init__(self, message: str) -> None: + super().__init__(message=message, code="ROOT_POLICY_CONFLICT", status_code=409) + + +def build_future_root_policy_snapshot( + proposal: Mapping[str, object], + baseline: LibraryIngestPolicy, +) -> dict[str, object]: + """Freeze a proposal together with its explicit-policy comparison baseline.""" + normalized = normalize_root_policy_definition(proposal) + normalized.update( + { + "expected_root_policy_id": baseline.root_policy_id, + "expected_root_policy_revision": baseline.policy_revision, + "prior_policy": _explicit_policy_snapshot_from_effective(baseline), + } + ) + return normalized + + +def apply_future_root_policy_to_ingest_policy( + baseline: LibraryIngestPolicy, + snapshot: Mapping[str, object], + *, + source_import_job_id: int | None = None, +) -> LibraryIngestPolicy: + """Use a frozen proposal for this job's managed placement before activation.""" + proposal = normalize_root_policy_definition(snapshot) + expected_revision = _snapshot_int( + snapshot.get("expected_root_policy_revision", baseline.policy_revision), + field_name="expected_root_policy_revision", + ) + return replace( + baseline, + series_folder_template=str(proposal["series_path_template"]), + series_path_template=str(proposal["series_path_template"]), + comic_file_template=str(proposal["comic_file_template"]), + annual_file_template=str(proposal["annual_file_template"]), + non_standard_file_template=str(proposal["non_standard_file_template"]), + single_non_standard_file_template=str(proposal["single_non_standard_file_template"]), + replace_illegal_characters=bool(proposal["replace_illegal_characters"]), + colon_replacement=str(proposal["colon_replacement"]), + policy_source=LibraryRootPolicySource.IMPORT_ADOPTION, + root_policy_id=None, + policy_revision=expected_revision + 1, + source_import_job_id=source_import_job_id, + ) + + +async def activate_future_root_policy( + session: AsyncSession, + job: ImportJob, + *, + successful_registration_count: int, +) -> ImportJobAction | None: + """Apply a proposed root policy once a job has registered at least one file.""" + if ( + not job.future_layout_requested + or successful_registration_count < 1 + or not job.future_root_policy_snapshot + ): + return None + + existing_action = await _load_policy_action(session, job.id) + if job.future_root_policy_applied_at is not None: + return existing_action + + if job.target_library_root_id is None: + raise ValidationError("Future library layout requires a target library root.") + + root = await session.scalar( + select(LibraryRoot).where(LibraryRoot.id == job.target_library_root_id).with_for_update() + ) + if root is None or not root.enabled: + raise ValidationError("Future library layout requires an enabled target library root.") + + snapshot = dict(job.future_root_policy_snapshot) + proposal = normalize_root_policy_definition(snapshot) + current = await session.scalar( + select(LibraryRootPolicy) + .where(LibraryRootPolicy.library_root_id == root.id) + .with_for_update() + ) + + if current is not None and _policy_matches_job(current, job, proposal): + action = existing_action or await _record_policy_action( + session, + job, + current, + prior_policy=snapshot.get("prior_policy"), + ) + job.future_root_policy_applied_at = job.future_root_policy_applied_at or datetime.now(UTC) + await session.flush() + return action + + _assert_expected_policy(current, snapshot) + prior_policy = _serialize_root_policy(current) + applied_revision = int(current.revision if current is not None else 0) + 1 + if current is None: + current = LibraryRootPolicy( + library_root_id=root.id, + schema_version=1, + series_path_template=str(proposal["series_path_template"]), + comic_file_template=str(proposal["comic_file_template"]), + annual_file_template=str(proposal["annual_file_template"]), + non_standard_file_template=str(proposal["non_standard_file_template"]), + single_non_standard_file_template=str(proposal["single_non_standard_file_template"]), + replace_illegal_characters=bool(proposal["replace_illegal_characters"]), + colon_replacement=str(proposal["colon_replacement"]), + source=LibraryRootPolicySource.IMPORT_ADOPTION, + source_import_job_id=job.id, + revision=applied_revision, + ) + session.add(current) + else: + _assign_proposal(current, proposal) + current.source = LibraryRootPolicySource.IMPORT_ADOPTION + current.source_import_job_id = job.id + current.revision = applied_revision + await session.flush() + + action = existing_action or await _record_policy_action( + session, + job, + current, + prior_policy=prior_policy, + ) + job.future_root_policy_applied_at = datetime.now(UTC) + await session.flush() + return action + + +async def rollback_future_root_policy( + session: AsyncSession, + *, + job: ImportJob, + action: ImportJobAction, +) -> None: + """Restore a policy only while the job still owns the applied revision.""" + if action.status == ImportJobActionStatus.ROLLED_BACK: + return + + payload = dict(action.payload or {}) + root_id = int(payload.get("library_root_id") or 0) + current = await session.scalar( + select(LibraryRootPolicy) + .where(LibraryRootPolicy.library_root_id == root_id) + .with_for_update() + ) + applied_policy_id = int(payload.get("applied_policy_id") or 0) + applied_revision = int(payload.get("applied_revision") or 0) + if ( + current is None + or current.id != applied_policy_id + or current.revision != applied_revision + or current.source_import_job_id != job.id + or current.source != LibraryRootPolicySource.IMPORT_ADOPTION + ): + raise RootPolicyActivationConflictError( + "Library root policy has a newer edit; rollback preserved the current policy." + ) + + prior_policy = payload.get("prior_policy") + if prior_policy is None: + await session.delete(current) + elif isinstance(prior_policy, dict): + _restore_prior_policy( + current, + prior_policy, + next_revision=applied_revision + 1, + ) + else: + raise ValidationError("Root policy rollback journal is invalid.") + + job.future_root_policy_applied_at = None + action.status = ImportJobActionStatus.ROLLED_BACK + action.rolled_back_at = datetime.now(UTC) + await session.flush() + + +async def _load_policy_action( + session: AsyncSession, + job_id: int, +) -> ImportJobAction | None: + return cast( + "ImportJobAction | None", + await session.scalar( + select(ImportJobAction) + .where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.action_type == _POLICY_ACTION_TYPE, + ) + .order_by(ImportJobAction.sequence_no.desc()) + .limit(1) + ), + ) + + +async def _record_policy_action( + session: AsyncSession, + job: ImportJob, + policy: LibraryRootPolicy, + *, + prior_policy: object, +) -> ImportJobAction: + return await record_action( + session, + job, + phase="import", + action_type=_POLICY_ACTION_TYPE, + payload={ + "library_root_id": policy.library_root_id, + "prior_policy": prior_policy, + "applied_policy_id": policy.id, + "applied_revision": policy.revision, + "applied_policy": _serialize_root_policy(policy), + }, + ) + + +def _assert_expected_policy( + current: LibraryRootPolicy | None, + snapshot: Mapping[str, object], +) -> None: + expected_id_raw = snapshot.get("expected_root_policy_id") + expected_id = ( + None + if expected_id_raw is None + else _snapshot_int(expected_id_raw, field_name="expected_root_policy_id") + ) + expected_revision = _snapshot_int( + snapshot.get("expected_root_policy_revision"), + field_name="expected_root_policy_revision", + ) + current_id = current.id if current is not None else None + current_revision = int(current.revision if current is not None else 0) + if current_id != expected_id or current_revision != expected_revision: + raise RootPolicyActivationConflictError( + "Library root policy changed after import setup; the proposed policy was not applied." + ) + + +def _policy_matches_job( + current: LibraryRootPolicy, + job: ImportJob, + proposal: Mapping[str, object], +) -> bool: + return ( + current.source == LibraryRootPolicySource.IMPORT_ADOPTION + and current.source_import_job_id == job.id + and all(getattr(current, key) == proposal[key] for key in _TEMPLATE_KEYS) + and current.replace_illegal_characters == proposal["replace_illegal_characters"] + and current.colon_replacement == proposal["colon_replacement"] + ) + + +def _assign_proposal( + policy: LibraryRootPolicy, + proposal: Mapping[str, object], +) -> None: + policy.schema_version = 1 + for key in _TEMPLATE_KEYS: + setattr(policy, key, str(proposal[key])) + policy.replace_illegal_characters = bool(proposal["replace_illegal_characters"]) + policy.colon_replacement = str(proposal["colon_replacement"]) + + +def _restore_prior_policy( + policy: LibraryRootPolicy, + prior: Mapping[str, object], + *, + next_revision: int, +) -> None: + proposal = normalize_root_policy_definition(prior) + _assign_proposal(policy, proposal) + policy.source = LibraryRootPolicySource(str(prior["source"])) + source_job_id = prior.get("source_import_job_id") + policy.source_import_job_id = ( + None + if source_job_id is None + else _snapshot_int(source_job_id, field_name="source_import_job_id") + ) + policy.revision = next_revision + + +def normalize_root_policy_definition(value: Mapping[str, object]) -> dict[str, object]: + """Validate and normalize a complete persisted root naming policy.""" + if value.get("schema_version", 1) != 1: + raise ValidationError("Unsupported future root policy schema version.") + result: dict[str, object] = {"schema_version": 1} + for key in _TEMPLATE_KEYS: + raw = value.get(key) + if not isinstance(raw, str) or not raw.strip(): + raise ValidationError("Future root policy templates must be complete.") + result[key] = raw + replace_illegal = value.get("replace_illegal_characters") + if not isinstance(replace_illegal, bool): + raise ValidationError("Future root policy has an invalid replacement setting.") + result["replace_illegal_characters"] = replace_illegal + colon_replacement = value.get("colon_replacement") + if colon_replacement not in _COLON_REPLACEMENTS: + raise ValidationError("Future root policy has an invalid colon replacement.") + result["colon_replacement"] = colon_replacement + try: + validate_series_path_template(str(result["series_path_template"])) + for key in _TEMPLATE_KEYS[1:]: + validate_library_file_template(str(result[key])) + except ValueError as exc: + raise ValidationError(str(exc)) from exc + return result + + +def _snapshot_int(value: object, *, field_name: str) -> int: + if value is None: + return 0 + if isinstance(value, bool) or not isinstance(value, (int, str)): + raise ValidationError(f"Future root policy {field_name} must be an integer.") + try: + return int(value) + except ValueError as exc: + raise ValidationError(f"Future root policy {field_name} must be an integer.") from exc + + +def _explicit_policy_snapshot_from_effective( + baseline: LibraryIngestPolicy, +) -> dict[str, object] | None: + if baseline.root_policy_id is None: + return None + return { + "id": baseline.root_policy_id, + "schema_version": 1, + "series_path_template": baseline.series_path_template or baseline.series_folder_template, + "comic_file_template": baseline.comic_file_template, + "annual_file_template": baseline.annual_file_template, + "non_standard_file_template": baseline.non_standard_file_template, + "single_non_standard_file_template": baseline.single_non_standard_file_template, + "replace_illegal_characters": baseline.replace_illegal_characters, + "colon_replacement": baseline.colon_replacement, + "source": str(baseline.policy_source), + "source_import_job_id": baseline.source_import_job_id, + "revision": baseline.policy_revision, + } + + +def _serialize_root_policy(policy: LibraryRootPolicy | None) -> dict[str, object] | None: + if policy is None: + return None + return { + "id": policy.id, + "schema_version": policy.schema_version, + "series_path_template": policy.series_path_template, + "comic_file_template": policy.comic_file_template, + "annual_file_template": policy.annual_file_template, + "non_standard_file_template": policy.non_standard_file_template, + "single_non_standard_file_template": policy.single_non_standard_file_template, + "replace_illegal_characters": policy.replace_illegal_characters, + "colon_replacement": policy.colon_replacement, + "source": str(policy.source), + "source_import_job_id": policy.source_import_job_id, + "revision": policy.revision, + } diff --git a/src/pullbox/services/import_safety_bulk_review.py b/src/pullbox/services/import_safety_bulk_review.py new file mode 100644 index 00000000..0938b88f --- /dev/null +++ b/src/pullbox/services/import_safety_bulk_review.py @@ -0,0 +1,967 @@ +"""Bounded, job-scoped bulk review for structured import safety categories.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from hashlib import sha256 +from typing import TYPE_CHECKING, Any, Final, Literal + +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +from sqlalchemy import case, false, func, select + +from pullbox.core.config_resolver import get_application_secret +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.models.audit_log import AuditEventType +from pullbox.models.import_job import ( + ImportControlRequest, + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobStatus, + ImportSourceType, +) +from pullbox.services.audit_service import AuditService +from pullbox.services.import_counters import recompute_file_counters, recompute_series_counters +from pullbox.services.import_review_actions import ( + apply_safety_allow_once_to_file, + apply_safety_skip_to_file, + prepare_series_for_safety_rematch, +) +from pullbox.services.import_safety_diagnostics import ImportSafetyCategory +from pullbox.services.import_story_arc_resolution import ( + refresh_story_arc_entries_for_import_files, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +IMPORT_SAFETY_BULK_CONFIRMATION: Final = "ALLOW ONCE" +IMPORT_SAFETY_BULK_SKIP_CONFIRMATION: Final = "SKIP FILES" +IMPORT_SAFETY_BULK_PAGE_SIZE: Final = 200 +IMPORT_SAFETY_PREVIEW_EXAMPLE_LIMIT: Final = 3 +IMPORT_SAFETY_SNAPSHOT_PAGE_SIZE: Final = 20_000 + +_PREVIEW_TOKEN_SALT: Final = "import-safety-category-preview-v1" +_PREVIEW_TOKEN_MAX_AGE_SECONDS: Final = 15 * 60 +_PREVIEW_TOKEN_VERSION: Final = 1 +_ALLOW_ONCE_ACTION: Final = "allow_once" +_SKIP_ACTION: Final = "skip" +_BULK_OVERRIDEABLE_CATEGORIES: Final = frozenset({ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT}) +_BULK_SKIPPABLE_CATEGORIES: Final = frozenset({ImportSafetyCategory.SINGLE_PAGE_COMIC}) + + +@dataclass(frozen=True, slots=True) +class ImportSafetyBulkSnapshot: + """Aggregate scope fingerprint that does not retain file names or paths.""" + + matching_count: int + eligible_count: int + min_file_id: int | None + max_file_id: int | None + max_updated_at: str | None + scope_digest: str + + +@dataclass(frozen=True, slots=True) +class ImportSafetyBulkPreview: + """Sanitized preview for one structured safety category in one job.""" + + job_id: int + source_type: ImportSourceType + category: ImportSafetyCategory + matching_count: int + affected_count: int + skipped_count: int + examples: tuple[str, ...] + overrideable: bool + preview_token: str | None + + +@dataclass(frozen=True, slots=True) +class ImportSafetyBulkResult: + """Counts from one category-specific safety review operation.""" + + job_id: int + source_type: ImportSourceType + category: ImportSafetyCategory + affected_count: int + skipped_count: int + pages_processed: int + + +class ImportSafetyBulkInterruptedError(Exception): + """Raised after bounded pages commit when the job ceases to be actionable.""" + + def __init__(self, result: ImportSafetyBulkResult, *, reason: str) -> None: + self.result = result + self.reason = reason + super().__init__("The safety bulk action stopped because the import job changed state.") + + +def _category_expression() -> Any: + return ImportedFile.diagnostics["safety_block"]["category"].as_string() + + +def _overrideable_expression() -> Any: + return ImportedFile.diagnostics["safety_block"]["overrideable"].as_boolean() + + +def _category_filters(job_id: int, category: ImportSafetyCategory) -> tuple[Any, ...]: + return ( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + _category_expression() == category.value, + ) + + +def _eligible_expression(category: ImportSafetyCategory) -> Any: + if category is not ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT: + return false() + return _overrideable_expression().is_(True) + + +def _snapshot_updated_at(value: datetime | None) -> str | None: + return value.isoformat(timespec="microseconds") if value is not None else None + + +async def _load_scope_digest( + session: AsyncSession, + job_id: int, + category: ImportSafetyCategory, + *, + max_file_id: int | None, + expected_count: int, +) -> str: + """Hash exact scoped row identities in bounded keyset pages without retaining them.""" + digest = sha256() + if max_file_id is None or expected_count == 0: + return digest.hexdigest() + cursor = 0 + hashed_count = 0 + while cursor < max_file_id: + rows = ( + await session.execute( + select( + ImportedFile.id, + ImportedFile.updated_at, + _overrideable_expression(), + ) + .where( + *_category_filters(job_id, category), + ImportedFile.id > cursor, + ImportedFile.id <= max_file_id, + ) + .order_by(ImportedFile.id) + .limit(IMPORT_SAFETY_SNAPSHOT_PAGE_SIZE) + ) + ).all() + if not rows: + break + for file_id, updated_at, overrideable in rows: + digest.update( + ( + f"{int(file_id)}|{_snapshot_updated_at(updated_at)}|" + f"{1 if overrideable is True else 0}\n" + ).encode() + ) + cursor = int(file_id) + hashed_count += 1 + # A preview or confirmation may cover 200K rows. Observe persisted + # control state at each 20K-scalar boundary without retaining the scope. + await _load_review_job(session, job_id) + if hashed_count != expected_count: + raise ValidationError("The safety category changed while it was being previewed.") + return digest.hexdigest() + + +async def _load_review_job(session: AsyncSession, job_id: int) -> ImportJob: + job = await session.get(ImportJob, job_id, populate_existing=True) + if job is None: + raise NotFoundError("ImportJob", job_id) + if job.status != ImportJobStatus.REVIEW: + raise ValidationError("Job must be in REVIEW state to update safety review files") + if job.control_request != ImportControlRequest.NONE: + raise ValidationError("The import job has a pending control request") + return job + + +async def _load_snapshot( + session: AsyncSession, + job_id: int, + category: ImportSafetyCategory, +) -> ImportSafetyBulkSnapshot: + eligible = _eligible_expression(category) + row = ( + await session.execute( + select( + func.count(ImportedFile.id), + func.coalesce(func.sum(case((eligible, 1), else_=0)), 0), + func.min(ImportedFile.id), + func.max(ImportedFile.id), + func.max(ImportedFile.updated_at), + ).where(*_category_filters(job_id, category)) + ) + ).one() + matching_count = int(row[0] or 0) + max_file_id = int(row[3]) if row[3] is not None else None + scope_digest = await _load_scope_digest( + session, + job_id, + category, + max_file_id=max_file_id, + expected_count=matching_count, + ) + return ImportSafetyBulkSnapshot( + matching_count=matching_count, + eligible_count=int(row[1] or 0), + min_file_id=int(row[2]) if row[2] is not None else None, + max_file_id=max_file_id, + max_updated_at=_snapshot_updated_at(row[4]), + scope_digest=scope_digest, + ) + + +def _safe_example_name(value: str) -> str: + normalized = value.replace("\\", "/").rstrip("/") + leaf = normalized.rsplit("/", maxsplit=1)[-1] + safe_leaf = "".join(character for character in leaf if character >= " " and character != "\x7f") + return (safe_leaf or "File")[:200] + + +async def _load_bounded_examples( + session: AsyncSession, + job_id: int, + category: ImportSafetyCategory, + *, + limit: int, +) -> tuple[str, ...]: + if limit <= 0: + return () + names = ( + ( + await session.execute( + select(ImportedFile.file_name) + .where(*_category_filters(job_id, category)) + .order_by(ImportedFile.id) + .limit(min(limit, IMPORT_SAFETY_PREVIEW_EXAMPLE_LIMIT)) + ) + ) + .scalars() + .all() + ) + return tuple(_safe_example_name(str(name)) for name in names) + + +def _serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(get_application_secret(), salt=_PREVIEW_TOKEN_SALT) + + +def _snapshot_payload(snapshot: ImportSafetyBulkSnapshot) -> dict[str, object]: + return { + "matching_count": snapshot.matching_count, + "eligible_count": snapshot.eligible_count, + "min_file_id": snapshot.min_file_id, + "max_file_id": snapshot.max_file_id, + "max_updated_at": snapshot.max_updated_at, + "scope_digest": snapshot.scope_digest, + } + + +def _build_preview_token( + *, + job: ImportJob, + category: ImportSafetyCategory, + actor_id: int, + snapshot: ImportSafetyBulkSnapshot, + action: str = _ALLOW_ONCE_ACTION, +) -> str: + return str( + _serializer().dumps( + { + "version": _PREVIEW_TOKEN_VERSION, + "job_id": job.id, + "source_type": job.source_type.value, + "category": category.value, + "action": action, + "actor_id": actor_id, + "snapshot": _snapshot_payload(snapshot), + } + ) + ) + + +def _load_preview_token(token: str) -> Mapping[str, object]: + try: + payload = _serializer().loads(token, max_age=_PREVIEW_TOKEN_MAX_AGE_SECONDS) + except SignatureExpired as exc: + raise ValidationError("The safety preview expired. Preview the category again.") from exc + except BadSignature as exc: + raise ValidationError("The safety preview is invalid. Preview the category again.") from exc + if not isinstance(payload, Mapping): + raise ValidationError("The safety preview is invalid. Preview the category again.") + return payload + + +def _token_int(payload: Mapping[str, object], key: str) -> int: + value = payload.get(key) + if isinstance(value, bool) or not isinstance(value, int): + raise ValidationError("The safety preview is invalid. Preview the category again.") + return value + + +def _token_snapshot(payload: Mapping[str, object]) -> ImportSafetyBulkSnapshot: + raw_snapshot = payload.get("snapshot") + if not isinstance(raw_snapshot, Mapping): + raise ValidationError("The safety preview is invalid. Preview the category again.") + + matching_count = _token_int(raw_snapshot, "matching_count") + eligible_count = _token_int(raw_snapshot, "eligible_count") + raw_min_id = raw_snapshot.get("min_file_id") + raw_max_id = raw_snapshot.get("max_file_id") + if raw_min_id is not None and (isinstance(raw_min_id, bool) or not isinstance(raw_min_id, int)): + raise ValidationError("The safety preview is invalid. Preview the category again.") + if raw_max_id is not None and (isinstance(raw_max_id, bool) or not isinstance(raw_max_id, int)): + raise ValidationError("The safety preview is invalid. Preview the category again.") + raw_updated_at = raw_snapshot.get("max_updated_at") + if raw_updated_at is not None and not isinstance(raw_updated_at, str): + raise ValidationError("The safety preview is invalid. Preview the category again.") + raw_scope_digest = raw_snapshot.get("scope_digest") + if ( + not isinstance(raw_scope_digest, str) + or len(raw_scope_digest) != 64 + or any(character not in "0123456789abcdef" for character in raw_scope_digest) + ): + raise ValidationError("The safety preview is invalid. Preview the category again.") + return ImportSafetyBulkSnapshot( + matching_count=matching_count, + eligible_count=eligible_count, + min_file_id=raw_min_id, + max_file_id=raw_max_id, + max_updated_at=raw_updated_at, + scope_digest=raw_scope_digest, + ) + + +def _validate_preview_token_scope( + payload: Mapping[str, object], + *, + job: ImportJob, + category: ImportSafetyCategory, + actor_id: int, + action: str = _ALLOW_ONCE_ACTION, +) -> ImportSafetyBulkSnapshot: + if ( + payload.get("version") != _PREVIEW_TOKEN_VERSION + or _token_int(payload, "job_id") != job.id + or payload.get("source_type") != job.source_type.value + or payload.get("category") != category.value + or payload.get("action") != action + or _token_int(payload, "actor_id") != actor_id + ): + raise ValidationError("The safety preview does not match this job and category.") + return _token_snapshot(payload) + + +async def preview_import_safety_category( + session: AsyncSession, + job_id: int, + category: ImportSafetyCategory, + *, + actor_id: int, + example_limit: int = IMPORT_SAFETY_PREVIEW_EXAMPLE_LIMIT, +) -> ImportSafetyBulkPreview: + """Preview one existing structured safety category without mutating rows.""" + job = await _load_review_job(session, job_id) + snapshot = await _load_snapshot(session, job_id, category) + if snapshot.matching_count == 0: + raise ValidationError( + "No safety-blocked files in this job use the selected structured category." + ) + + can_override = category in _BULK_OVERRIDEABLE_CATEGORIES and snapshot.eligible_count > 0 + examples = await _load_bounded_examples( + session, + job_id, + category, + limit=example_limit, + ) + preview_token = ( + _build_preview_token( + job=job, + category=category, + actor_id=actor_id, + snapshot=snapshot, + ) + if can_override + else None + ) + return ImportSafetyBulkPreview( + job_id=job.id, + source_type=job.source_type, + category=category, + matching_count=snapshot.matching_count, + affected_count=snapshot.eligible_count if can_override else 0, + skipped_count=( + snapshot.matching_count - snapshot.eligible_count + if can_override + else snapshot.matching_count + ), + examples=examples, + overrideable=can_override, + preview_token=preview_token, + ) + + +async def preview_import_safety_category_skip( + session: AsyncSession, + job_id: int, + category: ImportSafetyCategory, + *, + actor_id: int, + example_limit: int = IMPORT_SAFETY_PREVIEW_EXAMPLE_LIMIT, +) -> ImportSafetyBulkPreview: + """Preview a source-preserving category skip without mutating any file.""" + if category not in _BULK_SKIPPABLE_CATEGORIES: + raise ValidationError("This safety category cannot be bulk-skipped.") + job = await _load_review_job(session, job_id) + snapshot = await _load_snapshot(session, job_id, category) + if snapshot.matching_count == 0: + raise ValidationError( + "No safety-blocked files in this job use the selected structured category." + ) + skip_snapshot = replace(snapshot, eligible_count=snapshot.matching_count) + examples = await _load_bounded_examples( + session, + job_id, + category, + limit=example_limit, + ) + return ImportSafetyBulkPreview( + job_id=job.id, + source_type=job.source_type, + category=category, + matching_count=skip_snapshot.matching_count, + affected_count=skip_snapshot.matching_count, + skipped_count=0, + examples=examples, + overrideable=True, + preview_token=_build_preview_token( + job=job, + category=category, + actor_id=actor_id, + snapshot=skip_snapshot, + action=_SKIP_ACTION, + ), + ) + + +def _result( + *, + job_id: int, + source_type: ImportSourceType, + category: ImportSafetyCategory, + affected_count: int, + original_matching_count: int, + pages_processed: int, +) -> ImportSafetyBulkResult: + return ImportSafetyBulkResult( + job_id=job_id, + source_type=source_type, + category=category, + affected_count=affected_count, + skipped_count=max(original_matching_count - affected_count, 0), + pages_processed=pages_processed, + ) + + +async def _write_durable_bulk_audit( + session: AsyncSession, + result: ImportSafetyBulkResult, + *, + actor_id: int, + actor_username: str | None, + source_ip: str | None, + outcome: Literal["requested", "completed", "interrupted"], + action: str = _ALLOW_ONCE_ACTION, +) -> None: + """Commit a fixed-field, path-free audit record before returning.""" + action_label = "skip" if action == _SKIP_ACTION else "override" + detail_by_outcome = { + "requested": f"Import safety category {action_label} requested.", + "completed": f"Import safety category {action_label} completed.", + "interrupted": f"Import safety category {action_label} interrupted.", + } + await AuditService.log_event( + session, + AuditEventType.IMPORT_SAFETY_BULK_OVERRIDE, + source_ip=source_ip, + user_id=actor_id, + username=actor_username, + detail=detail_by_outcome[outcome], + metadata={ + "job_id": result.job_id, + "source_type": result.source_type.value, + "category": result.category.value, + "action": action, + "affected_count": result.affected_count, + "skipped_count": result.skipped_count, + "outcome": outcome, + }, + ) + await session.commit() + + +async def allow_import_safety_category_once( + session: AsyncSession, + job_id: int, + category: ImportSafetyCategory, + *, + actor_id: int, + actor_username: str | None = None, + source_ip: str | None = None, + preview_token: str, + page_size: int = IMPORT_SAFETY_BULK_PAGE_SIZE, +) -> ImportSafetyBulkResult: + """Apply one previewed size-limit exception in bounded committed pages.""" + if category not in _BULK_OVERRIDEABLE_CATEGORIES: + raise ValidationError("This safety category cannot be bulk-overridden.") + if page_size < 1 or page_size > IMPORT_SAFETY_BULK_PAGE_SIZE: + raise ValidationError( + f"Safety bulk page size must be between 1 and {IMPORT_SAFETY_BULK_PAGE_SIZE}." + ) + + job = await _load_review_job(session, job_id) + payload = _load_preview_token(preview_token) + preview_snapshot = _validate_preview_token_scope( + payload, + job=job, + category=category, + actor_id=actor_id, + ) + if preview_snapshot.eligible_count <= 0 or preview_snapshot.max_file_id is None: + raise ValidationError("This safety category has no eligible files to override.") + + requested_result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=preview_snapshot.eligible_count, + original_matching_count=preview_snapshot.matching_count, + pages_processed=0, + ) + await _write_durable_bulk_audit( + session, + requested_result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="requested", + ) + + # The requested audit commit releases the read transaction. Revalidate the + # exact row-identity digest before the first mutable page so a concurrent + # category or eligibility change cannot reuse the signed preview. + try: + job = await _load_review_job(session, job_id) + post_audit_snapshot = await _load_snapshot(session, job_id, category) + except (NotFoundError, ValidationError): + interrupted_result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=0, + original_matching_count=preview_snapshot.matching_count, + pages_processed=0, + ) + await _write_durable_bulk_audit( + session, + interrupted_result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="interrupted", + ) + raise + if post_audit_snapshot != preview_snapshot: + interrupted_result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=0, + original_matching_count=preview_snapshot.matching_count, + pages_processed=0, + ) + await _write_durable_bulk_audit( + session, + interrupted_result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="interrupted", + ) + raise ValidationError("The safety category changed. Preview it again before confirming.") + + affected_count = 0 + pages_processed = 0 + cursor = 0 + allowed_at = datetime.now(UTC) + max_file_id = preview_snapshot.max_file_id + max_updated_at = ( + datetime.fromisoformat(preview_snapshot.max_updated_at) + if preview_snapshot.max_updated_at is not None + else None + ) + + while True: + current_job = await session.get(ImportJob, job_id, populate_existing=True) + if ( + current_job is None + or current_job.status != ImportJobStatus.REVIEW + or current_job.control_request != ImportControlRequest.NONE + ): + partial_result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=affected_count, + original_matching_count=preview_snapshot.matching_count, + pages_processed=pages_processed, + ) + await _write_durable_bulk_audit( + session, + partial_result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="interrupted", + ) + if affected_count > 0: + raise ImportSafetyBulkInterruptedError(partial_result, reason="job_state_changed") + raise ValidationError("The import job is no longer available for safety review.") + + page = list( + ( + await session.execute( + select(ImportedFile) + .where( + *_category_filters(job_id, category), + _eligible_expression(category), + ImportedFile.id > cursor, + ImportedFile.id <= max_file_id, + *( + (ImportedFile.updated_at <= max_updated_at,) + if max_updated_at is not None + else () + ), + ) + .order_by(ImportedFile.id) + .limit(page_size) + ) + ) + .scalars() + .all() + ) + if not page: + break + + affected_series_ids: set[int] = set() + affected_file_ids: list[int] = [] + for imp_file in page: + cursor = max(cursor, imp_file.id) + diagnostics = imp_file.diagnostics or {} + raw_block = diagnostics.get("safety_block") + if not isinstance(raw_block, Mapping): + continue + if ( + raw_block.get("category") != category.value + or raw_block.get("overrideable") is not True + ): + continue + apply_safety_allow_once_to_file(imp_file, allowed_at=allowed_at) + affected_file_ids.append(imp_file.id) + affected_series_ids.add(imp_file.import_series_id) + affected_count += 1 + + if affected_series_ids: + imported_series = list( + ( + await session.execute( + select(ImportedSeries).where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.id.in_(sorted(affected_series_ids)), + ) + ) + ) + .scalars() + .all() + ) + for series in imported_series: + prepare_series_for_safety_rematch(series) + + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job_id, + import_file_ids=affected_file_ids, + ) + + await session.flush() + await session.commit() + pages_processed += 1 + session.sync_session.expunge_all() + + result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=affected_count, + original_matching_count=preview_snapshot.matching_count, + pages_processed=pages_processed, + ) + if affected_count != preview_snapshot.eligible_count: + await _write_durable_bulk_audit( + session, + result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="interrupted", + ) + if affected_count > 0: + raise ImportSafetyBulkInterruptedError(result, reason="scope_changed_during_apply") + raise ValidationError("The safety category changed. Preview it again before confirming.") + await _write_durable_bulk_audit( + session, + result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="completed", + ) + return result + + +async def skip_import_safety_category( + session: AsyncSession, + job_id: int, + category: ImportSafetyCategory, + *, + actor_id: int, + actor_username: str | None = None, + source_ip: str | None = None, + preview_token: str, + page_size: int = IMPORT_SAFETY_BULK_PAGE_SIZE, +) -> ImportSafetyBulkResult: + """Skip one previewed one-page category while preserving every source file.""" + if category not in _BULK_SKIPPABLE_CATEGORIES: + raise ValidationError("This safety category cannot be bulk-skipped.") + if page_size < 1 or page_size > IMPORT_SAFETY_BULK_PAGE_SIZE: + raise ValidationError( + f"Safety bulk page size must be between 1 and {IMPORT_SAFETY_BULK_PAGE_SIZE}." + ) + + job = await _load_review_job(session, job_id) + payload = _load_preview_token(preview_token) + preview_snapshot = _validate_preview_token_scope( + payload, + job=job, + category=category, + actor_id=actor_id, + action=_SKIP_ACTION, + ) + if preview_snapshot.matching_count <= 0 or preview_snapshot.max_file_id is None: + raise ValidationError("This safety category has no files to skip.") + + requested_result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=preview_snapshot.matching_count, + original_matching_count=preview_snapshot.matching_count, + pages_processed=0, + ) + await _write_durable_bulk_audit( + session, + requested_result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="requested", + action=_SKIP_ACTION, + ) + + try: + job = await _load_review_job(session, job_id) + current_snapshot = replace( + await _load_snapshot(session, job_id, category), + eligible_count=preview_snapshot.matching_count, + ) + except (NotFoundError, ValidationError): + interrupted_result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=0, + original_matching_count=preview_snapshot.matching_count, + pages_processed=0, + ) + await _write_durable_bulk_audit( + session, + interrupted_result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="interrupted", + action=_SKIP_ACTION, + ) + raise + if current_snapshot != preview_snapshot: + interrupted_result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=0, + original_matching_count=preview_snapshot.matching_count, + pages_processed=0, + ) + await _write_durable_bulk_audit( + session, + interrupted_result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="interrupted", + action=_SKIP_ACTION, + ) + raise ValidationError("The safety category changed. Preview it again before confirming.") + + affected_count = 0 + pages_processed = 0 + cursor = 0 + max_file_id = preview_snapshot.max_file_id + max_updated_at = ( + datetime.fromisoformat(preview_snapshot.max_updated_at) + if preview_snapshot.max_updated_at is not None + else None + ) + while True: + current_job = await session.get(ImportJob, job_id, populate_existing=True) + if ( + current_job is None + or current_job.status != ImportJobStatus.REVIEW + or current_job.control_request != ImportControlRequest.NONE + ): + partial_result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=affected_count, + original_matching_count=preview_snapshot.matching_count, + pages_processed=pages_processed, + ) + await _write_durable_bulk_audit( + session, + partial_result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome="interrupted", + action=_SKIP_ACTION, + ) + if affected_count: + raise ImportSafetyBulkInterruptedError(partial_result, reason="job_state_changed") + raise ValidationError("The import job is no longer available for safety review.") + + page = list( + ( + await session.execute( + select(ImportedFile) + .where( + *_category_filters(job_id, category), + ImportedFile.id > cursor, + ImportedFile.id <= max_file_id, + *((ImportedFile.updated_at <= max_updated_at,) if max_updated_at else ()), + ) + .order_by(ImportedFile.id) + .limit(page_size) + ) + ) + .scalars() + .all() + ) + if not page: + break + + affected_series_ids: set[int] = set() + affected_file_ids: list[int] = [] + for imp_file in page: + cursor = max(cursor, imp_file.id) + diagnostics = imp_file.diagnostics or {} + raw_block = diagnostics.get("safety_block") + if not isinstance(raw_block, Mapping) or raw_block.get("category") != category.value: + continue + apply_safety_skip_to_file(imp_file) + affected_file_ids.append(imp_file.id) + affected_series_ids.add(imp_file.import_series_id) + affected_count += 1 + + if affected_series_ids: + imported_series = list( + ( + await session.execute( + select(ImportedSeries).where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.id.in_(sorted(affected_series_ids)), + ) + ) + ) + .scalars() + .all() + ) + for series in imported_series: + series.selected_for_import = False + await recompute_file_counters( + session, + current_job, + series_ids=sorted(affected_series_ids), + ) + await recompute_series_counters(session, current_job) + + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job_id, + import_file_ids=affected_file_ids, + ) + + await session.flush() + await session.commit() + pages_processed += 1 + session.sync_session.expunge_all() + + result = _result( + job_id=job_id, + source_type=job.source_type, + category=category, + affected_count=affected_count, + original_matching_count=preview_snapshot.matching_count, + pages_processed=pages_processed, + ) + outcome: Literal["completed", "interrupted"] = ( + "completed" if affected_count == preview_snapshot.matching_count else "interrupted" + ) + await _write_durable_bulk_audit( + session, + result, + actor_id=actor_id, + actor_username=actor_username, + source_ip=source_ip, + outcome=outcome, + action=_SKIP_ACTION, + ) + if outcome == "interrupted": + if affected_count: + raise ImportSafetyBulkInterruptedError(result, reason="scope_changed_during_apply") + raise ValidationError("The safety category changed. Preview it again before confirming.") + return result diff --git a/src/pullbox/services/import_safety_diagnostics.py b/src/pullbox/services/import_safety_diagnostics.py new file mode 100644 index 00000000..1bbb100e --- /dev/null +++ b/src/pullbox/services/import_safety_diagnostics.py @@ -0,0 +1,489 @@ +"""Typed, sanitized diagnostics for actionable import safety failures.""" + +from __future__ import annotations + +import enum +import re +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping, Sequence + + +class ImportSafetyCategory(enum.StrEnum): + """Stable review categories for import safety and source-integrity failures.""" + + PERMISSION_UNREADABLE = "permission_unreadable" + ARCHIVE_INSPECTION_FAILED = "archive_inspection_failed" + ZERO_BYTE = "zero_byte" + ARCHIVE_NO_PAGES = "archive_no_pages" + SINGLE_PAGE_COMIC = "single_page_comic" + DECOMPRESSION_SIZE_LIMIT = "decompression_size_limit" + DANGEROUS_PATH_OR_PAYLOAD = "dangerous_path_or_payload" + OUTSIDE_APPROVED_ROOT = "outside_approved_root" + UNSUPPORTED_FILE_TYPE = "unsupported_file_type" + SOURCE_CHANGED = "source_changed" + SOURCE_MISSING = "source_missing" + UNKNOWN = "unknown" + + +_CATEGORY_LABELS: dict[ImportSafetyCategory, str] = { + ImportSafetyCategory.PERMISSION_UNREADABLE: "Unreadable or permission denied", + ImportSafetyCategory.ARCHIVE_INSPECTION_FAILED: "Archive inspection failed", + ImportSafetyCategory.ZERO_BYTE: "Zero-byte file", + ImportSafetyCategory.ARCHIVE_NO_PAGES: "No comic pages", + ImportSafetyCategory.SINGLE_PAGE_COMIC: "One-page archive", + ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT: "Decompression-size limit", + ImportSafetyCategory.DANGEROUS_PATH_OR_PAYLOAD: "Dangerous link, path, or payload", + ImportSafetyCategory.OUTSIDE_APPROVED_ROOT: "Outside approved root", + ImportSafetyCategory.UNSUPPORTED_FILE_TYPE: "Unsupported file type", + ImportSafetyCategory.SOURCE_CHANGED: "Source changed or unavailable", + ImportSafetyCategory.SOURCE_MISSING: "Recorded file not found", + ImportSafetyCategory.UNKNOWN: "Other safety failure", +} + + +def import_safety_category_label(category: ImportSafetyCategory) -> str: + """Return the stable user-facing label for a safety category.""" + return _CATEGORY_LABELS[category] + + +_SANITIZED_REASONS: dict[ImportSafetyCategory, str] = { + ImportSafetyCategory.PERMISSION_UNREADABLE: ( + "Pullbox could not read this file. Check its permissions and try again." + ), + ImportSafetyCategory.ARCHIVE_INSPECTION_FAILED: ( + "Pullbox could not inspect this archive. It may be corrupt, incomplete, " + "or temporarily unavailable." + ), + ImportSafetyCategory.ZERO_BYTE: ( + "The file is empty (zero bytes). Replace it with a complete file and retry." + ), + ImportSafetyCategory.ARCHIVE_NO_PAGES: ( + "The archive contains no non-empty comic image pages. Metadata alone is not a comic. " + "Replace the file or skip it; its series identity is preserved." + ), + ImportSafetyCategory.SINGLE_PAGE_COMIC: ( + "The archive contains one image page. It may be cover art, a damaged archive, " + "or an intentional one-page comic. Review it before importing." + ), + ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT: ( + "The archive exceeds Pullbox's configured decompressed-size limit." + ), + ImportSafetyCategory.DANGEROUS_PATH_OR_PAYLOAD: ( + "The file contains a dangerous path, link, or payload and cannot be overridden." + ), + ImportSafetyCategory.OUTSIDE_APPROVED_ROOT: ( + "The file is outside the approved import or library root." + ), + ImportSafetyCategory.UNSUPPORTED_FILE_TYPE: ("The file type is not supported for import."), + ImportSafetyCategory.SOURCE_CHANGED: ( + "The source changed or became unavailable after scanning. Rescan before retrying." + ), + ImportSafetyCategory.SOURCE_MISSING: ( + "No file was found at the recorded path, and Pullbox could not find a safe, verified " + "replacement. The saved filename may be outdated, or the file may have been moved or " + "removed. Verify the source location or skip this missing reference." + ), + ImportSafetyCategory.UNKNOWN: ( + "Pullbox blocked this file because its safety inspection did not complete safely." + ), +} + +_RETRYABLE_CATEGORIES = frozenset( + { + ImportSafetyCategory.PERMISSION_UNREADABLE, + ImportSafetyCategory.ARCHIVE_INSPECTION_FAILED, + ImportSafetyCategory.ZERO_BYTE, + ImportSafetyCategory.ARCHIVE_NO_PAGES, + ImportSafetyCategory.OUTSIDE_APPROVED_ROOT, + ImportSafetyCategory.SOURCE_CHANGED, + ImportSafetyCategory.SOURCE_MISSING, + } +) + +_PERMISSION_CODES = frozenset({"permission_denied", "source_unreadable", "unreadable"}) +_INSPECTION_CODES = frozenset({"archive_inspection_failed", "corrupt_archive", "inspection_failed"}) +_ZERO_BYTE_CODES = frozenset({"zero_byte", "zero_byte_file", "empty_file"}) +_CONTENT_CODES = frozenset({"archive_no_pages", "single_page_comic"}) +_SIZE_CODES = frozenset( + { + "archive_decompressed_size", + "archive_decompressed_size_limit", + "decompression_size_limit", + "pillow_decompression_bomb", + } +) +_DANGEROUS_CODES = frozenset( + { + "dangerous_archive_path", + "dangerous_path_or_payload", + "dangerous_payload", + "path_traversal", + "source_path_unsafe", + "unsafe_path_mapping", + "invalid_path_text", + } +) +_OUTSIDE_ROOT_CODES = frozenset( + { + "outside_approved_root", + "source_outside_root", + "outside_root", + "source_root_ambiguous", + } +) +_UNSUPPORTED_CODES = frozenset({"unsupported_file_type", "unsupported_extension"}) +_SOURCE_CHANGED_CODES = frozenset( + { + "source_changed", + "source_missing", + "source_signature_missing", + "source_signature_unsupported", + "source_unavailable", + "source_root_unconfirmed", + "source_root_changed", + "source_identity_changed", + } +) + + +@dataclass(frozen=True, slots=True) +class ImportSafetyClassification: + """Pure classification result safe to persist or expose in review UI.""" + + category: ImportSafetyCategory + code: str + sanitized_reason: str + retryable: bool + overrideable: bool + + +def _normalized_token(value: str | None) -> str: + if value is None: + return "" + return value.strip().lower().replace("-", "_").replace(" ", "_") + + +def _contains_any(value: str, markers: Sequence[str]) -> bool: + return any(marker in value for marker in markers) + + +def classify_import_safety_failure( + reason: str, + *, + kind: str | None = None, + code: str | None = None, + overrideable_hint: bool | None = None, +) -> ImportSafetyClassification: + """Classify raw failure evidence without returning raw paths or exception text. + + Exact machine codes take precedence over human text. The override hint can + make a size-limit block stricter, but can never make a dangerous or + containment failure overrideable. + """ + normalized_reason = reason.strip().lower() + normalized_kind = _normalized_token(kind) + normalized_code = _normalized_token(code) + evidence_tokens = {token for token in (normalized_code, normalized_kind) if token} + stable_reason_tokens = ( + _PERMISSION_CODES + | _INSPECTION_CODES + | _ZERO_BYTE_CODES + | _CONTENT_CODES + | _SIZE_CODES + | _DANGEROUS_CODES + | _OUTSIDE_ROOT_CODES + | _UNSUPPORTED_CODES + | _SOURCE_CHANGED_CODES + ) + if _normalized_token(reason) in stable_reason_tokens: + evidence_tokens.add(_normalized_token(reason)) + + category: ImportSafetyCategory + stable_code: str + if evidence_tokens & _DANGEROUS_CODES or _contains_any( + normalized_reason, + ( + "path traversal", + "dangerous executable", + "dangerous file", + "dangerous payload", + "unsafe path component", + "unsafe archive member", + "symbolic link", + "symlink", + ), + ): + category = ImportSafetyCategory.DANGEROUS_PATH_OR_PAYLOAD + if normalized_code in _DANGEROUS_CODES: + stable_code = normalized_code + elif "path traversal" in normalized_reason or "unsafe path" in normalized_reason: + stable_code = "dangerous_archive_path" + else: + stable_code = "dangerous_payload" + elif evidence_tokens & _OUTSIDE_ROOT_CODES or _contains_any( + normalized_reason, + ( + "outside approved root", + "outside enabled library root", + "outside its library root", + "outside the import root", + "outside configured root", + "escapes the approved root", + ), + ): + category = ImportSafetyCategory.OUTSIDE_APPROVED_ROOT + stable_code = normalized_code or "outside_approved_root" + elif evidence_tokens & _PERMISSION_CODES or _contains_any( + normalized_reason, + ("permissionerror", "permission denied", "not readable", "unreadable"), + ): + category = ImportSafetyCategory.PERMISSION_UNREADABLE + stable_code = normalized_code or "permission_denied" + elif ( + evidence_tokens & _ZERO_BYTE_CODES + or _contains_any( + normalized_reason, + ("zero-byte", "zero byte", "file is empty", "empty file"), + ) + or re.search(r"(? dict[str, object]: + """Build a sanitized persisted/review payload from raw safety evidence. + + ``details`` is accepted so callers can pass existing exception evidence, + but is deliberately not copied into the result because it commonly holds + host/container absolute paths or unsafe archive member names. + """ + del details + classification = classify_import_safety_failure( + reason, + kind=kind, + code=code, + overrideable_hint=overrideable_hint, + ) + return { + "kind": kind or "file_safety_blocked", + "category": classification.category.value, + "code": classification.code, + "reason": classification.sanitized_reason, + "sanitized_reason": classification.sanitized_reason, + "source": source, + "retryable": classification.retryable, + "overrideable": classification.overrideable, + } + + +def normalize_import_safety_diagnostics( + diagnostics: Mapping[str, object], + *, + default_kind: str = "file_safety_blocked", + default_source: str = "file_safety", +) -> dict[str, object]: + """Normalize new or legacy safety diagnostics to the safe typed contract.""" + raw_reason = diagnostics.get("sanitized_reason") or diagnostics.get("reason") or "" + raw_kind = diagnostics.get("kind") or default_kind + raw_code = diagnostics.get("code") or diagnostics.get("category") + raw_source = diagnostics.get("source") or default_source + overrideable_hint = diagnostics.get("overrideable") + return build_import_safety_diagnostics( + str(raw_reason), + kind=str(raw_kind), + code=str(raw_code) if raw_code is not None else None, + source=str(raw_source), + overrideable_hint=(overrideable_hint if isinstance(overrideable_hint, bool) else None), + ) + + +def _safe_example_name(value: str) -> str: + normalized = value.replace("\\", "/").rstrip("/") + leaf = normalized.rsplit("/", maxsplit=1)[-1] + safe_leaf = "".join(character for character in leaf if character >= " " and character != "\x7f") + return safe_leaf or "File" + + +@dataclass(slots=True) +class _ImportSafetySummaryBucket: + category: ImportSafetyCategory + reason: object + retryable: bool + overrideable: bool + count: int = 0 + overrideable_count: int = 0 + codes: set[str] = field(default_factory=set) + examples: list[str] = field(default_factory=list) + + +@dataclass(slots=True) +class ImportSafetyFailureSummaryAccumulator: + """Accumulate a complete safety summary with bounded retained detail.""" + + example_limit: int = 3 + code_limit: int = 16 + _buckets: dict[ImportSafetyCategory, _ImportSafetySummaryBucket] = field( + init=False, + default_factory=dict, + ) + + def add(self, example: str, raw_diagnostics: Mapping[str, object]) -> None: + """Add one failure while retaining only bounded codes and examples.""" + diagnostics = normalize_import_safety_diagnostics(raw_diagnostics) + try: + category = ImportSafetyCategory(str(diagnostics["category"])) + except ValueError: + category = ImportSafetyCategory.UNKNOWN + bucket = self._buckets.get(category) + if bucket is None: + bucket = _ImportSafetySummaryBucket( + category=category, + reason=diagnostics["sanitized_reason"], + retryable=bool(diagnostics["retryable"]), + overrideable=bool(diagnostics["overrideable"]), + ) + self._buckets[category] = bucket + + bucket.count += 1 + code = str(diagnostics["code"]) + if code in bucket.codes or len(bucket.codes) < max(self.code_limit, 0): + bucket.codes.add(code) + safe_example = _safe_example_name(example) + if ( + len(bucket.examples) < max(self.example_limit, 0) + and safe_example not in bucket.examples + ): + bucket.examples.append(safe_example) + bucket.retryable = bucket.retryable and bool(diagnostics["retryable"]) + bucket.overrideable = bucket.overrideable and bool(diagnostics["overrideable"]) + if bool(diagnostics["overrideable"]): + bucket.overrideable_count += 1 + + def summaries(self) -> list[dict[str, object]]: + """Return deterministic category summaries for every accumulated row.""" + summaries: list[dict[str, object]] = [] + for category in ImportSafetyCategory: + bucket = self._buckets.get(category) + if bucket is None: + continue + summaries.append( + { + "category": category.value, + "label": _CATEGORY_LABELS[category], + "count": bucket.count, + "codes": sorted(bucket.codes), + "reason": bucket.reason, + "retryable": bucket.retryable, + "overrideable": bucket.overrideable, + "overrideable_count": bucket.overrideable_count, + "examples": bucket.examples, + "bulk_overrideable": ( + category is ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT + and bucket.overrideable_count > 0 + ), + } + ) + return summaries + + +def summarize_import_safety_failures( + failures: Iterable[tuple[str, Mapping[str, object]]], + *, + example_limit: int = 3, +) -> list[dict[str, object]]: + """Return deterministic category counts with bounded basename-only examples.""" + accumulator = ImportSafetyFailureSummaryAccumulator(example_limit=example_limit) + for example, raw_diagnostics in failures: + accumulator.add(example, raw_diagnostics) + return accumulator.summaries() diff --git a/src/pullbox/services/import_safety_source_cleanup.py b/src/pullbox/services/import_safety_source_cleanup.py new file mode 100644 index 00000000..bddf8009 --- /dev/null +++ b/src/pullbox/services/import_safety_source_cleanup.py @@ -0,0 +1,301 @@ +"""Explicit source cleanup for one-page import artifacts.""" + +from __future__ import annotations + +import asyncio +import os +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Final + +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +from sqlalchemy import func, select + +from pullbox.config import get_settings +from pullbox.core.config_resolver import get_application_secret +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.library_file_ownership import ( + build_file_identity_signature, + validate_file_identity_signature, +) +from pullbox.models.audit_log import AuditEventType +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobStatus, + ImportSeriesStatus, + ImportSourceType, +) +from pullbox.models.library import LibraryRoot +from pullbox.services.audit_service import AuditService +from pullbox.services.import_counters import recompute_file_counters, recompute_series_counters +from pullbox.services.import_review_actions import apply_safety_skip_to_file +from pullbox.services.import_runtime_settings import load_import_utility_trash_folder +from pullbox.services.import_safety_diagnostics import ImportSafetyCategory +from pullbox.services.import_story_arc_resolution import ( + refresh_story_arc_entries_for_import_files, +) +from pullbox.utilities.settings import ( + move_file_to_utility_trash, + resolve_trash_directory, + restore_file_from_utility_trash, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +_TOKEN_SALT: Final = "import-one-page-source-cleanup-v1" +_TOKEN_MAX_AGE_SECONDS: Final = 15 * 60 + + +@dataclass(frozen=True, slots=True) +class OnePageSourceCleanupPreview: + job_id: int + file_id: int + file_name: str + can_move_to_trash: bool + unavailable_reason: str + preview_token: str | None + + +@dataclass(frozen=True, slots=True) +class OnePageSourceCleanupResult: + trash_path: Path + + +@dataclass(frozen=True, slots=True) +class _CleanupContext: + job: ImportJob + imported_file: ImportedFile + imported_series: ImportedSeries + source: Path + signature: dict[str, int | str] + trash_dir: Path | None + unavailable_reason: str + + +def _serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(get_application_secret(), salt=_TOKEN_SALT) + + +def _is_within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + +async def _load_context( + session: AsyncSession, + job_id: int, + file_id: int, +) -> _CleanupContext: + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if job.status not in {ImportJobStatus.REVIEW, ImportJobStatus.COMPLETED}: + raise ValidationError( + "Source cleanup is available during review or after a completed import." + ) + imported_file = await session.get(ImportedFile, file_id) + if imported_file is None or imported_file.import_job_id != job_id: + raise NotFoundError("ImportedFile", file_id) + block = imported_file.diagnostics.get("safety_block", {}) + if ( + imported_file.status is not ImportedFileStatus.SAFETY_BLOCKED + or not isinstance(block, Mapping) + or block.get("category") != ImportSafetyCategory.SINGLE_PAGE_COMIC.value + ): + raise ValidationError("Only a one-page archive in safety review can use this action.") + imported_series = await session.get(ImportedSeries, imported_file.import_series_id) + if imported_series is None or imported_series.import_job_id != job_id: + raise NotFoundError("ImportedSeries", imported_file.import_series_id) + + raw_source = Path(imported_file.file_path).expanduser() + try: + if raw_source.is_symlink(): + raise ValidationError("Symlinked source files cannot be removed from import review.") + source = raw_source.resolve(strict=True) + signature = build_file_identity_signature(source) + validate_file_identity_signature(dict(imported_file.source_signature or {}), signature) + except ValidationError: + raise + except Exception as exc: + raise ValidationError( + "The source file changed or is unavailable. Re-scan before removing it." + ) from exc + + unavailable_reason = "" + if job.source_type is ImportSourceType.MYLAR3: + roots = list( + ( + await session.scalars( + select(LibraryRoot).where( + LibraryRoot.enabled.is_(True), + LibraryRoot.allow_referenced_registrations.is_(True), + ) + ) + ).all() + ) + containing = [] + for root in roots: + try: + resolved_root = Path(root.path).expanduser().resolve(strict=True) + except (OSError, RuntimeError, ValueError): + continue + if _is_within(source, resolved_root): + containing.append(root) + if not containing or not any(root.allow_managed_writes for root in containing): + unavailable_reason = ( + "This Mylar source is registered as reference-only. Enable managed writes for " + "its library root only if you intentionally want Pullbox to move source files." + ) + else: + try: + source_root = Path(job.source_path).expanduser().resolve(strict=True) + except (OSError, RuntimeError, ValueError): + source_root = source.parent + if not _is_within(source, source_root) or not os.access(source.parent, os.W_OK): + unavailable_reason = "Pullbox does not have permission to move this source file." + + configured_trash = await load_import_utility_trash_folder(session) + settings = get_settings() + trash_dir = resolve_trash_directory( + configured_trash, + library_root=settings.library_root, + data_dir=settings.data_dir, + ) + if trash_dir is None: + unavailable_reason = ( + unavailable_reason + or "Configure the Trash folder in Media Management before removing source files." + ) + return _CleanupContext( + job=job, + imported_file=imported_file, + imported_series=imported_series, + source=source, + signature=signature, + trash_dir=trash_dir, + unavailable_reason=unavailable_reason, + ) + + +async def preview_one_page_source_cleanup( + session: AsyncSession, + job_id: int, + file_id: int, + *, + actor_id: int, +) -> OnePageSourceCleanupPreview: + """Preview an individual source move without changing the source or database.""" + context = await _load_context(session, job_id, file_id) + token = None + if not context.unavailable_reason: + token = str( + _serializer().dumps( + { + "job_id": job_id, + "file_id": file_id, + "actor_id": actor_id, + "signature": context.signature, + } + ) + ) + return OnePageSourceCleanupPreview( + job_id=job_id, + file_id=file_id, + file_name=context.imported_file.file_name, + can_move_to_trash=token is not None, + unavailable_reason=context.unavailable_reason, + preview_token=token, + ) + + +def _load_token(token: str) -> Mapping[str, object]: + try: + payload = _serializer().loads(token, max_age=_TOKEN_MAX_AGE_SECONDS) + except SignatureExpired as exc: + raise ValidationError("The source cleanup preview expired. Preview it again.") from exc + except BadSignature as exc: + raise ValidationError("The source cleanup preview is invalid. Preview it again.") from exc + if not isinstance(payload, Mapping): + raise ValidationError("The source cleanup preview is invalid. Preview it again.") + return payload + + +async def move_one_page_source_to_trash( + session: AsyncSession, + job_id: int, + file_id: int, + *, + actor_id: int, + preview_token: str, + actor_username: str | None = None, + source_ip: str | None = None, +) -> OnePageSourceCleanupResult: + """Move one explicitly previewed source to Trash and mark it skipped.""" + payload = _load_token(preview_token) + if ( + payload.get("job_id") != job_id + or payload.get("file_id") != file_id + or payload.get("actor_id") != actor_id + ): + raise ValidationError("The source cleanup preview does not match this file.") + expected_signature = payload.get("signature") + if not isinstance(expected_signature, Mapping): + raise ValidationError("The source cleanup preview is invalid. Preview it again.") + + context = await _load_context(session, job_id, file_id) + if context.unavailable_reason or context.trash_dir is None: + raise ValidationError(context.unavailable_reason or "Trash is not configured.") + validate_file_identity_signature(dict(expected_signature), context.signature) + + relative_path = Path("import-review") / str(job_id) / context.source.name + trash_path = await asyncio.to_thread( + move_file_to_utility_trash, + context.source, + context.trash_dir, + relative_path=relative_path, + ) + try: + apply_safety_skip_to_file(context.imported_file) + await refresh_story_arc_entries_for_import_files( + session, + import_job_id=job_id, + import_file_ids=[context.imported_file.id], + ) + context.imported_series.selected_for_import = False + await recompute_file_counters( + session, + context.job, + series_ids=[context.imported_series.id], + ) + remaining_file_count = await session.scalar( + select(func.count(ImportedFile.id)).where( + ImportedFile.import_series_id == context.imported_series.id, + ImportedFile.status != ImportedFileStatus.SKIPPED, + ) + ) + if not remaining_file_count: + context.imported_series.status = ImportSeriesStatus.SKIPPED + await recompute_series_counters(session, context.job) + await AuditService.log_event( + session, + AuditEventType.IMPORT_SAFETY_SOURCE_TRASH, + source_ip=source_ip, + user_id=actor_id, + username=actor_username, + detail="One reviewed import source moved to Trash.", + metadata={"job_id": job_id, "file_id": file_id}, + ) + await session.commit() + except Exception: + await session.rollback() + await asyncio.to_thread(restore_file_from_utility_trash, trash_path, context.source) + raise + return OnePageSourceCleanupResult(trash_path=trash_path) diff --git a/src/pullbox/services/import_scan_helpers.py b/src/pullbox/services/import_scan_helpers.py index d16c8895..69c08a4c 100644 --- a/src/pullbox/services/import_scan_helpers.py +++ b/src/pullbox/services/import_scan_helpers.py @@ -2,20 +2,34 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable +import asyncio +import time +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from dataclasses import asdict +from functools import partial from pathlib import Path from typing import TYPE_CHECKING +import structlog from sqlalchemy import delete as sa_delete +from sqlalchemy import select as sa_select from pullbox.core.file_safety import ( FileSafetyError, + FileSafetyInspection, classify_resource_safety_exception, get_archive_size_limit_bytes, is_dangerous_file_blocking_enabled, run_safety_checks, ) +from pullbox.core.import_resources import bounded_thread_map, detect_import_resources +from pullbox.core.source_metadata import archive_entry_issue_hint_from_names from pullbox.models.import_job import ImportedFile, ImportedSeries, ImportJob +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.services.import_content_inspection import inspect_import_content +from pullbox.services.import_safety_diagnostics import build_import_safety_diagnostics +from pullbox.services.import_scan_reconciliation import reconcile_discovered_mylar_paths if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -23,11 +37,25 @@ from pullbox.core.collection_scanner import DiscoveredFile, DiscoveredSeries -FileSafetyCheck = Callable[["AsyncSession", Path], Awaitable[None]] +FileSafetyCheck = Callable[ + ["AsyncSession", Path], + Awaitable[FileSafetyInspection | None], +] +FileSafetyProgress = Callable[[int, int, str], Awaitable[None]] +logger = structlog.get_logger(__name__) async def reset_scan_artifacts(session: AsyncSession, job: ImportJob) -> None: """Clear scan-produced rows and counters before a fresh/recovered scan run.""" + staged_arc_ids = sa_select(ImportedStoryArc.id).where(ImportedStoryArc.import_job_id == job.id) + await session.execute( + sa_delete(ImportedStoryArcEntry).where( + ImportedStoryArcEntry.imported_story_arc_id.in_(staged_arc_ids) + ) + ) + await session.execute( + sa_delete(ImportedStoryArc).where(ImportedStoryArc.import_job_id == job.id) + ) await session.execute(sa_delete(ImportedFile).where(ImportedFile.import_job_id == job.id)) await session.execute(sa_delete(ImportedSeries).where(ImportedSeries.import_job_id == job.id)) @@ -53,19 +81,61 @@ async def reset_scan_artifacts(session: AsyncSession, job: ImportJob) -> None: await session.flush() -async def _build_default_file_safety_check(session: AsyncSession) -> FileSafetyCheck: - """Build a per-batch file safety checker with immutable config values.""" +InspectionResult = tuple[ + str, FileSafetyInspection | None, dict[str, object], FileSafetyError | None +] + + +@asynccontextmanager +async def _inspection_results( + session: AsyncSession, + paths: list[str], + *, + check_file_safety: FileSafetyCheck | None, + workers: int, +) -> AsyncIterator[AsyncIterator[InspectionResult]]: + if check_file_safety is not None: + + async def serial_results() -> AsyncGenerator[InspectionResult, None]: + # Injected checkers may use the caller's database session. + for path in paths: + try: + inspection = await check_file_safety(session, Path(path)) + content = ( + await asyncio.to_thread(inspect_import_content, Path(path), inspection) + if inspection is not None + else {} + ) + yield path, inspection, content, None + except FileSafetyError as exc: + yield path, None, {}, exc + + iterator = serial_results() + try: + yield iterator + finally: + await iterator.aclose() + return + block_dangerous = await is_dangerous_file_blocking_enabled(session) max_archive_size = await get_archive_size_limit_bytes(session) - async def _check_file_safety(_session: AsyncSession, path: Path) -> None: - run_safety_checks( - path, - block_dangerous=block_dangerous, - max_archive_size=max_archive_size, - ) + def inspect(path: str) -> InspectionResult: + try: + inspection = run_safety_checks( + Path(path), + block_dangerous=block_dangerous, + max_archive_size=max_archive_size, + ) + content = ( + inspect_import_content(Path(path), inspection) if inspection is not None else {} + ) + return path, inspection, content, None + except FileSafetyError as exc: + return path, None, {}, exc - return _check_file_safety + async with bounded_thread_map(inspect, paths, workers=workers) as results: + yield results async def validate_discovered_files_safety( @@ -73,34 +143,118 @@ async def validate_discovered_files_safety( discovered_list: list[DiscoveredSeries], *, check_file_safety: FileSafetyCheck | None = None, + progress_callback: FileSafetyProgress | None = None, + worker_count: int = 0, ) -> None: """Run safety checks once per unique discovered source file.""" - effective_check_file_safety = check_file_safety - if effective_check_file_safety is None: - effective_check_file_safety = await _build_default_file_safety_check(session) - files_by_path: dict[str, list[DiscoveredFile]] = {} for discovered in discovered_list: for discovered_file in discovered.files: + # Source containment/signature failures are never archive overrides. + if isinstance(discovered_file.metadata_diagnostics.get("file_safety"), dict): + continue files_by_path.setdefault(discovered_file.file_path, []).append(discovered_file) - for file_path, discovered_files in files_by_path.items(): - try: - await effective_check_file_safety(session, Path(file_path)) - except FileSafetyError as exc: - resource_block = classify_resource_safety_exception(exc) - safety_block = ( - resource_block.to_diagnostics() - if resource_block is not None - else { - "kind": "file_safety_blocked", - "reason": exc.reason, - "details": list(exc.details), - "source": "file_safety", - "overrideable": False, - } - ) - for discovered_file in discovered_files: - metadata_diagnostics = dict(discovered_file.metadata_diagnostics) - metadata_diagnostics["file_safety"] = safety_block - discovered_file.metadata_diagnostics = metadata_diagnostics + total = len(files_by_path) + if progress_callback is not None: + await progress_callback(0, total, "") + resources = await asyncio.to_thread(detect_import_resources) + workers = ( + resources.inspection_workers(requested=worker_count) if check_file_safety is None else 1 + ) + started_at = time.monotonic() + completed = 0 + async with _inspection_results( + session, + list(files_by_path), + check_file_safety=check_file_safety, + workers=workers, + ) as results: + async for file_path, inspection, content_diagnostics, error in results: + completed += 1 + discovered_files = files_by_path[file_path] + try: + if error is not None: + raise error + except FileSafetyError as exc: + resource_block = classify_resource_safety_exception(exc) + safety_block = build_import_safety_diagnostics( + exc.reason, + details=exc.details, + kind=(resource_block.kind if resource_block is not None else None), + source=(resource_block.source if resource_block is not None else "file_safety"), + overrideable_hint=( + resource_block.overrideable if resource_block is not None else False + ), + ) + for discovered_file in discovered_files: + metadata_diagnostics = dict(discovered_file.metadata_diagnostics) + metadata_diagnostics["file_safety"] = safety_block + discovered_file.metadata_diagnostics = metadata_diagnostics + else: + if inspection is None: + continue + for discovered_file in discovered_files: + discovered_file.metadata_diagnostics = { + **discovered_file.metadata_diagnostics, + **content_diagnostics, + } + archive_report = next( + ( + report + for report in inspection.archives + if report.archive_path == Path(file_path) + ), + None, + ) + if archive_report is None: + continue + for discovered_file in discovered_files: + metadata_diagnostics = dict(discovered_file.metadata_diagnostics) + if archive_report.comicinfo_entry_count == 0: + async with bounded_thread_map( + partial( + archive_entry_issue_hint_from_names, + expected_series_name=discovered_file.parsed_series, + ), + [list(archive_report.entry_names)], + workers=1, + ) as hints: + archive_hint = await anext(hints) + metadata_diagnostics.update( + { + "archive_metadata_loaded": True, + "archive_metadata_deferred": False, + "archive_entry_issue_hint_checked": True, + "has_comicinfo": False, + } + ) + if archive_hint is not None: + metadata_diagnostics["archive_entry_issue_hint"] = dict(archive_hint) + discovered_file.metadata_diagnostics = metadata_diagnostics + continue + archive_evidence: dict[str, object] = { + "member_index_scanned": True, + "comicinfo_entry_count": archive_report.comicinfo_entry_count, + } + if archive_report.comicinfo_entry is not None: + archive_evidence["comicinfo_entry"] = archive_report.comicinfo_entry + if archive_report.comicinfo is not None: + archive_evidence["comicinfo"] = asdict(archive_report.comicinfo) + if archive_report.comicinfo_error is not None: + archive_evidence["comicinfo_error"] = archive_report.comicinfo_error + metadata_diagnostics["archive_member_evidence"] = archive_evidence + discovered_file.metadata_diagnostics = metadata_diagnostics + finally: + if progress_callback is not None: + await progress_callback(completed, total, file_path) + + await asyncio.to_thread(reconcile_discovered_mylar_paths, discovered_list) + logger.info( + "import_archive_inspection_batch", + files_checked=completed, + inspection_workers=workers, + effective_cpus=resources.cpu_count, + available_memory_mib=resources.available_memory_bytes // (1024**2), + duration_ms=round((time.monotonic() - started_at) * 1000), + ) diff --git a/src/pullbox/services/import_scan_materialization.py b/src/pullbox/services/import_scan_materialization.py index 301513a9..1af45367 100644 --- a/src/pullbox/services/import_scan_materialization.py +++ b/src/pullbox/services/import_scan_materialization.py @@ -5,6 +5,8 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING +from sqlalchemy import insert + from pullbox.models.import_job import ( ImportedFile, ImportedFileStatus, @@ -19,6 +21,30 @@ from pullbox.models.import_job import ImportJob +_SOURCE_LAYOUT_REVIEW_REASON = "selected_layout_no_match" +_SOURCE_LAYOUT_REVIEW_MESSAGE = ( + "This file does not fit the selected source layout. Review its series before importing." +) +_MYLAR_FOLDER_SCOPE_REVIEW_MESSAGE = ( + "This unrecorded file appears to belong to another series in the selected Mylar folder." +) + + +def _requires_source_layout_review(metadata_diagnostics: dict[str, object]) -> bool: + """Return whether a selected no-fallback layout requires explicit review.""" + layout = metadata_diagnostics.get("source_layout") + return ( + isinstance(layout, dict) + and layout.get("review_required") is True + and layout.get("review_reason") == _SOURCE_LAYOUT_REVIEW_REASON + ) + + +def _requires_mylar_folder_scope_review(metadata_diagnostics: dict[str, object]) -> bool: + """Return whether an unrecorded file contradicts its owning Mylar series folder.""" + return isinstance(metadata_diagnostics.get("mylar3_folder_scope_conflict"), dict) + + async def materialize_discovered_scan_results( session: AsyncSession, job: ImportJob, @@ -27,6 +53,27 @@ async def materialize_discovered_scan_results( """Persist discovered scanner output as import review series/file rows.""" series_pairs: list[tuple[DiscoveredSeries, ImportedSeries]] = [] for discovered in discovered_list: + mylar_path_incompatible = ( + dict(discovered.diagnostics).get("kind") == "mylar3_path_incompatible" + ) + layout_review_count = sum( + _requires_source_layout_review(dict(discovered_file.metadata_diagnostics)) + for discovered_file in discovered.files + ) + all_files_require_layout_review = bool(discovered.files) and layout_review_count == len( + discovered.files + ) + series_diagnostics = dict(discovered.diagnostics) + if layout_review_count: + series_diagnostics["source_layout_review_files"] = layout_review_count + if all_files_require_layout_review: + series_diagnostics.update( + { + "kind": "source_layout_review", + "reason": _SOURCE_LAYOUT_REVIEW_REASON, + "rejection_reason": _SOURCE_LAYOUT_REVIEW_MESSAGE, + } + ) item = ImportedSeries( import_job_id=job.id, raw_series_name=discovered.raw_series_name, @@ -36,8 +83,12 @@ async def materialize_discovered_scan_results( sample_paths=[str(p) for p in discovered.sample_paths], source_folder=discovered.source_folder, has_files=discovered.has_files, - status=ImportSeriesStatus.PENDING, - diagnostics=dict(discovered.diagnostics), + status=( + ImportSeriesStatus.NO_MATCH + if all_files_require_layout_review or mylar_path_incompatible + else ImportSeriesStatus.PENDING + ), + diagnostics=series_diagnostics, ) if discovered.mylar3_cv_id: item.cv_id = discovered.mylar3_cv_id @@ -56,16 +107,20 @@ async def materialize_discovered_scan_results( await session.flush() total_files = 0 + file_rows: list[dict[str, object]] = [] for discovered, series_item in series_pairs: series_file_count = 0 for df in discovered.files: metadata_diagnostics = dict(df.metadata_diagnostics) safety_block = metadata_diagnostics.pop("file_safety", None) - file_status = ( - ImportedFileStatus.SAFETY_BLOCKED - if isinstance(safety_block, dict) - else ImportedFileStatus.PENDING - ) + source_layout_review = _requires_source_layout_review(metadata_diagnostics) + mylar_folder_scope_review = _requires_mylar_folder_scope_review(metadata_diagnostics) + if isinstance(safety_block, dict): + file_status = ImportedFileStatus.SAFETY_BLOCKED + elif source_layout_review or mylar_folder_scope_review: + file_status = ImportedFileStatus.NO_MATCH + else: + file_status = ImportedFileStatus.PENDING diagnostics = { "source_issue_type": df.issue_type.value, "comicvine_series_id": df.comicvine_series_id, @@ -74,10 +129,34 @@ async def materialize_discovered_scan_results( "metadata_signals": dict(df.metadata_signals), "source_metadata": metadata_diagnostics, } + cross_folder_reconciliation = metadata_diagnostics.get( + "mylar3_cross_folder_reconciliation" + ) + if isinstance(cross_folder_reconciliation, dict): + diagnostics["mylar3_cross_folder_reconciliation"] = dict( + cross_folder_reconciliation + ) if isinstance(safety_block, dict): diagnostics["safety_block"] = safety_block + elif source_layout_review: + diagnostics.update( + { + "kind": "source_layout_review", + "reason": "selected_layout_no_match", + "rejection_reason": _SOURCE_LAYOUT_REVIEW_MESSAGE, + } + ) + elif mylar_folder_scope_review: + diagnostics.update( + { + "kind": "source_scope_review", + "reason": "mylar3_folder_scope_conflict", + "rejection_reason": _MYLAR_FOLDER_SCOPE_REVIEW_MESSAGE, + "preserve_series_match": True, + } + ) error_message = safety_block.get("reason") if isinstance(safety_block, dict) else None - file_item = ImportedFile( + file_item = dict( import_job_id=job.id, import_series_id=series_item.id, file_path=df.file_path, @@ -90,15 +169,23 @@ async def materialize_discovered_scan_results( has_comicinfo=df.has_comicinfo, comicvine_issue_id=df.comicvine_issue_id, issue_number_raw=df.issue_number_raw, + source_folder_cohort_key=df.source_folder_cohort_key, + source_ordinal=df.source_ordinal, + source_signature=dict(df.source_signature), status=file_status, include_in_import=False, error_message=error_message, diagnostics=diagnostics, ) - session.add(file_item) + file_rows.append(file_item) + if len(file_rows) >= 500: + await session.execute(insert(ImportedFile), file_rows) + file_rows.clear() series_file_count += 1 series_item.files_total = series_file_count total_files += series_file_count + if file_rows: + await session.execute(insert(ImportedFile), file_rows) if total_files: await session.flush() diff --git a/src/pullbox/services/import_scan_pipeline.py b/src/pullbox/services/import_scan_pipeline.py index c62e8119..9a7291ef 100644 --- a/src/pullbox/services/import_scan_pipeline.py +++ b/src/pullbox/services/import_scan_pipeline.py @@ -4,21 +4,51 @@ import asyncio import contextlib +import inspect import time from datetime import UTC, datetime from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast -from pullbox.core.exceptions import JobCancelledError, JobPausedError, NotFoundError +from sqlalchemy import func, select + +from pullbox.core.exceptions import ( + JobCancelledError, + JobPausedError, + NotFoundError, + ValidationError, +) +from pullbox.core.library_layout import SourceLayoutSpec +from pullbox.core.mylar3_reader import ( + Mylar3ArcSettingsSnapshot, + Mylar3CollectionSnapshot, + Mylar3ImportMetadataSnapshot, +) from pullbox.core.sqlite_lock import is_sqlite_locked_error from pullbox.models.import_job import ( + ImportedSeries, + ImportFileHandlingMode, ImportJob, ImportJobStatus, ImportSeriesStatus, ImportSourceType, ) from pullbox.schemas.import_job import ImportProgressEvent +from pullbox.services.import_mylar_scan_progress import MylarScanProgress from pullbox.services.import_progress_runtime import current_item_payload +from pullbox.services.import_referenced_sources import ( + load_mylar_reference_root_boundaries, + validate_mylar_in_place_files, +) +from pullbox.services.import_story_arc_resolution import ( + StoryArcResolutionResult, + resolve_staged_story_arc_entries, +) +from pullbox.services.import_story_arc_staging import ( + StoryArcStagingResult, + stage_folder_story_arcs, + stage_mylar_story_arcs, +) from pullbox.services.import_workflow_state import ( SCAN_PROGRESS_ANALYZE_START, SCAN_PROGRESS_FILE_MATCH_START, @@ -49,10 +79,7 @@ SlowPhaseDelayFunc = Callable[[], Awaitable[None]] ResolveFileExtensionsFunc = Callable[[AsyncSession, str | None], Awaitable[frozenset[str]]] AutoDetectPathMapFunc = Callable[[Path], dict[str, str] | None] - ValidateDiscoveredFilesFunc = Callable[ - [AsyncSession, list[DiscoveredSeries]], - Awaitable[None], - ] + ValidateDiscoveredFilesFunc = Callable[..., Awaitable[None]] MaterializeScanResultsFunc = Callable[..., Awaitable[Any]] RaiseIfCancelledFunc = Callable[[AsyncSession, int], Awaitable[None]] DeduplicateSeriesFunc = Callable[..., Awaitable[None]] @@ -62,6 +89,16 @@ PhaseProgressFunc = Callable[[int, int, int, int], int] +def _scan_item_percentage(phase: str, progress: int, current_series: str | None) -> int: + # Named discovery events describe a completed cohort, not overall work. + if current_series: + return 100 + if phase == "scanning": + span = SCAN_PROGRESS_MATERIALIZE_END - SCAN_PROGRESS_MATERIALIZE_START + return max(0, min(100, round((progress - SCAN_PROGRESS_MATERIALIZE_START) * 100 / span))) + return 0 + + def _scan_failure_message(exc: Exception) -> str: """Return a user-facing import scan failure message.""" if is_sqlite_locked_error(exc): @@ -72,6 +109,57 @@ def _scan_failure_message(exc: Exception) -> str: return str(exc) +async def _log_story_arc_staging_summary( + session: AsyncSession, + *, + job_id: int, + source_type: ImportSourceType, + result: StoryArcStagingResult, + log_event: LogEventFunc, +) -> None: + """Log path-free counts for review-only story-arc staging.""" + await log_event( + session, + job_id, + "INFO", + "import_story_arc_staging_completed", + message="Story-arc evidence staged for review", + source_type=source_type.value, + arcs_staged=result.arcs_staged, + entries_staged=result.entries_staged, + needs_review=result.needs_review, + cohorts_examined=result.cohorts_examined, + cohorts_skipped=result.cohorts_skipped, + readlist_present=result.readlist_present, + readlist_count=result.readlist_count, + ) + + +async def _log_story_arc_resolution_summary( + session: AsyncSession, + *, + job_id: int, + result: StoryArcResolutionResult, + log_event: LogEventFunc, +) -> None: + """Log path-free story-arc resolution counts.""" + await log_event( + session, + job_id, + "INFO", + "import_story_arc_resolution_completed", + message="Staged story-arc entries resolved for review", + entries_examined=result.entries_examined, + resolved=result.resolved, + pending=result.pending, + missing=result.missing, + ambiguous=result.ambiguous, + conflicts=result.conflicts, + skipped=result.skipped, + linked_files=result.linked_files, + ) + + async def run_import_scan_pipeline( session: AsyncSession, job_id: int, @@ -172,19 +260,12 @@ async def emit_scan_progress( message=message, current_series=current_series, current_series_status=current_series_status, - estimated_seconds_remaining=estimate_remaining_seconds( - ( - job.import_started_at - if status == ImportJobStatus.IMPORTING - else job.scan_started_at - ), - progress, - ), + estimated_seconds_remaining=None, **current_item_payload( kind="series" if current_series else "scan", stage=phase, name=current_series, - progress_pct=progress, + progress_pct=_scan_item_percentage(phase, progress, current_series), ), **job_stats(job), ), @@ -216,19 +297,12 @@ async def forward_live_scan_progress( message=message, current_series=current_series, current_series_status=current_series_status, - estimated_seconds_remaining=estimate_remaining_seconds( - ( - job.import_started_at - if status == ImportJobStatus.IMPORTING - else job.scan_started_at - ), - progress, - ), + estimated_seconds_remaining=None, **current_item_payload( kind="series" if current_series else "scan", stage=phase, name=current_series, - progress_pct=progress, + progress_pct=_scan_item_percentage(phase, progress, current_series), ), **job_stats(job), ), @@ -238,25 +312,31 @@ async def forward_live_scan_progress( ) scan_materialized_incrementally = False + discovered_list: list[DiscoveredSeries] = [] scan_phase_started_at = time.monotonic() if job.source_type == ImportSourceType.MYLAR3: - discovered_list = await _load_mylar3_discovered_series( + discovered_count = await _load_mylar3_discovered_series( session, job, job_id=job_id, mylar3_reader_cls=mylar3_reader_cls, auto_detect_mylar3_path_map=auto_detect_mylar3_path_map, log_event=log_event, + validate_discovered_files_safety=validate_discovered_files_safety, + materialize_discovered_scan_results=materialize_discovered_scan_results, + raise_if_cancelled=raise_if_cancelled, + progress_callback=progress_callback, ) await emit_scan_progress( status=ImportJobStatus.SCANNING, phase="scanning", - message=f"Loaded {len(discovered_list)} series from Mylar3.", + message=f"Loaded {discovered_count} series from Mylar3.", progress=SCAN_PROGRESS_MATERIALIZE_END, ) + scan_materialized_incrementally = True else: - discovered_list = await _scan_collection_discovered_series( + discovered_count = await _scan_collection_discovered_series( session, job, scanner_cls=scanner_cls, @@ -266,6 +346,7 @@ async def forward_live_scan_progress( validate_discovered_files_safety=validate_discovered_files_safety, materialize_discovered_scan_results=materialize_discovered_scan_results, log_event=log_event, + raise_if_cancelled=raise_if_cancelled, ) scan_materialized_incrementally = True @@ -273,7 +354,7 @@ async def forward_live_scan_progress( await emit_scan_progress( status=ImportJobStatus.SCANNING, phase="scanning", - message=f"Scan complete: {len(discovered_list)} series ready for analysis.", + message=f"Scan complete: {discovered_count} series ready for analysis.", progress=SCAN_PROGRESS_MATERIALIZE_END, ) @@ -285,13 +366,31 @@ async def forward_live_scan_progress( discovered_list, ) + if job.source_type == ImportSourceType.FILESYSTEM: + + async def check_folder_staging_cancellation() -> None: + await raise_if_cancelled(session, job_id) + + story_arc_staging = await stage_folder_story_arcs( + session, + import_job_id=job_id, + cancellation_check=check_folder_staging_cancellation, + ) + await _log_story_arc_staging_summary( + session, + job_id=job_id, + source_type=job.source_type, + result=story_arc_staging, + log_event=log_event, + ) + await log_event( session, job_id, "INFO", "import_scan_completed", - message=f"Scan complete: {len(discovered_list)} series found", - series_found=len(discovered_list), + message=f"Scan complete: {discovered_count} series found", + series_found=discovered_count, duration_ms=round((time.monotonic() - scan_phase_started_at) * 1000), ) scan_duration_ms = round((time.monotonic() - scan_phase_started_at) * 1000) @@ -386,16 +485,17 @@ async def forward_live_scan_progress( job_id=job_id, status=ImportJobStatus.FILE_MATCHING, phase="file_matching", - progress=SCAN_PROGRESS_FILE_MATCH_START, - message="Matching files to issues...", - estimated_seconds_remaining=estimate_remaining_seconds( - job.scan_started_at, - SCAN_PROGRESS_FILE_MATCH_START, + progress=int( + (job.progress_snapshot or {}).get( + "progress", SCAN_PROGRESS_FILE_MATCH_START + ) ), + message="Matching files to issues...", + estimated_seconds_remaining=None, **current_item_payload( kind="scan", stage="file_matching", - progress_pct=SCAN_PROGRESS_FILE_MATCH_START, + progress_pct=0, ), **job_stats(job), ), @@ -406,6 +506,21 @@ async def forward_live_scan_progress( file_matching_started_at = time.monotonic() await run_file_matching(session, job, progress_callback=progress_callback) file_matching_duration_ms = round((time.monotonic() - file_matching_started_at) * 1000) + + async def check_story_arc_resolution_cancellation() -> None: + await raise_if_cancelled(session, job_id) + + story_arc_resolution = await resolve_staged_story_arc_entries( + session, + import_job_id=job_id, + cancellation_check=check_story_arc_resolution_cancellation, + ) + await _log_story_arc_resolution_summary( + session, + job_id=job_id, + result=story_arc_resolution, + log_event=log_event, + ) await raise_if_cancelled(session, job_id) job.status = ImportJobStatus.REVIEW @@ -507,45 +622,274 @@ async def _load_mylar3_discovered_series( mylar3_reader_cls: Any, auto_detect_mylar3_path_map: AutoDetectPathMapFunc, log_event: LogEventFunc, -) -> list[DiscoveredSeries]: + validate_discovered_files_safety: ValidateDiscoveredFilesFunc | None = None, + materialize_discovered_scan_results: MaterializeScanResultsFunc | None = None, + raise_if_cancelled: RaiseIfCancelledFunc | None = None, + progress_callback: ProgressCallback | None = None, +) -> int: db_path = Path(job.source_path) if db_path.is_dir(): db_path = db_path / "mylar.db" + if not job.mylar3_path_map_confirmed: + raise ValidationError( + "Return to Import Step 1 and confirm the Mylar path mapping before scanning." + ) path_map: dict[str, str] | None = job.mylar3_path_map - if not path_map: - detected = await asyncio.to_thread(lambda: auto_detect_mylar3_path_map(db_path)) - path_map = detected - if detected: - await log_event( - session, - job_id, - "INFO", - "mylar3_path_map_detected", - message="Auto-detected Mylar3 path mapping", - path_map=detected, - ) - else: - await log_event( - session, - job_id, - "DEBUG", - "mylar3_path_map_not_detected", - message="No Mylar3 path mapping auto-detected", - db_path=str(db_path), - ) + in_place = job.file_handling_mode == ImportFileHandlingMode.IN_PLACE + in_place_root_boundaries = ( + await load_mylar_reference_root_boundaries(session) if in_place else () + ) + reader_options: dict[str, object] = {} + if in_place: + reader_options["include_missing_files"] = True + reader_options["reference_root_boundaries"] = tuple( + (boundary.lexical, boundary.resolved) for boundary in in_place_root_boundaries + ) reader = mylar3_reader_cls( db_path=db_path, path_map=path_map or None, + source_layout=SourceLayoutSpec.from_dict(dict(job.source_layout_snapshot or {})), + **reader_options, ) - discovered_list = await reader.read_series() - job.scan_total_files = sum(series.file_count for series in discovered_list) - job.scan_total_dirs = len( - {series.source_folder for series in discovered_list if series.source_folder} + if raise_if_cancelled is not None: + await raise_if_cancelled(session, job_id) + + async def check_mylar_staging_cancellation() -> None: + if raise_if_cancelled is not None: + await raise_if_cancelled(session, job_id) + + read_import_metadata = getattr(reader, "read_import_metadata", None) + iter_story_arc_pages = getattr(reader, "iter_import_story_arc_pages", None) + iter_series_pages = getattr(reader, "iter_import_series_pages", None) + paged_reader = ( + inspect.iscoroutinefunction(read_import_metadata) + and inspect.isasyncgenfunction(iter_story_arc_pages) + and inspect.isasyncgenfunction(iter_series_pages) + ) + legacy_snapshot: Mylar3CollectionSnapshot | None = None + if paged_reader: + metadata = cast("Mylar3ImportMetadataSnapshot", await reader.read_import_metadata()) + else: + legacy_snapshot = await _read_mylar3_collection_snapshot(reader) + metadata = Mylar3ImportMetadataSnapshot( + storyarcs_present=legacy_snapshot.storyarcs_present, + readlist_present=legacy_snapshot.readlist_present, + readlist_count=legacy_snapshot.readlist_count, + arc_settings=legacy_snapshot.arc_settings, + series_count=len(legacy_snapshot.series), + ) + + story_arc_staging = StoryArcStagingResult( + readlist_present=metadata.readlist_present, + readlist_count=metadata.readlist_count, + ) + + async def stage_arc_page( + arc_page: tuple[Any, ...], + *, + source_ordinal_offset: int, + ) -> StoryArcStagingResult: + page_snapshot = Mylar3CollectionSnapshot( + series=(), + story_arcs=cast("Any", arc_page), + storyarcs_present=metadata.storyarcs_present, + readlist_present=metadata.readlist_present, + readlist_count=metadata.readlist_count, + arc_settings=metadata.arc_settings, + ) + return await stage_mylar_story_arcs( + session, + import_job_id=job_id, + snapshot=page_snapshot, + source_ordinal_offset=source_ordinal_offset, + cancellation_check=check_mylar_staging_cancellation, + ) + + def combine_staging( + current: StoryArcStagingResult, + page_result: StoryArcStagingResult, + ) -> StoryArcStagingResult: + return StoryArcStagingResult( + arcs_staged=current.arcs_staged + page_result.arcs_staged, + entries_staged=current.entries_staged + page_result.entries_staged, + needs_review=current.needs_review + page_result.needs_review, + cohorts_examined=current.cohorts_examined + page_result.cohorts_examined, + cohorts_skipped=current.cohorts_skipped + page_result.cohorts_skipped, + readlist_present=metadata.readlist_present, + readlist_count=metadata.readlist_count, + ) + + source_ordinal_offset = 0 + if paged_reader: + async for raw_arc_page in reader.iter_import_story_arc_pages(): + await check_mylar_staging_cancellation() + arc_page = tuple(raw_arc_page) + if not arc_page: + continue + page_result = await stage_arc_page( + arc_page, + source_ordinal_offset=source_ordinal_offset, + ) + story_arc_staging = combine_staging(story_arc_staging, page_result) + source_ordinal_offset += len(arc_page) + await session.commit() + elif legacy_snapshot is not None: + page_result = await stage_arc_page( + tuple(legacy_snapshot.story_arcs), + source_ordinal_offset=0, + ) + story_arc_staging = combine_staging(story_arc_staging, page_result) + + await _log_story_arc_staging_summary( + session, + job_id=job_id, + source_type=job.source_type, + result=story_arc_staging, + log_event=log_event, ) - job.series_found = len(discovered_list) await session.commit() - return list(discovered_list) + + scan_progress = MylarScanProgress( + session, + job, + metadata.series_count, + check_mylar_staging_cancellation, + callback=progress_callback, + ) + + path_status_counts: dict[str, int] = {} + mapping_applied_series = 0 + incompatible_series = 0 + series_count = 0 + file_count = 0 + fallback_source_folders: set[str] = set() + + async def persist_series_page(raw_page: tuple[Any, ...]) -> None: + nonlocal incompatible_series, mapping_applied_series, series_count, file_count + await check_mylar_staging_cancellation() + page = cast("list[DiscoveredSeries]", list(raw_page)) + if not page: + return + page_started_at = time.monotonic() + source_rows_read = getattr(reader, "import_series_rows_read", None) + scan_progress.source_page_end = ( + source_rows_read + if isinstance(source_rows_read, int) + else scan_progress.source_completed + len(page) + ) + await scan_progress.report_safety(0, sum(len(item.files) for item in page), "") + inspection_started_at = time.monotonic() + if in_place: + await asyncio.to_thread( + validate_mylar_in_place_files, + page, + in_place_root_boundaries, + ) + if validate_discovered_files_safety is not None: + await validate_discovered_files_safety( + session, page, progress_callback=scan_progress.report_safety + ) + if materialize_discovered_scan_results is not None: + persistence_started_at = time.monotonic() + await materialize_discovered_scan_results(session, job, page) + else: + persistence_started_at = time.monotonic() + persistence_duration_ms = round((time.monotonic() - persistence_started_at) * 1000) + inspection_duration_ms = round((persistence_started_at - inspection_started_at) * 1000) + job.scan_completed_at = None + + for series in page: + series_count += 1 + file_count += series.file_count + if materialize_discovered_scan_results is None and series.source_folder: + fallback_source_folders.add(series.source_folder) + path_details = series.diagnostics.get("mylar3_path") + if not isinstance(path_details, dict): + continue + status = path_details.get("status") + if isinstance(status, str): + path_status_counts[status] = path_status_counts.get(status, 0) + 1 + if status not in {"local", "mapped"}: + incompatible_series += 1 + if path_details.get("mapping_applied") is True: + mapping_applied_series += 1 + + job.scan_total_files = file_count + job.series_found = series_count + await log_event( + session, + job_id, + "INFO", + "import_mylar_batch_scanned", + message=f"Prepared {series_count} series and {file_count} file records from Mylar.", + series_found=series_count, + files_found=file_count, + inspection_duration_ms=inspection_duration_ms, + persistence_duration_ms=persistence_duration_ms, + duration_ms=round((time.monotonic() - page_started_at) * 1000), + ) + await scan_progress.checkpoint_page() + await check_mylar_staging_cancellation() + + if paged_reader: + async for raw_series_page in reader.iter_import_series_pages(): + await persist_series_page(tuple(raw_series_page)) + elif legacy_snapshot is not None: + await persist_series_page(tuple(legacy_snapshot.series)) + + if materialize_discovered_scan_results is not None: + distinct_source_folders = await session.scalar( + select(func.count(func.distinct(ImportedSeries.source_folder))).where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.source_folder.is_not(None), + ImportedSeries.source_folder != "", + ) + ) + job.scan_total_dirs = int(distinct_source_folders or 0) + else: + job.scan_total_dirs = len(fallback_source_folders) + + await log_event( + session, + job_id, + "INFO", + "mylar3_path_resolution", + message="Resolved Mylar comic folders", + path_status_counts=dict(sorted(path_status_counts.items())), + mapping_applied_series=mapping_applied_series, + incompatible_series=incompatible_series, + ) + job.scan_total_files = file_count + job.series_found = series_count + job.scan_completed_at = datetime.now(UTC) + await session.commit() + return series_count + + +async def _read_mylar3_collection_snapshot(reader: Any) -> Mylar3CollectionSnapshot: + """Read one complete Mylar snapshot, with a series-only test-double fallback.""" + read_collection = getattr(reader, "read_collection", None) + if callable(read_collection) and inspect.iscoroutinefunction(read_collection): + return cast("Mylar3CollectionSnapshot", await read_collection()) + + read_snapshot = getattr(reader, "read_snapshot", None) + if callable(read_snapshot) and inspect.iscoroutinefunction(read_snapshot): + return cast("Mylar3CollectionSnapshot", await read_snapshot()) + + discovered_list = await reader.read_series() + return Mylar3CollectionSnapshot( + series=tuple(discovered_list), + story_arcs=(), + storyarcs_present=False, + readlist_present=False, + readlist_count=0, + arc_settings=Mylar3ArcSettingsSnapshot( + present=False, + parse_warnings=(), + values=(), + ), + ) async def _scan_collection_discovered_series( @@ -559,10 +903,11 @@ async def _scan_collection_discovered_series( validate_discovered_files_safety: ValidateDiscoveredFilesFunc | None = None, materialize_discovered_scan_results: MaterializeScanResultsFunc | None = None, log_event: LogEventFunc | None = None, -) -> list[DiscoveredSeries]: + raise_if_cancelled: RaiseIfCancelledFunc | None = None, +) -> int: custom_exts = await resolve_import_file_extensions(session, job.file_formats) - checked_paths: set[str] = set() batch: list[DiscoveredSeries] = [] + series_found = 0 last_batch_flush_at = time.monotonic() async def forward_live_scan_progress( @@ -598,6 +943,10 @@ async def capture_scan_totals(files_found: int, directories_visited: int) -> Non job.scan_total_dirs = directories_visited job.scan_total_files = files_found + async def check_scan_cancellation() -> None: + if raise_if_cancelled is not None: + await raise_if_cancelled(session, job.id) + materialized_files = 0 last_materialized_emit = 0.0 @@ -638,18 +987,22 @@ async def flush_scan_batch(*, force: bool = False) -> None: batch_to_flush = list(batch) batch.clear() last_batch_flush_at = now + inspection_started_at = time.monotonic() if validate_discovered_files_safety is not None: batch_for_validation: list[DiscoveredSeries] = [] + batch_checked_paths: set[str] = set() for discovered in batch_to_flush: pending_files = [ discovered_file for discovered_file in discovered.files - if discovered_file.file_path not in checked_paths + if discovered_file.file_path not in batch_checked_paths ] if not pending_files: continue - checked_paths.update(discovered_file.file_path for discovered_file in pending_files) + batch_checked_paths.update( + discovered_file.file_path for discovered_file in pending_files + ) batch_for_validation.append( discovered.__class__( raw_series_name=discovered.raw_series_name, @@ -671,8 +1024,14 @@ async def flush_scan_batch(*, force: bool = False) -> None: if batch_for_validation: await validate_discovered_files_safety(session, batch_for_validation) + persistence_started_at = time.monotonic() if materialize_discovered_scan_results is not None: await materialize_discovered_scan_results(session, job, batch_to_flush) + # The shared materializer also supports one-shot callers and sets + # this counter to the size of the list it receives. Restore the + # cumulative scanner count after each incremental batch so large + # scans do not appear to contain only their final (partial) batch. + job.series_found = series_found if log_event is not None: await log_event( @@ -680,11 +1039,13 @@ async def flush_scan_batch(*, force: bool = False) -> None: job.id, "DEBUG", "import_scan_batch_discovered", - message=( - f"Discovered {len(batch_to_flush)} series ({len(discovered_list)} total so far)" - ), + message=f"Discovered {len(batch_to_flush)} series ({series_found} total so far)", series_found_in_batch=len(batch_to_flush), - total_series_found=len(discovered_list), + total_series_found=series_found, + inspection_duration_ms=round( + (persistence_started_at - inspection_started_at) * 1000 + ), + persistence_duration_ms=round((time.monotonic() - persistence_started_at) * 1000), sample_series=[item.raw_series_name for item in batch_to_flush[:5]], ) @@ -700,7 +1061,7 @@ async def flush_scan_batch(*, force: bool = False) -> None: max(job.scan_total_files, materialized_files, 1), ), message=( - f"Discovered {len(discovered_list)} series · " + f"Discovered {series_found} series · " f"{materialized_files}/{max(job.scan_total_files, 1)} files processed." ), current_series_name=batch_to_flush[-1].raw_series_name if batch_to_flush else None, @@ -712,35 +1073,39 @@ async def flush_scan_batch(*, force: bool = False) -> None: progress_callback=capture_scan_totals, file_progress_callback=capture_materialized_file, inventory_progress_callback=capture_inventory_progress, + cancellation_check=check_scan_cancellation if raise_if_cancelled is not None else None, extensions=custom_exts, + source_layout=SourceLayoutSpec.from_dict(dict(job.source_layout_snapshot or {})), ) - discovered_list: list[DiscoveredSeries] = [] - file_paths_mode = list(job.selected_file_paths or []) if file_paths_mode: - discovered_list = await scanner.scan_files(file_paths_mode) - job.scan_total_files = sum(series.file_count for series in discovered_list) - job.scan_total_dirs = len({series.source_folder for series in discovered_list}) - job.series_found = len(discovered_list) - batch.extend(discovered_list) + selected_discovered = await scanner.scan_files( + file_paths_mode, + root_path=job.source_path, + ) + series_found = len(selected_discovered) + job.scan_total_files = sum(series.file_count for series in selected_discovered) + job.scan_total_dirs = len({series.source_folder for series in selected_discovered}) + job.series_found = series_found + batch.extend(selected_discovered) await flush_scan_batch(force=True) await forward_live_scan_progress( phase="scanning", - message=f"Prepared {len(discovered_list)} series from the selected files.", + message=f"Prepared {series_found} series from the selected files.", progress=SCAN_PROGRESS_MATERIALIZE_END, ) - return discovered_list + return series_found async for series in scanner.scan(job.source_path): - discovered_list.append(series) + series_found += 1 batch.append(series) - job.series_found = len(discovered_list) + job.series_found = series_found await flush_scan_batch() await forward_live_scan_progress( phase="scanning", message=( - f"Discovered {len(discovered_list)} series · " + f"Discovered {series_found} series · " f"{materialized_files}/" f"{max(job.scan_total_files, 1)} files processed." ), @@ -755,4 +1120,4 @@ async def flush_scan_batch(*, force: bool = False) -> None: await flush_scan_batch(force=True) - return discovered_list + return series_found diff --git a/src/pullbox/services/import_scan_reconciliation.py b/src/pullbox/services/import_scan_reconciliation.py new file mode 100644 index 00000000..d90a5160 --- /dev/null +++ b/src/pullbox/services/import_scan_reconciliation.py @@ -0,0 +1,507 @@ +"""Fold stale Mylar references into inspected files before staging review rows.""" + +from __future__ import annotations + +from collections import defaultdict +from hashlib import sha256 +from pathlib import Path +from typing import TYPE_CHECKING + +from pullbox.core.name_matcher import NameMatcher +from pullbox.core.source_metadata import MetadataSignal, SourceMetadata, SourceMetadataExtractor +from pullbox.services.import_path_identity import ( + reconciliation_evidence, + same_trusted_issue, + unchanged_same_folder_pair, +) +from pullbox.services.import_source_metadata import cached_mylar_sidecar_data + +if TYPE_CHECKING: + from pullbox.core.collection_scanner import DiscoveredFile, DiscoveredSeries + + +def reconcile_discovered_mylar_paths(discovered_list: list[DiscoveredSeries]) -> None: + """Reuse archive evidence; never issue another provider or archive request.""" + extractor = SourceMetadataExtractor() + for series in discovered_list: + missing: dict[tuple[Path, int], list[DiscoveredFile]] = defaultdict(list) + for file in series.files: + block = file.metadata_diagnostics.get("file_safety") + if ( + isinstance(block, dict) + and block.get("code") == "source_missing" + and file.metadata_signals.get("comicvine_issue_id") == "mylar3" + and file.comicvine_issue_id + and not file.source_signature + ): + missing[(Path(file.file_path).parent, file.comicvine_issue_id)].append(file) + if not missing: + continue + candidates: dict[tuple[Path, int], list[tuple[DiscoveredFile, SourceMetadata]]] = ( + defaultdict(list) + ) + for file in series.files: + evidence = file.metadata_diagnostics.get("archive_member_evidence") + path = Path(file.file_path) + if not isinstance(evidence, dict) or evidence.get("member_index_scanned") is not True: + continue + sidecar = cached_mylar_sidecar_data(file.metadata_diagnostics) + metadata = extractor.from_path( + path, + sidecar_data=sidecar or {}, + archive_member_evidence=evidence, + include_archive_entry_issue_hint=False, + ) + if metadata.comicvine_issue_id: + candidates[(path.parent, metadata.comicvine_issue_id)].append((file, metadata)) + removed: set[str] = set() + for key, records in missing.items(): + matches = candidates.get(key, []) + if len(records) != 1 or len(matches) != 1: + continue + record = records[0] + actual, metadata = matches[0] + if actual.metadata_diagnostics.get("file_safety") or actual.metadata_diagnostics.get( + "identity_conflicts" + ): + continue + base = SourceMetadata( + original_title=record.file_name, + series_name=record.parsed_series, + issue_number=record.parsed_issue_number, + issue_type=record.issue_type, + comicvine_issue_id=record.comicvine_issue_id, + comicvine_series_id=record.comicvine_series_id, + signals={"comicvine_issue_id": MetadataSignal.MYLAR3}, + diagnostics=record.metadata_diagnostics, + ) + if not same_trusted_issue(base, metadata) or not unchanged_same_folder_pair( + Path(record.file_path), Path(actual.file_path), dict(actual.source_signature) + ): + continue + diagnostics = dict(actual.metadata_diagnostics) + diagnostics.pop("mylar3_folder_scope_conflict", None) + diagnostics.pop("mylar3_unrecorded_file", None) + recorded_issue = record.metadata_diagnostics.get("mylar3_issue") + if isinstance(recorded_issue, dict): + diagnostics["mylar3_issue"] = dict(recorded_issue) + diagnostics["mylar3_path_reconciliation"] = reconciliation_evidence( + record.file_path, + actual.file_path, + key[1], + recorded_series_name=base.series_name, + actual_series_name=metadata.series_name, + ) + actual.metadata_diagnostics = diagnostics + actual.parsed_series = record.parsed_series + actual.parsed_issue_number = record.parsed_issue_number + actual.issue_number_raw = record.issue_number_raw + actual.issue_type = record.issue_type + actual.comicvine_issue_id = metadata.comicvine_issue_id + actual.comicvine_series_id = record.comicvine_series_id + actual.has_comicinfo = True + signals = dict(actual.metadata_signals) + signals["comicvine_issue_id"] = MetadataSignal.COMICINFO.value + signals["comicvine_series_id"] = MetadataSignal.COMICINFO.value + signals["issue_number"] = MetadataSignal.COMICINFO.value + signals["series_name"] = MetadataSignal.MYLAR3.value + actual.metadata_signals = signals + removed.add(record.file_path) + if removed: + series.files = [file for file in series.files if file.file_path not in removed] + _refresh_series_shape(series) + _reconcile_cross_folder_mylar_paths(discovered_list, extractor) + + +def _content_hash(path: str) -> str | None: + digest = sha256() + try: + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + except OSError: + return None + return digest.hexdigest() + + +def _source_metadata( + file: DiscoveredFile, + extractor: SourceMetadataExtractor, +) -> SourceMetadata | None: + evidence = file.metadata_diagnostics.get("archive_member_evidence") + if not isinstance(evidence, dict) or evidence.get("member_index_scanned") is not True: + return None + sidecar = cached_mylar_sidecar_data(file.metadata_diagnostics) + return extractor.from_path( + Path(file.file_path), + sidecar_data=sidecar or {}, + archive_member_evidence=evidence, + include_archive_entry_issue_hint=False, + ) + + +def _recorded_metadata(file: DiscoveredFile) -> SourceMetadata: + return SourceMetadata( + original_title=file.file_name, + series_name=file.parsed_series, + issue_number=file.parsed_issue_number, + issue_type=file.issue_type, + comicvine_issue_id=file.comicvine_issue_id, + comicvine_series_id=file.comicvine_series_id, + signals={"comicvine_issue_id": MetadataSignal.MYLAR3}, + diagnostics=file.metadata_diagnostics, + ) + + +def _series_issue_filename_key( + *, + series_id: int | None, + issue_number: float | None, + file_name: str, +) -> tuple[int, float, str] | None: + if series_id is None or issue_number is None or not file_name: + return None + return int(series_id), float(issue_number), file_name.casefold() + + +def _same_trusted_series_issue_filename( + recorded: DiscoveredFile, + actual_file: DiscoveredFile, + actual: SourceMetadata, +) -> bool: + """Accept a stale Mylar issue ID only when every independent slot signal agrees.""" + if ( + not recorded.comicvine_series_id + or actual.comicvine_series_id != recorded.comicvine_series_id + or actual.comicvine_issue_id is None + or actual.signals.get("comicvine_series_id") is not MetadataSignal.COMICINFO + or actual.signals.get("comicvine_issue_id") is not MetadataSignal.COMICINFO + or actual.signals.get("series_name") is not MetadataSignal.COMICINFO + or actual.signals.get("issue_number") is not MetadataSignal.COMICINFO + or recorded.file_name.casefold() != actual_file.file_name.casefold() + or recorded.parsed_issue_number != actual.issue_number + or recorded.issue_type != actual.issue_type + or recorded.metadata_diagnostics.get("identity_conflicts") + or actual.diagnostics.get("identity_conflicts") + ): + return False + return bool( + recorded.parsed_series + and actual.series_name + and NameMatcher.normalize(recorded.parsed_series) + == NameMatcher.normalize(actual.series_name) + ) + + +def _choose_cross_folder_canonical( + recorded: DiscoveredFile, + candidates: list[tuple[DiscoveredSeries, DiscoveredFile, SourceMetadata]], +) -> ( + tuple[ + tuple[DiscoveredSeries, DiscoveredFile, SourceMetadata], + list[tuple[DiscoveredSeries, DiscoveredFile, SourceMetadata]], + ] + | None +): + exact_name = [item for item in candidates if item[1].file_name == recorded.file_name] + preferred = exact_name if exact_name else candidates + if len(preferred) == 1: + canonical = preferred[0] + else: + hashes = {_content_hash(item[1].file_path) for item in preferred} + if None in hashes or len(hashes) != 1: + return None + canonical = min( + preferred, key=lambda item: (item[1].file_name.casefold(), item[1].file_path) + ) + + canonical_hash = _content_hash(canonical[1].file_path) + identical: list[tuple[DiscoveredSeries, DiscoveredFile, SourceMetadata]] = [] + if canonical_hash is not None: + for item in candidates: + if item is canonical or item[1].file_size != canonical[1].file_size: + continue + if _content_hash(item[1].file_path) == canonical_hash: + identical.append(item) + return canonical, identical + + +def _apply_cross_folder_identity( + file: DiscoveredFile, + recorded: DiscoveredFile, + metadata: SourceMetadata, + *, + source_series: DiscoveredSeries, + role: str, + method: str = "verified_cross_folder_issue_identity", + canonical_path: str | None = None, +) -> None: + diagnostics = dict(file.metadata_diagnostics) + diagnostics.pop("mylar3_folder_scope_conflict", None) + diagnostics.pop("mylar3_unrecorded_file", None) + recorded_issue = recorded.metadata_diagnostics.get("mylar3_issue") + if isinstance(recorded_issue, dict): + diagnostics["mylar3_issue"] = dict(recorded_issue) + actual_issue_id = int(metadata.comicvine_issue_id or recorded.comicvine_issue_id or 0) + recorded_issue_id = int(recorded.comicvine_issue_id or 0) + evidence: dict[str, object] = { + "recorded_path": recorded.file_path, + "actual_path": file.file_path, + "comicvine_issue_id": actual_issue_id, + "comicvine_series_id": int(recorded.comicvine_series_id or 0), + "method": method, + "role": role, + "source_series": source_series.raw_series_name, + } + if recorded_issue_id != actual_issue_id: + evidence["recorded_comicvine_issue_id"] = recorded_issue_id + if ( + recorded.parsed_series + and metadata.series_name + and NameMatcher.normalize(recorded.parsed_series) + != NameMatcher.normalize(metadata.series_name) + ): + evidence["series_name_alias"] = { + "recorded": recorded.parsed_series, + "actual": metadata.series_name, + "accepted_by": "exact_comicvine_series_and_issue_identity", + } + if canonical_path is not None: + evidence["canonical_path"] = canonical_path + diagnostics["mylar3_cross_folder_reconciliation"] = evidence + file.metadata_diagnostics = diagnostics + file.parsed_series = recorded.parsed_series + file.parsed_issue_number = recorded.parsed_issue_number + file.issue_number_raw = recorded.issue_number_raw + file.issue_type = recorded.issue_type + file.comicvine_issue_id = actual_issue_id or None + file.comicvine_series_id = recorded.comicvine_series_id + file.has_comicinfo = True + signals = dict(file.metadata_signals) + signals["comicvine_issue_id"] = MetadataSignal.COMICINFO.value + signals["comicvine_series_id"] = signals.get("comicvine_series_id", MetadataSignal.MYLAR3.value) + signals["issue_number"] = signals.get("issue_number", MetadataSignal.MYLAR3.value) + signals["series_name"] = signals.get("series_name", MetadataSignal.MYLAR3.value) + file.metadata_signals = signals + + +def _has_diagnostic_value( + file: DiscoveredFile, + diagnostic_name: str, + key: str, + value: str, +) -> bool: + block = file.metadata_diagnostics.get(diagnostic_name) + return isinstance(block, dict) and block.get(key) == value + + +def _refresh_series_shape(series: DiscoveredSeries) -> None: + conflicts = [ + file for file in series.files if "mylar3_folder_scope_conflict" in file.metadata_diagnostics + ] + diagnostics = dict(series.diagnostics) + if conflicts: + diagnostics["mylar3_folder_scope"] = { + "review_required": True, + "unrecorded_file_count": sum( + "mylar3_issue" not in file.metadata_diagnostics for file in series.files + ), + "conflicting_file_count": len(conflicts), + "examples": [file.file_name for file in conflicts[:5]], + } + else: + diagnostics.pop("mylar3_folder_scope", None) + + recovered_files = [ + file + for file in series.files + if _has_diagnostic_value( + file, + "mylar3_cross_folder_reconciliation", + "role", + "canonical", + ) + ] + missing_files = [ + file + for file in series.files + if _has_diagnostic_value(file, "file_safety", "code", "source_missing") + ] + if ( + recovered_files + and diagnostics.get("kind") == "mylar3_path_incompatible" + and diagnostics.get("reason") == "source_missing" + ): + diagnostics.pop("kind", None) + diagnostics.pop("reason", None) + diagnostics.pop("rejection_reason", None) + path_details = diagnostics.get("mylar3_path") + if isinstance(path_details, dict): + path_details = dict(path_details) + path_details["status"] = "partially_reconciled" if missing_files else "reconciled" + diagnostics["mylar3_path"] = path_details + diagnostics["mylar3_path_recovery"] = { + "status": "partial" if missing_files else "complete", + "recovered_file_count": len(recovered_files), + "remaining_missing_file_count": len(missing_files), + } + series.diagnostics = diagnostics + series.file_count = len(series.files) + series.sample_paths = [file.file_path for file in series.files[:5]] + series.has_files = bool(series.files) + + +def _reconcile_cross_folder_mylar_paths( + discovered_list: list[DiscoveredSeries], + extractor: SourceMetadataExtractor, +) -> None: + """Link proven misplaced files to one missing Mylar issue without moving sources.""" + missing_by_issue: dict[int, list[tuple[DiscoveredSeries, DiscoveredFile]]] = defaultdict(list) + missing_by_slot: dict[ + tuple[int, float, str], + list[tuple[DiscoveredSeries, DiscoveredFile]], + ] = defaultdict(list) + candidates_by_issue: dict[ + int, + list[tuple[DiscoveredSeries, DiscoveredFile, SourceMetadata]], + ] = defaultdict(list) + candidates_by_slot: dict[ + tuple[int, float, str], + list[tuple[DiscoveredSeries, DiscoveredFile, SourceMetadata]], + ] = defaultdict(list) + for series in discovered_list: + for file in series.files: + block = file.metadata_diagnostics.get("file_safety") + if ( + isinstance(block, dict) + and block.get("code") == "source_missing" + and file.comicvine_issue_id + and file.comicvine_series_id + and file.metadata_signals.get("comicvine_issue_id") == MetadataSignal.MYLAR3.value + and not file.source_signature + ): + missing_by_issue[int(file.comicvine_issue_id)].append((series, file)) + slot = _series_issue_filename_key( + series_id=file.comicvine_series_id, + issue_number=file.parsed_issue_number, + file_name=file.file_name, + ) + if slot is not None: + missing_by_slot[slot].append((series, file)) + continue + if ( + not file.source_signature + or "mylar3_folder_scope_conflict" not in file.metadata_diagnostics + or file.metadata_diagnostics.get("file_safety") + or file.metadata_diagnostics.get("identity_conflicts") + ): + continue + metadata = _source_metadata(file, extractor) + if ( + metadata is None + or metadata.comicvine_issue_id is None + or metadata.signals.get("comicvine_issue_id") is not MetadataSignal.COMICINFO + ): + continue + candidates_by_issue[int(metadata.comicvine_issue_id)].append((series, file, metadata)) + slot = _series_issue_filename_key( + series_id=metadata.comicvine_series_id, + issue_number=metadata.issue_number, + file_name=file.file_name, + ) + if slot is not None: + candidates_by_slot[slot].append((series, file, metadata)) + + changed_series: dict[int, DiscoveredSeries] = {} + used_paths: set[str] = set() + resolved_records: set[int] = set() + + def apply_choice( + target_series: DiscoveredSeries, + recorded: DiscoveredFile, + canonical: tuple[DiscoveredSeries, DiscoveredFile, SourceMetadata], + identical: list[tuple[DiscoveredSeries, DiscoveredFile, SourceMetadata]], + *, + method: str, + ) -> None: + source_series, canonical_file, metadata = canonical + moved = [canonical, *identical] + _apply_cross_folder_identity( + canonical_file, + recorded, + metadata, + source_series=source_series, + role="canonical", + method=method, + ) + for duplicate_series, duplicate, duplicate_metadata in identical: + _apply_cross_folder_identity( + duplicate, + recorded, + duplicate_metadata, + source_series=duplicate_series, + role="identical_duplicate", + method=method, + canonical_path=canonical_file.file_path, + ) + target_series.files = [file for file in target_series.files if file is not recorded] + for owner, file, _metadata in moved: + owner.files = [owned_file for owned_file in owner.files if owned_file is not file] + changed_series[id(owner)] = owner + used_paths.add(file.file_path) + target_series.files.extend(item[1] for item in moved) + changed_series[id(target_series)] = target_series + resolved_records.add(id(recorded)) + + for issue_id, records in missing_by_issue.items(): + candidates = [ + item + for item in candidates_by_issue.get(issue_id, []) + if item[1].file_path not in used_paths + ] + if len(records) != 1 or not candidates: + continue + target_series, recorded = records[0] + recorded_metadata = _recorded_metadata(recorded) + verified = [ + item + for item in candidates + if same_trusted_issue(recorded_metadata, item[2]) + and ( + item[2].comicvine_series_id is None + or item[2].comicvine_series_id == recorded.comicvine_series_id + ) + ] + choice = _choose_cross_folder_canonical(recorded, verified) if verified else None + if choice is None: + continue + canonical, identical = choice + apply_choice( + target_series, + recorded, + canonical, + identical, + method="verified_cross_folder_issue_identity", + ) + + for slot, records in missing_by_slot.items(): + unresolved = [item for item in records if id(item[1]) not in resolved_records] + candidates = [ + item for item in candidates_by_slot.get(slot, []) if item[1].file_path not in used_paths + ] + if len(unresolved) != 1 or len(candidates) != 1: + continue + target_series, recorded = unresolved[0] + candidate = candidates[0] + if not _same_trusted_series_issue_filename(recorded, candidate[1], candidate[2]): + continue + apply_choice( + target_series, + recorded, + candidate, + [], + method="verified_cross_folder_series_issue_filename", + ) + + for series in changed_series.values(): + _refresh_series_shape(series) diff --git a/src/pullbox/services/import_series_deduplication.py b/src/pullbox/services/import_series_deduplication.py index a80101ac..0f0daf92 100644 --- a/src/pullbox/services/import_series_deduplication.py +++ b/src/pullbox/services/import_series_deduplication.py @@ -10,6 +10,10 @@ from sqlalchemy import or_ as sa_or from sqlalchemy import select as sa_select +from pullbox.core.issue_numbers import ( + issue_number_text_matches_numeric, + normalize_issue_number_text, +) from pullbox.core.name_matcher import NameMatcher from pullbox.core.type_semantics import series_types_compatible from pullbox.models.import_job import ( @@ -29,18 +33,22 @@ ) from pullbox.services.import_progress_runtime import ( ScanReviewFileMatchProfile, + ScanReviewProgressPlan, ScanReviewSeriesMatchProfile, estimate_remaining_work_seconds, - scan_review_completed_weight, + scan_review_analysis_weight, + scan_review_file_match_weight, scan_review_progress_pct, - scan_review_progress_plan, + scan_review_series_match_weight, ) if TYPE_CHECKING: from collections.abc import Awaitable, Callable - from datetime import datetime + from datetime import date, datetime + from sqlalchemy.engine import RowMapping from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql import Select ProgressCallback = Callable[[ImportProgressEvent], Awaitable[None]] RaiseIfCancelledFunc = Callable[[AsyncSession, int], Awaitable[None]] @@ -65,6 +73,21 @@ class _ExistingSeriesCandidate(NamedTuple): comicvine_url: str | None +class _ScanReviewProfileRow(NamedTuple): + id: int + file_count: int + direct_match: bool + has_files: bool + issue_count: int | None + + +class _ImportedIssueTargetRow(NamedTuple): + id: int + issue_number: float + issue_number_text: str | None + parsed_year: int | None + + async def deduplicate_import_series( session: AsyncSession, job: ImportJob, @@ -76,98 +99,133 @@ async def deduplicate_import_series( estimate_remaining_seconds: EstimateRemainingFunc, job_stats: JobStatsFunc, progress_callback: ProgressCallback | None = None, + item_page_size: int = 100, + candidate_page_size: int = 250, + profile_page_size: int = 500, + issue_target_page_size: int = 250, ) -> None: """Tag imported series rows as duplicate when they already exist in the library.""" started_at = time.monotonic() - persist_batch_size = 100 - last_checkpoint_at = time.monotonic() - items_result = await session.execute( - sa_select(ImportedSeries).where( - ImportedSeries.import_job_id == job.id, - ImportedSeries.status == ImportSeriesStatus.PENDING, - ) - ) - items = items_result.scalars().all() - - requested_cv_ids = {item.cv_id for item in items if item.cv_id is not None} - requested_titles = { - item.raw_series_name.strip().lower() for item in items if item.raw_series_name - } - requested_years = {item.raw_year for item in items if item.raw_year is not None} - - existing_rows = await _load_existing_series_candidates( + item_page_size = max(item_page_size, 1) + candidate_page_size = max(candidate_page_size, 1) + profile_page_size = max(profile_page_size, 1) + issue_target_page_size = max(issue_target_page_size, 1) + total_items = await _count_import_series_by_status( session, - requested_cv_ids=requested_cv_ids, - requested_titles=requested_titles, - requested_years=requested_years, + job_id=job.id, + status=ImportSeriesStatus.PENDING, ) - - cv_id_map: dict[int, _ExistingSeriesCandidate] = {} - existing_list: list[_ExistingSeriesCandidate] = [] - for row in existing_rows: - if row.comicvine_id is not None: - cv_id_map[row.comicvine_id] = row - existing_list.append(row) - - duplicate_count = 0 - total_items = len(items) - progress_plan = scan_review_progress_plan( - analysis_series_count=total_items, - series_match_profiles=[ - ScanReviewSeriesMatchProfile( - file_count=int(item.files_total or item.file_count or 0), - direct_match=bool(item.cv_id), - ) - for item in items - ], - file_match_profiles=[ - ScanReviewFileMatchProfile( - file_count=int(item.files_total or item.file_count or 0), - issue_count=item.cv_issue_count, - ) - for item in items - if item.has_files - ], + duplicate_count = await _count_import_series_by_status( + session, + job_id=job.id, + status=ImportSeriesStatus.DUPLICATE, ) - for idx, item in enumerate(items): - await raise_if_cancelled(session, job.id) - duplicate_found = await _mark_cv_id_duplicate( + progress_plan = ( + await _build_scan_review_progress_plan( session, - item, job_id=job.id, - cv_id_map=cv_id_map, - log_event=log_event, - ) or await _mark_name_year_duplicate( + page_size=profile_page_size, + raise_if_cancelled=raise_if_cancelled, + ) + if progress_callback + else None + ) + + processed_items = 0 + after_id = 0 + while True: + await raise_if_cancelled(session, job.id) + items = await _load_pending_import_series_page( session, - item, job_id=job.id, - existing_list=existing_list, - log_event=log_event, - ) - if duplicate_found: - duplicate_count += 1 - - should_checkpoint = ( - idx == 0 - or (idx + 1) % persist_batch_size == 0 - or idx == total_items - 1 - or (time.monotonic() - last_checkpoint_at) >= 0.5 + after_id=after_id, + page_size=item_page_size, ) - if should_checkpoint: - await session.commit() - last_checkpoint_at = time.monotonic() + if not items: + break - if progress_callback and should_checkpoint: - completed_weight = scan_review_completed_weight( - progress_plan, - phase="analyzing", - completed_items=idx + 1, + item_by_id = {item.id: item for item in items} + cv_id_map = await _load_existing_cv_candidates( + session, + requested_cv_ids={item.cv_id for item in items if item.cv_id is not None}, + ) + matched_item_ids: set[int] = set() + for item in items: + if await _mark_cv_id_duplicate( + session, + item, + job_id=job.id, + cv_id_map=cv_id_map, + log_event=log_event, + ): + duplicate_count += 1 + matched_item_ids.add(item.id) + + candidate_items = [item for item in items if item.id not in matched_item_ids] + requested_titles = { + item.raw_series_name.strip().lower() for item in candidate_items if item.raw_series_name + } + requested_years = {item.raw_year for item in candidate_items if item.raw_year is not None} + item_ids_by_title: dict[str, list[int]] = {} + item_ids_by_year: dict[int, list[int]] = {} + for item in candidate_items: + item_ids_by_title.setdefault(item.raw_series_name.strip().lower(), []).append(item.id) + if item.raw_year is not None: + item_ids_by_year.setdefault(item.raw_year, []).append(item.id) + + candidate_after_id = 0 + while candidate_items: + await raise_if_cancelled(session, job.id) + candidates = await _load_existing_name_candidate_page( + session, + requested_titles=requested_titles, + requested_years=requested_years, + after_id=candidate_after_id, + page_size=candidate_page_size, + ) + if not candidates: + break + + for candidate_index, candidate in enumerate(candidates): + if candidate_index and candidate_index % 25 == 0: + await raise_if_cancelled(session, job.id) + relevant_item_ids = set(item_ids_by_title.get(candidate.title.lower(), ())) + if candidate.year_start is not None: + for year in range(candidate.year_start - 1, candidate.year_start + 2): + relevant_item_ids.update(item_ids_by_year.get(year, ())) + for imported_series_id in sorted(relevant_item_ids): + if imported_series_id in matched_item_ids: + continue + item = item_by_id[imported_series_id] + if await _mark_name_year_duplicate_candidate( + session, + item, + job_id=job.id, + existing=candidate, + log_event=log_event, + raise_if_cancelled=raise_if_cancelled, + issue_target_page_size=issue_target_page_size, + ): + duplicate_count += 1 + matched_item_ids.add(imported_series_id) + + candidate_after_id = candidates[-1].id + + job.series_duplicate = duplicate_count + await session.commit() + processed_items += len(items) + after_id = items[-1].id + last_item = items[-1] + + if progress_callback and progress_plan is not None: + completed_weight = progress_plan.analysis_weight * min( + processed_items / max(total_items, 1), + 1.0, ) progress = scan_review_progress_pct( progress_plan, completed_weight=completed_weight, ) - job.series_duplicate = duplicate_count await emit_progress( session, job, @@ -176,9 +234,9 @@ async def deduplicate_import_series( status=ImportJobStatus.ANALYZING, phase="analyzing", progress=progress, - message=f"Analyzing {idx + 1}/{total_items}...", - current_series=item.raw_series_name, - current_series_status=item.status, + message=f"Analyzing {processed_items}/{total_items}...", + current_series=last_item.raw_series_name, + current_series_status=last_item.status, estimated_seconds_remaining=estimate_remaining_work_seconds( job.scan_completed_at or job.scan_started_at, completed_units=completed_weight, @@ -189,6 +247,8 @@ async def deduplicate_import_series( progress_callback, ) + del item_by_id, items + job.series_duplicate = duplicate_count await session.flush() @@ -203,14 +263,184 @@ async def deduplicate_import_series( ) -async def _load_existing_series_candidates( +async def _count_import_series_by_status( + session: AsyncSession, + *, + job_id: int, + status: ImportSeriesStatus, +) -> int: + count = await session.scalar( + sa_select(sa_func.count(ImportedSeries.id)).where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status == status, + ) + ) + return int(count or 0) + + +async def _load_pending_import_series_page( + session: AsyncSession, + *, + job_id: int, + after_id: int, + page_size: int, +) -> list[ImportedSeries]: + result = await session.execute( + sa_select(ImportedSeries) + .where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status == ImportSeriesStatus.PENDING, + ImportedSeries.id > after_id, + ) + .order_by(ImportedSeries.id) + .limit(page_size) + ) + return list(result.scalars()) + + +async def _load_scan_review_profile_page( + session: AsyncSession, + *, + job_id: int, + after_id: int, + page_size: int, +) -> list[_ScanReviewProfileRow]: + result = await session.execute( + sa_select( + ImportedSeries.id, + ImportedSeries.files_total, + ImportedSeries.file_count, + ImportedSeries.cv_id, + ImportedSeries.has_files, + ImportedSeries.cv_issue_count, + ) + .where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status == ImportSeriesStatus.PENDING, + ImportedSeries.id > after_id, + ) + .order_by(ImportedSeries.id) + .limit(page_size) + ) + return [ + _ScanReviewProfileRow( + id=row.id, + file_count=int(row.files_total or row.file_count or 0), + direct_match=bool(row.cv_id), + has_files=bool(row.has_files), + issue_count=row.cv_issue_count, + ) + for row in result + ] + + +async def _build_scan_review_progress_plan( + session: AsyncSession, + *, + job_id: int, + page_size: int, + raise_if_cancelled: RaiseIfCancelledFunc, +) -> ScanReviewProgressPlan: + analysis_weight = 0.0 + series_match_weight = 0.0 + file_match_weight = 0.0 + after_id = 0 + while True: + await raise_if_cancelled(session, job_id) + profiles = await _load_scan_review_profile_page( + session, + job_id=job_id, + after_id=after_id, + page_size=page_size, + ) + if not profiles: + break + analysis_weight += scan_review_analysis_weight(len(profiles)) + for profile in profiles: + series_match_weight += scan_review_series_match_weight( + ScanReviewSeriesMatchProfile( + file_count=profile.file_count, + direct_match=profile.direct_match, + ) + ) + if profile.has_files: + file_match_weight += scan_review_file_match_weight( + ScanReviewFileMatchProfile( + file_count=profile.file_count, + issue_count=profile.issue_count, + ) + ) + after_id = profiles[-1].id + + return ScanReviewProgressPlan( + analysis_weights=(analysis_weight,) if analysis_weight else (), + series_match_weights=(series_match_weight,) if series_match_weight else (), + file_match_weights=(file_match_weight,) if file_match_weight else (), + ) + + +async def _load_existing_cv_candidates( session: AsyncSession, *, requested_cv_ids: set[int], +) -> dict[int, _ExistingSeriesCandidate]: + if not requested_cv_ids: + return {} + result = await session.execute( + _existing_series_candidate_select() + .where(Series.comicvine_id.in_(requested_cv_ids)) + .order_by(Series.id) + ) + candidates: dict[int, _ExistingSeriesCandidate] = {} + for row in result.mappings(): + candidate = _existing_series_candidate_from_row(row) + if candidate.comicvine_id is not None: + candidates[candidate.comicvine_id] = candidate + return candidates + + +async def _load_existing_name_candidate_page( + session: AsyncSession, + *, requested_titles: set[str], requested_years: set[int], + after_id: int, + page_size: int, ) -> list[_ExistingSeriesCandidate]: - existing_query = sa_select( + existing_filters: list[ColumnElement[bool]] = [] + if requested_titles: + existing_filters.append(sa_func.lower(Series.title).in_(requested_titles)) + if requested_years: + widened_years = {year + delta for year in requested_years for delta in (-1, 0, 1)} + existing_filters.append(Series.year_start.in_(sorted(widened_years))) + if not existing_filters: + return [] + + result = await session.execute( + _existing_series_candidate_select() + .where( + Series.id > after_id, + sa_or(*existing_filters), + ) + .order_by(Series.id) + .limit(page_size) + ) + return [_existing_series_candidate_from_row(row) for row in result.mappings()] + + +def _existing_series_candidate_select() -> Select[ + tuple[ + int, + int | None, + str, + int | None, + SeriesType, + str, + int, + str | None, + ] +]: + return sa_select( Series.id, Series.comicvine_id, Series.title, @@ -220,31 +450,19 @@ async def _load_existing_series_candidates( Series.issue_count, Series.comicvine_url, ).outerjoin(Publisher, Publisher.id == Series.publisher_id) - existing_filters: list[ColumnElement[bool]] = [] - if requested_cv_ids: - existing_filters.append(Series.comicvine_id.in_(requested_cv_ids)) - if requested_titles: - existing_filters.append(sa_func.lower(Series.title).in_(requested_titles)) - if requested_years: - widened_years = {year + delta for year in requested_years for delta in (-1, 0, 1)} - existing_filters.append(Series.year_start.in_(sorted(widened_years))) - if existing_filters: - existing_query = existing_query.where(sa_or(*existing_filters)) - existing_result = await session.execute(existing_query) - return [ - _ExistingSeriesCandidate( - id=row.id, - comicvine_id=row.comicvine_id, - title=row.title, - year_start=row.year_start, - series_type=row.series_type, - publisher_name=row.publisher_name, - issue_count=row.issue_count, - comicvine_url=row.comicvine_url, - ) - for row in existing_result.all() - ] + +def _existing_series_candidate_from_row(row: RowMapping) -> _ExistingSeriesCandidate: + return _ExistingSeriesCandidate( + id=row.id, + comicvine_id=row.comicvine_id, + title=row.title, + year_start=row.year_start, + series_type=row.series_type, + publisher_name=row.publisher_name, + issue_count=row.issue_count, + comicvine_url=row.comicvine_url, + ) async def _mark_cv_id_duplicate( @@ -286,67 +504,67 @@ async def _mark_cv_id_duplicate( return True -async def _mark_name_year_duplicate( +async def _mark_name_year_duplicate_candidate( session: AsyncSession, item: ImportedSeries, *, job_id: int, - existing_list: list[_ExistingSeriesCandidate], + existing: _ExistingSeriesCandidate, log_event: LogEventFunc, + raise_if_cancelled: RaiseIfCancelledFunc, + issue_target_page_size: int, ) -> bool: candidate_series_type = series_type_from_import_diagnostics(item.diagnostics) - for existing in existing_list: - matched_by_name_year = is_same_series( - item.raw_series_name, - item.raw_year, - candidate_series_type, - existing.title, - existing.year_start, - existing.series_type, - ) - matched_by_issue_target = ( - False - if matched_by_name_year - else await _exact_title_series_supports_imported_issue_targets( - session, - item, - existing, - candidate_series_type=candidate_series_type, - ) - ) - if not matched_by_name_year and not matched_by_issue_target: - continue - - item.status = ImportSeriesStatus.DUPLICATE - item.series_id = existing.id - name_match = NameMatcher().match(item.raw_series_name, existing.title) - _hydrate_duplicate_cv_fields(item, existing, match_score=name_match.similarity) - item.diagnostics = { - "kind": "duplicate_series", - "duplicate_reason": ( - "name_year" if matched_by_name_year else "exact_title_issue_target" - ), - "existing_series_id": existing.id, - "existing_series_title": existing.title, - "existing_series_year": existing.year_start, - "duplicate_match_score": name_match.similarity, - } - - await log_event( + matched_by_name_year = is_same_series( + item.raw_series_name, + item.raw_year, + candidate_series_type, + existing.title, + existing.year_start, + existing.series_type, + ) + matched_by_issue_target = ( + False + if matched_by_name_year + else await _exact_title_series_supports_imported_issue_targets( session, - job_id, - "DEBUG", - "import_dedup_name_year_match", - message=f"Duplicate: '{item.raw_series_name}' matches '{existing.title}'", - raw_series_name=item.raw_series_name, - raw_year=item.raw_year, - existing_title=existing.title, - existing_year=existing.year_start, - existing_series_id=existing.id, + item, + existing, + candidate_series_type=candidate_series_type, + job_id=job_id, + page_size=issue_target_page_size, + raise_if_cancelled=raise_if_cancelled, ) - return True + ) + if not matched_by_name_year and not matched_by_issue_target: + return False - return False + item.status = ImportSeriesStatus.DUPLICATE + item.series_id = existing.id + name_match = NameMatcher().match(item.raw_series_name, existing.title) + _hydrate_duplicate_cv_fields(item, existing, match_score=name_match.similarity) + item.diagnostics = { + "kind": "duplicate_series", + "duplicate_reason": ("name_year" if matched_by_name_year else "exact_title_issue_target"), + "existing_series_id": existing.id, + "existing_series_title": existing.title, + "existing_series_year": existing.year_start, + "duplicate_match_score": name_match.similarity, + } + + await log_event( + session, + job_id, + "DEBUG", + "import_dedup_name_year_match", + message=f"Duplicate: '{item.raw_series_name}' matches '{existing.title}'", + raw_series_name=item.raw_series_name, + raw_year=item.raw_year, + existing_title=existing.title, + existing_year=existing.year_start, + existing_series_id=existing.id, + ) + return True async def _exact_title_series_supports_imported_issue_targets( @@ -355,6 +573,9 @@ async def _exact_title_series_supports_imported_issue_targets( existing: _ExistingSeriesCandidate, *, candidate_series_type: SeriesType | None, + job_id: int, + page_size: int, + raise_if_cancelled: RaiseIfCancelledFunc, ) -> bool: """Allow ongoing exact-title duplicates when file years are issue release years.""" if not item.raw_series_name or not existing.title: @@ -368,43 +589,142 @@ async def _exact_title_series_supports_imported_issue_targets( ): return False - files_result = await session.execute( - sa_select(ImportedFile.parsed_issue_number, ImportedFile.parsed_year).where( - ImportedFile.import_series_id == item.id, + supported_any = False + after_id = 0 + while True: + await raise_if_cancelled(session, job_id) + requested_targets = await _load_imported_issue_target_page( + session, + import_series_id=item.id, + after_id=after_id, + page_size=page_size, + ) + if not requested_targets: + break + + normalized_text_by_file_id = { + target.id: _normalized_imported_issue_text(target) for target in requested_targets + } + exact_numbers = { + exact_text + for exact_text in normalized_text_by_file_id.values() + if exact_text is not None + } + legacy_numbers = { + target.issue_number + for target in requested_targets + if normalized_text_by_file_id[target.id] is None + } + exact_issue_dates, unambiguous_legacy_issue_dates = await _load_issue_target_dates( + session, + series_id=existing.id, + exact_numbers=exact_numbers, + legacy_numbers=legacy_numbers, + ) + for target in requested_targets: + exact_text = normalized_text_by_file_id[target.id] + release_date = ( + exact_issue_dates.get(exact_text) + if exact_text is not None + else unambiguous_legacy_issue_dates.get(target.issue_number) + ) + if release_date is None: + continue + if target.parsed_year is not None and abs(target.parsed_year - release_date.year) > 1: + return False + supported_any = True + after_id = requested_targets[-1].id + + return supported_any + + +async def _load_imported_issue_target_page( + session: AsyncSession, + *, + import_series_id: int, + after_id: int, + page_size: int, +) -> list[_ImportedIssueTargetRow]: + result = await session.execute( + sa_select( + ImportedFile.id, + ImportedFile.parsed_issue_number, + ImportedFile.issue_number_raw, + ImportedFile.parsed_year, + ) + .where( + ImportedFile.import_series_id == import_series_id, ImportedFile.parsed_issue_number.is_not(None), + ImportedFile.id > after_id, ) + .order_by(ImportedFile.id) + .limit(page_size) ) - requested_targets = [ - (float(row.parsed_issue_number), row.parsed_year) - for row in files_result.all() + return [ + _ImportedIssueTargetRow( + id=int(row.id), + issue_number=float(row.parsed_issue_number), + issue_number_text=row.issue_number_raw, + parsed_year=row.parsed_year, + ) + for row in result if row.parsed_issue_number is not None ] - if not requested_targets: - return False - requested_issue_numbers = {issue_number for issue_number, _year in requested_targets} - issues_result = await session.execute( - sa_select(Issue.issue_number, Issue.release_date).where( - Issue.series_id == existing.id, - Issue.issue_number.in_(requested_issue_numbers), - ) + +def _normalized_imported_issue_text(target: _ImportedIssueTargetRow) -> str | None: + if not target.issue_number_text: + return None + try: + normalized = normalize_issue_number_text(target.issue_number_text) + except ValueError: + return None + return ( + normalized if issue_number_text_matches_numeric(target.issue_number, normalized) else None ) - existing_issue_by_number = { - float(row.issue_number): row.release_date for row in issues_result.all() - } - if not existing_issue_by_number: - return False - supported_any = False - for issue_number, parsed_year in requested_targets: - release_date = existing_issue_by_number.get(issue_number) - if release_date is None: - continue - if parsed_year is not None and abs(parsed_year - release_date.year) > 1: - return False - supported_any = True - return supported_any +async def _load_issue_target_dates( + session: AsyncSession, + *, + series_id: int, + exact_numbers: set[str], + legacy_numbers: set[float], +) -> tuple[dict[str, date | None], dict[float, date | None]]: + exact_dates: dict[str, date | None] = {} + if exact_numbers: + exact_result = await session.execute( + sa_select(Issue.issue_number_text, Issue.release_date).where( + Issue.series_id == series_id, + Issue.issue_number_text.in_(exact_numbers), + ) + ) + exact_dates = { + str(row.issue_number_text): row.release_date + for row in exact_result + if row.issue_number_text is not None + } + + legacy_dates: dict[float, date | None] = {} + if legacy_numbers: + legacy_result = await session.execute( + sa_select( + Issue.issue_number, + sa_func.count(Issue.id).label("candidate_count"), + sa_func.max(Issue.release_date).label("release_date"), + ) + .where( + Issue.series_id == series_id, + Issue.issue_number.in_(legacy_numbers), + ) + .group_by(Issue.issue_number) + ) + legacy_dates = { + float(row.issue_number): row.release_date + for row in legacy_result + if int(row.candidate_count) == 1 + } + return exact_dates, legacy_dates def _hydrate_duplicate_cv_fields( diff --git a/src/pullbox/services/import_series_file_processor.py b/src/pullbox/services/import_series_file_processor.py index 087f3101..124b7515 100644 --- a/src/pullbox/services/import_series_file_processor.py +++ b/src/pullbox/services/import_series_file_processor.py @@ -18,7 +18,7 @@ from pullbox.core.library_policy import LibraryIngestPolicy from pullbox.models.import_job import ImportedSeries, ImportJob, ImportJobAction from pullbox.models.issue import Issue - from pullbox.models.library import LibraryFile, MatchConfidence + from pullbox.models.library import LibraryFile, LibraryFileStorageMode, MatchConfidence from pullbox.services.import_file_execution_protocols import ReportFileProgressFunc from pullbox.services.import_file_preparation import PreparedImportFile @@ -94,6 +94,8 @@ async def register_import_file( confidence: MatchConfidence, *, move_to_library: bool, + storage_mode: LibraryFileStorageMode, + expected_source_signature: dict[str, object] | None, library_root_id: int | None, transfer_method: str | None, normalize_to_cbz: bool | None = None, @@ -106,6 +108,13 @@ async def register_import_file( | None = None, comicinfo_progress_callback: Callable[[str, int, int, str], Awaitable[None] | None] | None = None, + recovery_imported_file_id: int | None = None, + recovery_original_source_path: Path | None = None, + replace_existing_library_file: bool = False, + replacement_trash_dir: Path | None = None, + preserve_replaced_artifact: bool = False, + source_scan_root: Path | None = None, + strict_import_target: bool = False, ) -> LibraryFile | LibraryFileRegistrationOutcome: return await register_import_library_file( session, @@ -114,6 +123,8 @@ async def register_import_file( issue, confidence, move_to_library=move_to_library, + storage_mode=storage_mode, + expected_source_signature=expected_source_signature, library_root_id=library_root_id, transfer_method=transfer_method, normalize_to_cbz=normalize_to_cbz, @@ -124,6 +135,13 @@ async def register_import_file( permission_policy=permission_policy, transfer_progress_callback=transfer_progress_callback, comicinfo_progress_callback=comicinfo_progress_callback, + recovery_imported_file_id=recovery_imported_file_id, + recovery_original_source_path=recovery_original_source_path, + replace_existing_library_file=replace_existing_library_file, + replacement_trash_dir=replacement_trash_dir, + preserve_replaced_artifact=preserve_replaced_artifact, + source_scan_root=source_scan_root, + strict_import_target=strict_import_target, ) return await core_processor( @@ -147,6 +165,7 @@ async def register_import_file( move_to_trash=move_to_trash, report_file_progress=report_file_progress, defer_comicinfo_enrichment=defer_comicinfo_enrichment, + revalidate_managed_sources=True, file_worker_count=file_worker_count, session_factory=session_factory, ) diff --git a/src/pullbox/services/import_series_matching.py b/src/pullbox/services/import_series_matching.py index fc1b9371..ccd03875 100644 --- a/src/pullbox/services/import_series_matching.py +++ b/src/pullbox/services/import_series_matching.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any import structlog +from sqlalchemy import func as sa_func from sqlalchemy import select as sa_select from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import async_sessionmaker @@ -39,11 +40,13 @@ ) from pullbox.services.import_progress_runtime import ( ScanReviewFileMatchProfile, + ScanReviewProgressPlan, ScanReviewSeriesMatchProfile, current_item_payload, - scan_review_completed_weight, + scan_review_analysis_weight, + scan_review_file_match_weight, scan_review_progress_pct, - scan_review_progress_plan, + scan_review_series_match_weight, ) from pullbox.services.import_series_match_state import clear_auto_cv_match_fields from pullbox.services.import_workflow_state import emit_live_progress @@ -70,6 +73,41 @@ def _filesystem_source_identity_evaluation( """Classify a filesystem series from trusted local identity only.""" normalized_query = NameMatcher.normalize(source_metadata.series_name or item.raw_series_name) identity_conflicts = source_metadata.diagnostics.get("identity_conflicts") + source_signal = source_metadata.signals.get("comicvine_series_id") + cv_id = source_metadata.comicvine_series_id or item.cv_id + match_method = item.cv_match_method + if match_method not in _DIRECT_CV_ID_MATCH_METHODS: + match_method = ( + { + MetadataSignal.MYLAR3: "mylar3_cv_id", + MetadataSignal.COMICINFO: "comicinfo_cv_id", + MetadataSignal.SIDECAR: "comicinfo_cv_id", + MetadataSignal.PULLBOX_FOLDER: "folder_cv_id", + }.get(source_signal) + if source_signal is not None + else None + ) + + # A Mylar database row is authoritative for series ownership during a Mylar + # migration. Conflicting file/sidecar evidence is retained for file-level + # review rather than invalidating the already-known series identity. + if cv_id is not None and match_method == "mylar3_cv_id": + evaluation = known_cv_id_evaluation_from_source( + int(cv_id), + match_method=match_method, + raw_name=item.raw_series_name, + raw_year=item.raw_year, + normalized_query=normalized_query, + source_metadata=source_metadata, + match_threshold=match_threshold, + ) + if isinstance(identity_conflicts, list) and identity_conflicts: + evaluation.diagnostics["identity_conflicts"] = [ + dict(conflict) for conflict in identity_conflicts if isinstance(conflict, dict) + ] + evaluation.diagnostics["file_identity_review_required"] = True + return evaluation + if isinstance(identity_conflicts, list) and identity_conflicts: return ComicVineMatchEvaluation( match=None, @@ -88,20 +126,6 @@ def _filesystem_source_identity_evaluation( }, ) - cv_id = source_metadata.comicvine_series_id or item.cv_id - match_method = item.cv_match_method - if match_method not in _DIRECT_CV_ID_MATCH_METHODS: - signal = source_metadata.signals.get("comicvine_series_id") - match_method = ( - { - MetadataSignal.MYLAR3: "mylar3_cv_id", - MetadataSignal.COMICINFO: "comicinfo_cv_id", - MetadataSignal.SIDECAR: "comicinfo_cv_id", - MetadataSignal.PULLBOX_FOLDER: "folder_cv_id", - }.get(signal) - if signal is not None - else None - ) if cv_id is not None and match_method in _DIRECT_CV_ID_MATCH_METHODS: return known_cv_id_evaluation_from_source( int(cv_id), @@ -148,8 +172,19 @@ class _VolumeSubtitleRebucketMatch: diagnostics: dict[str, Any] +@dataclass(frozen=True, slots=True) +class _ScanReviewProfile: + """Minimal persisted facts needed to aggregate matching progress.""" + + id: int + file_count: int + direct_match: bool + has_files: bool + issue_count: int | None + + if TYPE_CHECKING: - from collections.abc import Awaitable, Callable, Coroutine + from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine from datetime import datetime from typing import Protocol @@ -259,46 +294,203 @@ async def _log_detail_event_best_effort( ) -async def run_import_series_matching( +async def _count_pending_import_series(session: AsyncSession, *, job_id: int) -> int: + count = await session.scalar( + sa_select(sa_func.count(ImportedSeries.id)).where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status == ImportSeriesStatus.PENDING, + ) + ) + return int(count or 0) + + +async def _load_pending_import_series_page( session: AsyncSession, - job: ImportJob, *, - metadata_provider: Any, - source_metadata_for_series: SourceMetadataForSeriesFunc, - load_deferred_source_metadata_for_series: DeferredSourceMetadataForSeriesFunc | None = None, - evaluate_match: EvaluateMatchFunc, + job_id: int, + after_id: int, + page_size: int, +) -> list[ImportedSeries]: + result = await session.execute( + sa_select(ImportedSeries) + .where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status == ImportSeriesStatus.PENDING, + ImportedSeries.id > after_id, + ) + .order_by(ImportedSeries.id) + .limit(page_size) + ) + return list(result.scalars()) + + +async def _iter_pending_import_series( + session: AsyncSession, + *, + job_id: int, + page_size: int, raise_if_cancelled: RaiseIfCancelledFunc, - reclassify_duplicates: ReclassifyDuplicatesFunc, - recompute_series_counters: RecomputeCountersFunc, - log_event: LogEventFunc, - emit_progress: EmitProgressFunc, - phase_progress: PhaseProgressFunc, - estimate_remaining_seconds: EstimateRemainingFunc, - job_stats: JobStatsFunc, - maybe_slow_item_delay: SlowItemDelayFunc, - provider_free_filesystem: bool = False, - estimate_remaining_work_seconds: EstimateRemainingWorkFunc | None = None, - progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None = None, -) -> None: - """Run ComicVine matching for all pending imported series in a job.""" - started_at = time.monotonic() - persist_batch_size = 25 - last_checkpoint_at = time.monotonic() +) -> AsyncIterator[tuple[int, ImportedSeries, bool]]: + after_id = 0 + item_index = 0 + while True: + await raise_if_cancelled(session, job_id) + items = await _load_pending_import_series_page( + session, + job_id=job_id, + after_id=after_id, + page_size=page_size, + ) + if not items: + break + for page_index, item in enumerate(items): + yield item_index, item, page_index == len(items) - 1 + item_index += 1 + after_id = items[-1].id - async def _mark_pending_files_no_match(item: ImportedSeries) -> None: - pending_files_result = await session.execute( - sa_select(ImportedFile).where( - ImportedFile.import_series_id == item.id, - ImportedFile.status == ImportedFileStatus.PENDING, + +async def _load_pending_import_file_page( + session: AsyncSession, + *, + import_series_id: int, + after_id: int, + page_size: int, +) -> list[ImportedFile]: + result = await session.execute( + sa_select(ImportedFile) + .where( + ImportedFile.import_series_id == import_series_id, + ImportedFile.status == ImportedFileStatus.PENDING, + ImportedFile.id > after_id, + ) + .order_by(ImportedFile.id) + .limit(page_size) + ) + return list(result.scalars()) + + +async def _load_scan_review_profile_page( + session: AsyncSession, + *, + job_id: int, + after_id: int, + page_size: int, +) -> list[_ScanReviewProfile]: + result = await session.execute( + sa_select( + ImportedSeries.id, + ImportedSeries.files_total, + ImportedSeries.file_count, + ImportedSeries.cv_id, + ImportedSeries.has_files, + ImportedSeries.cv_issue_count, + ) + .where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.status == ImportSeriesStatus.PENDING, + ImportedSeries.id > after_id, + ) + .order_by(ImportedSeries.id) + .limit(page_size) + ) + return [ + _ScanReviewProfile( + id=int(row.id), + file_count=max(int(row.files_total or row.file_count or 0), 0), + direct_match=bool(row.cv_id), + has_files=bool(row.has_files), + issue_count=row.cv_issue_count, + ) + for row in result + ] + + +async def _build_scan_review_progress_plan( + session: AsyncSession, + *, + job_id: int, + analysis_series_count: int, + page_size: int, + raise_if_cancelled: RaiseIfCancelledFunc, +) -> ScanReviewProgressPlan: + series_match_weight = 0.0 + file_match_weight = 0.0 + after_id = 0 + while True: + await raise_if_cancelled(session, job_id) + profiles = await _load_scan_review_profile_page( + session, + job_id=job_id, + after_id=after_id, + page_size=page_size, + ) + if not profiles: + break + for profile in profiles: + series_match_weight += scan_review_series_match_weight( + ScanReviewSeriesMatchProfile( + file_count=profile.file_count, + direct_match=profile.direct_match, + ) ) + if profile.has_files: + file_match_weight += scan_review_file_match_weight( + ScanReviewFileMatchProfile( + file_count=profile.file_count, + issue_count=profile.issue_count, + ) + ) + after_id = profiles[-1].id + + analysis_weight = scan_review_analysis_weight(analysis_series_count) + return ScanReviewProgressPlan( + analysis_weights=(analysis_weight,) if analysis_weight else (), + series_match_weights=(series_match_weight,) if series_match_weight else (), + file_match_weights=(file_match_weight,) if file_match_weight else (), + ) + + +def _aggregate_matching_completed_weight( + plan: ScanReviewProgressPlan, + *, + completed_items: int, + total_items: int, + current_item_progress_pct: int | None = None, +) -> float: + current_fraction = max(min(int(current_item_progress_pct or 0), 100), 0) / 100 + completed_fraction = min( + max((completed_items + current_fraction) / max(total_items, 1), 0.0), + 1.0, + ) + return plan.analysis_weight + plan.series_match_weight * completed_fraction + + +async def _mark_pending_files_no_match( + session: AsyncSession, + item: ImportedSeries, + *, + job_id: int, + page_size: int, + raise_if_cancelled: RaiseIfCancelledFunc, +) -> None: + after_id = 0 + while True: + await raise_if_cancelled(session, job_id) + pending_files = await _load_pending_import_file_page( + session, + import_series_id=item.id, + after_id=after_id, + page_size=page_size, ) - pending_files = list(pending_files_result.scalars().all()) if not pending_files: - return - + break for imp_file in pending_files: diagnostics = dict(imp_file.diagnostics or {}) diagnostics.setdefault("kind", "series_no_match_file") + diagnostics.setdefault( + "reason", + str(dict(item.diagnostics or {}).get("reason") or "series_no_match"), + ) local_identity_missing = ( dict(item.diagnostics or {}).get("reason") == "trusted_source_identity_missing" ) @@ -322,14 +514,41 @@ async def _mark_pending_files_no_match(item: ImportedSeries) -> None: imp_file.is_preferred = False imp_file.error_message = None imp_file.diagnostics = diagnostics + after_id = pending_files[-1].id + await session.flush() - items_result = await session.execute( - sa_select(ImportedSeries).where( - ImportedSeries.import_job_id == job.id, - ImportedSeries.status == ImportSeriesStatus.PENDING, - ) - ) - items = list(items_result.scalars().all()) + +async def run_import_series_matching( + session: AsyncSession, + job: ImportJob, + *, + metadata_provider: Any, + source_metadata_for_series: SourceMetadataForSeriesFunc, + load_deferred_source_metadata_for_series: DeferredSourceMetadataForSeriesFunc | None = None, + evaluate_match: EvaluateMatchFunc, + raise_if_cancelled: RaiseIfCancelledFunc, + reclassify_duplicates: ReclassifyDuplicatesFunc, + recompute_series_counters: RecomputeCountersFunc, + log_event: LogEventFunc, + emit_progress: EmitProgressFunc, + phase_progress: PhaseProgressFunc, + estimate_remaining_seconds: EstimateRemainingFunc, + job_stats: JobStatsFunc, + maybe_slow_item_delay: SlowItemDelayFunc, + provider_free_filesystem: bool = False, + estimate_remaining_work_seconds: EstimateRemainingWorkFunc | None = None, + progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None = None, + item_page_size: int = 100, + file_page_size: int = 250, + profile_page_size: int = 500, +) -> None: + """Run ComicVine matching for all pending imported series in a job.""" + started_at = time.monotonic() + persist_batch_size = 25 + last_checkpoint_at = time.monotonic() + item_page_size = max(item_page_size, 1) + file_page_size = max(file_page_size, 1) + profile_page_size = max(profile_page_size, 1) matched_count = 0 no_match_count = 0 @@ -342,25 +561,19 @@ async def _mark_pending_files_no_match(item: ImportedSeries) -> None: rebucket_duration_ms = 0.0 rebucket_evaluated_files = 0 deferred_metadata_loads = 0 - total_items = len(items) + total_items = await _count_pending_import_series(session, job_id=job.id) + processed_items = 0 runtime_revision_state: dict[str, int] = {"value": int(job.progress_revision or 0)} - progress_plan = scan_review_progress_plan( - analysis_series_count=max(job.series_found or total_items, total_items), - series_match_profiles=[ - ScanReviewSeriesMatchProfile( - file_count=int(item.files_total or item.file_count or 0), - direct_match=bool(item.cv_id), - ) - for item in items - ], - file_match_profiles=[ - ScanReviewFileMatchProfile( - file_count=int(item.files_total or item.file_count or 0), - issue_count=item.cv_issue_count, - ) - for item in items - if item.has_files - ], + progress_plan = ( + await _build_scan_review_progress_plan( + session, + job_id=job.id, + analysis_series_count=max(job.series_found or total_items, total_items), + page_size=profile_page_size, + raise_if_cancelled=raise_if_cancelled, + ) + if progress_callback is not None + else None ) async def emit_matching_progress( @@ -374,11 +587,12 @@ async def emit_matching_progress( ) -> None: if progress_callback is None: return + assert progress_plan is not None - completed_weight = scan_review_completed_weight( + completed_weight = _aggregate_matching_completed_weight( progress_plan, - phase="matching", completed_items=idx, + total_items=total_items, current_item_progress_pct=current_item_progress_pct, ) progress = scan_review_progress_pct( @@ -551,7 +765,12 @@ async def _rebucket_collection_volume_subtitle_series_with_progress( await asyncio.gather(task, return_exceptions=True) raise - for idx, item in enumerate(items): + async for idx, item, is_page_last in _iter_pending_import_series( + session, + job_id=job.id, + page_size=item_page_size, + raise_if_cancelled=raise_if_cancelled, + ): await raise_if_cancelled(session, job.id) try: source_metadata_started_at = time.monotonic() @@ -613,7 +832,8 @@ async def _rebucket_collection_volume_subtitle_series_with_progress( cv_result = evaluation.match except ImportProviderDegradedError as exc: deferred_count += 1 - deferred_titles.append(item.raw_series_name) + if len(deferred_titles) < 10: + deferred_titles.append(item.raw_series_name) clear_auto_cv_match_fields(item) item.status = ImportSeriesStatus.PENDING item.diagnostics = { @@ -655,6 +875,7 @@ async def _rebucket_collection_volume_subtitle_series_with_progress( item.status = ImportSeriesStatus.MATCHED item.diagnostics = evaluation.diagnostics if evaluation is not None else {} matched_count += 1 + rebucket_result = VolumeSubtitleRebucketResult() if not provider_free_filesystem: rebucket_started_at = time.monotonic() rebucket_result = await _rebucket_collection_volume_subtitle_series_with_progress( @@ -667,27 +888,33 @@ async def _rebucket_collection_volume_subtitle_series_with_progress( matched_count += rebucket_result.created_series_count if rebucket_result.removed_source_series: matched_count -= 1 - continue - pending_detail_logs.append( - { - "level": "DEBUG", - "event": "import_series_match_detail", - "message": f"Matched '{item.raw_series_name}' -> CV {cv_result['cv_id']}", - "details": { - "raw_series_name": item.raw_series_name, - "cv_id": cv_result["cv_id"], - "cv_title": cv_result["cv_title"], - "score": cv_result["cv_match_score"], - "method": cv_result["cv_match_method"], - }, - } - ) + if not rebucket_result.removed_source_series: + pending_detail_logs.append( + { + "level": "DEBUG", + "event": "import_series_match_detail", + "message": f"Matched '{item.raw_series_name}' -> CV {cv_result['cv_id']}", + "details": { + "raw_series_name": item.raw_series_name, + "cv_id": cv_result["cv_id"], + "cv_title": cv_result["cv_title"], + "score": cv_result["cv_match_score"], + "method": cv_result["cv_match_method"], + }, + } + ) elif evaluation is not None: item.status = ImportSeriesStatus.NO_MATCH item.diagnostics = evaluation.diagnostics clear_auto_cv_match_fields(item) - await _mark_pending_files_no_match(item) + await _mark_pending_files_no_match( + session, + item, + job_id=job.id, + page_size=file_page_size, + raise_if_cancelled=raise_if_cancelled, + ) no_match_count += 1 diagnostics = dict(item.diagnostics or {}) @@ -717,7 +944,8 @@ async def _rebucket_collection_volume_subtitle_series_with_progress( should_checkpoint = ( idx == 0 or (idx + 1) % persist_batch_size == 0 - or idx == len(items) - 1 + or idx == total_items - 1 + or is_page_last or (time.monotonic() - last_checkpoint_at) >= 0.5 ) if should_checkpoint: @@ -737,11 +965,11 @@ async def _rebucket_collection_volume_subtitle_series_with_progress( ) if progress_callback and should_checkpoint: - total_items = len(items) - completed_weight = scan_review_completed_weight( + assert progress_plan is not None + completed_weight = _aggregate_matching_completed_weight( progress_plan, - phase="matching", completed_items=idx + 1, + total_items=total_items, ) progress = scan_review_progress_pct( progress_plan, @@ -784,6 +1012,16 @@ async def _rebucket_collection_volume_subtitle_series_with_progress( progress_callback, ) await maybe_slow_item_delay() + elif progress_callback: + # Completing the displayed series must not wait for a DB batch. + await emit_matching_progress( + item, + idx, + message=f"Completed series review {idx + 1}/{total_items}.", + current_item_progress_pct=100, + live_only=True, + ) + processed_items = idx + 1 if deferred_count: job.series_matched = matched_count @@ -799,9 +1037,9 @@ async def _rebucket_collection_volume_subtitle_series_with_progress( "import_matching_provider_degraded", message=job.error_message, deferred_count=deferred_count, - deferred_titles=deferred_titles[:10], + deferred_titles=deferred_titles, duration_ms=round((time.monotonic() - started_at) * 1000), - items_processed=len(items), + items_processed=processed_items, source_metadata_duration_ms=round(source_metadata_duration_ms), deferred_metadata_duration_ms=round(deferred_metadata_duration_ms), provider_evaluation_duration_ms=round(provider_evaluation_duration_ms), @@ -831,7 +1069,7 @@ async def _rebucket_collection_volume_subtitle_series_with_progress( matched=matched_count, no_match=no_match_count, duration_ms=round((time.monotonic() - started_at) * 1000), - items_processed=len(items), + items_processed=processed_items, source_metadata_duration_ms=round(source_metadata_duration_ms), deferred_metadata_duration_ms=round(deferred_metadata_duration_ms), provider_evaluation_duration_ms=round(provider_evaluation_duration_ms), diff --git a/src/pullbox/services/import_service.py b/src/pullbox/services/import_service.py index e7019051..769b327c 100644 --- a/src/pullbox/services/import_service.py +++ b/src/pullbox/services/import_service.py @@ -8,6 +8,7 @@ import asyncio from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, Any from uuid import uuid4 @@ -21,8 +22,17 @@ register_library_file_with_metadata, ) from pullbox.core.mylar3_reader import Mylar3Reader -from pullbox.models.import_job import ImportedSeries, ImportJob, ImportJobAction, ImportJobStatus -from pullbox.services.import_catalog_hydration import run_pending_catalog_hydration +from pullbox.models.import_job import ( + ImportedSeries, + ImportJob, + ImportJobAction, + ImportJobStatus, + ImportSourceType, +) +from pullbox.services.import_catalog_hydration import ( + ensure_catalog_hydration_retry_scheduled, + run_pending_catalog_hydration, +) from pullbox.services.import_comicinfo_enrichment import ( run_pending_import_comicinfo_enrichment, schedule_import_comicinfo_enrichment, @@ -46,6 +56,7 @@ next_action_sequence as next_import_action_sequence, ) from pullbox.services.import_job_actions import record_action as record_import_action +from pullbox.services.import_job_actions import record_actions as record_import_actions from pullbox.services.import_job_actions import rollback_action as rollback_import_action from pullbox.services.import_job_controls import ( raise_if_job_cancelled_immediately as raise_if_import_job_cancelled_immediately, @@ -71,6 +82,9 @@ from pullbox.services.import_matching import ( score_cv_result as _score_cv_result, # noqa: F401 - compatibility import ) +from pullbox.services.import_placement_recovery import ( + load_completed_import_placement_recovery, +) from pullbox.services.import_provider_cache import ( CachedImportMetadataProvider, build_import_scan_metadata_provider, @@ -103,6 +117,10 @@ from pullbox.services.import_service_matching import ImportServiceMatchingMixin from pullbox.services.import_service_recovery import ImportServiceRecoveryMixin from pullbox.services.import_service_review import ImportServiceReviewMixin +from pullbox.services.import_story_arc_placement_completion import ( + ImportStoryArcPlacementCompletionState, + finalize_import_story_arc_placements, +) from pullbox.services.import_workflow_state import ( emit_progress, estimate_remaining_seconds, @@ -117,9 +135,8 @@ from pullbox.utilities.sse import publish if TYPE_CHECKING: - from collections.abc import Awaitable, Callable + from collections.abc import Awaitable, Callable, Sequence from datetime import datetime - from pathlib import Path from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -127,7 +144,6 @@ from pullbox.core.events import EventBus from pullbox.core.library_permissions import LibraryPermissionPolicy from pullbox.core.library_policy import LibraryIngestPolicy - from pullbox.models.import_job import ImportJobAction from pullbox.models.issue import Issue from pullbox.models.library import LibraryFile, MatchConfidence from pullbox.providers.base import SeriesMetadata @@ -137,6 +153,7 @@ ImportProgressEvent, ) from pullbox.services.import_file_execution_protocols import ReportFileProgressFunc + from pullbox.services.import_job_actions import ImportJobActionSpec from pullbox.services.metadata_service import MetadataService from pullbox.services.series_service import SeriesService @@ -151,6 +168,7 @@ class RunImportResult: """Post-transaction follow-up work requested by import execution.""" schedule_comicinfo_enrichment: bool = False + schedule_story_arc_sync: bool = False # ── ImportService class ────────────────────────────────────────────────── @@ -325,9 +343,16 @@ async def _validate_discovered_files_safety( self, session: AsyncSession, discovered_list: list[DiscoveredSeries], + *, + progress_callback: Callable[[int, int, str], Awaitable[None]] | None = None, ) -> None: """Run safety checks on discovered source files before review/import.""" - await validate_import_discovered_files_safety(session, discovered_list) + await validate_import_discovered_files_safety( + session, + discovered_list, + progress_callback=progress_callback, + worker_count=self._settings.import_scan_worker_count, + ) def _build_scan_metadata_provider(self, session: AsyncSession) -> CachedImportMetadataProvider: """Return the Step 2 provider stack: persistent cache, then per-job cache.""" @@ -449,6 +474,15 @@ async def _record_action( payload=payload, ) + async def _record_actions( + self, + session: AsyncSession, + job: ImportJob, + specs: Sequence[ImportJobActionSpec], + ) -> list[ImportJobAction]: + """Persist one bounded rollback-journal action batch.""" + return await record_import_actions(session, job, specs) + async def _register_import_library_file( self, session: AsyncSession, @@ -469,6 +503,12 @@ async def _register_import_library_file( self._materialize_import_cbz_with_comicinfo_interruptible ), ) + recovery_imported_file_id = kwargs.pop("recovery_imported_file_id", None) + recovery_source_value = kwargs.pop("recovery_original_source_path", None) + recovery_original_source_path = ( + Path(recovery_source_value) if recovery_source_value is not None else source_path + ) + placement_action_id: int | None = None async def placement_started_callback( *, @@ -477,27 +517,112 @@ async def placement_started_callback( transfer_method: str, series_folder_created: bool, series_folder_path: Path, + created_directory_paths: tuple[Path, ...] = (), + directory_ownership_boundary_path: Path | None = None, temp_paths: tuple[Path, ...] = (), ) -> None: - await self._record_action( + nonlocal placement_action_id + action = await self._record_action( session, job, phase="import", action_type="library_file_placement_started", payload={ + "imported_file_id": recovery_imported_file_id, + "issue_id": issue.id, "destination_path": str(target_path), - "original_source_path": str(source_path), + "original_source_path": str(recovery_original_source_path), "artifact_source_path": str(artifact_source_path), "transfer_method": transfer_method, "created_series_folder": series_folder_created, "created_series_folder_path": str(series_folder_path), + "created_directory_paths": [str(path) for path in created_directory_paths], + "directory_ownership_boundary_path": ( + str(directory_ownership_boundary_path) + if directory_ownership_boundary_path is not None + else None + ), "temp_paths": [str(path) for path in temp_paths], + "placement_completed": False, }, ) + placement_action_id = action.id # This journal row must survive if archive materialization raises and # the caller rolls back the active session. await session.commit() + async def placement_completed_callback( + *, + target_path: Path, + destination_signature: dict[str, int | str], + ) -> None: + if placement_action_id is None: + raise RuntimeError("Import placement completed without a durable start record") + action = await session.get(ImportJobAction, placement_action_id) + if action is None: + raise RuntimeError("Import placement start record disappeared before completion") + payload = dict(action.payload or {}) + if str(payload.get("destination_path") or "") != str(target_path): + raise RuntimeError("Import placement completion target changed after planning") + payload["placement_completed"] = True + payload["destination_signature"] = dict(destination_signature) + action.payload = payload + await session.commit() + + source_scan_root = kwargs.pop( + "source_scan_root", + Path(job.source_path) if job.source_type == ImportSourceType.FILESYSTEM else None, + ) + kwargs.pop("strict_import_target", None) + + transfer_method = str(kwargs.get("transfer_method") or "") + recovery = None + if ( + isinstance(recovery_imported_file_id, int) + and not isinstance(recovery_imported_file_id, bool) + and issue.id is not None + and transfer_method + ): + recovery = await load_completed_import_placement_recovery( + session, + job_id=int(job.id), + imported_file_id=recovery_imported_file_id, + issue_id=int(issue.id), + source_path=recovery_original_source_path, + transfer_method=transfer_method, + ) + if recovery is not None: + recovery_kwargs = dict(kwargs) + recovery_kwargs.update( + { + "move_to_library": False, + "expected_source_signature": None, + "transfer_method": "recovered", + "normalize_to_cbz": False, + "update_embedded_comicinfo_from_match": False, + "comicinfo_payload": None, + "rename": False, + "recover_existing_managed_artifact": True, + } + ) + result = await register_library_file( + session, + recovery.destination_path, + issue, + confidence, + source_scan_root=None, + strict_import_target=True, + **recovery_kwargs, + ) + library_file = ( + result.library_file + if isinstance(result, LibraryFileRegistrationOutcome) + else result + ) + library_file.source_signature = dict(recovery.destination_signature) + await session.flush() + return result + result = await register_library_file( session, source_path, @@ -508,6 +633,10 @@ async def placement_started_callback( artifact_transfer=adapters.artifact_transfer, comicinfo_materializer=adapters.comicinfo_materializer, placement_started_callback=placement_started_callback, + placement_completed_callback=placement_completed_callback, + placement_temp_paths=adapters.placement_temp_paths, + source_scan_root=source_scan_root, + strict_import_target=True, **kwargs, ) await self._log_import_file_timing_events( @@ -555,9 +684,9 @@ async def rollback_import( job_id: int, *, progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None = None, - ) -> None: + ) -> bool: """Rollback durable import actions in reverse order.""" - await rollback_import_job( + return await rollback_import_job( session, job_id, rollback_action=self._rollback_action, @@ -579,6 +708,20 @@ async def run_import( ) -> RunImportResult: """Execute confirmed new-series imports plus duplicate-series file merges.""" try: + current_job = await session.get(ImportJob, job_id) + if ( + current_job is not None + and dict(current_job.progress_snapshot or {}).get("phase") == "story_arc_placements" + ): + outcome = await finalize_import_story_arc_placements(session, job_id) + return RunImportResult( + schedule_comicinfo_enrichment=( + outcome.state is ImportStoryArcPlacementCompletionState.COMPLETED + ), + schedule_story_arc_sync=( + outcome.state is ImportStoryArcPlacementCompletionState.PENDING + ), + ) await execute_import_job( session, job_id, @@ -586,6 +729,7 @@ async def run_import( process_series_files=self._process_series_files, raise_if_cancelled=self._raise_if_job_cancelled, record_action=self._record_action, + record_actions=self._record_actions, log_event=self._log_event, emit_progress=self._emit_progress, estimate_remaining_seconds=self._estimate_remaining_seconds, @@ -593,9 +737,16 @@ async def run_import( progress_callback=progress_callback, ) completed_job = await session.get(ImportJob, job_id) + story_arc_placements_pending = ( + completed_job is not None + and completed_job.status == ImportJobStatus.IMPORTING + and dict(completed_job.progress_snapshot or {}).get("phase") + == "story_arc_placements" + ) return RunImportResult( schedule_comicinfo_enrichment=completed_job is not None - and completed_job.status == ImportJobStatus.COMPLETED + and completed_job.status == ImportJobStatus.COMPLETED, + schedule_story_arc_sync=story_arc_placements_pending, ) finally: self._import_runtime_cache_by_job.pop(job_id, None) @@ -613,8 +764,15 @@ def schedule_comicinfo_enrichment( build_comicinfo_payload=self._build_comicinfo_payload_for_issue, apply_comicinfo=self._apply_comicinfo_to_imported_artifact, log_event=self._log_event, + prefetch_issue_metadata=self._metadata_service.prefetch_issue_metadata_batch, ) + def schedule_story_arc_sync(self) -> None: + """Nudge durable story-arc work only after its import transaction commits.""" + from pullbox.services.story_arc_sync_queue import request_story_arc_sync_now + + request_story_arc_sync_now() + async def recover_pending_comicinfo_enrichment( self, session_factory: async_sessionmaker[AsyncSession], @@ -625,6 +783,7 @@ async def recover_pending_comicinfo_enrichment( build_comicinfo_payload=self._build_comicinfo_payload_for_issue, apply_comicinfo=self._apply_comicinfo_to_imported_artifact, log_event=self._log_event, + prefetch_issue_metadata=self._metadata_service.prefetch_issue_metadata_batch, ) async def recover_pending_catalog_hydration( @@ -632,10 +791,15 @@ async def recover_pending_catalog_hydration( session_factory: async_sessionmaker[AsyncSession], ) -> int: """Resume full catalog hydration left pending after a restart.""" - return await run_pending_catalog_hydration( + recovered = await run_pending_catalog_hydration( + session_factory, + series_service=self._series_service, + ) + await ensure_catalog_hydration_retry_scheduled( session_factory, series_service=self._series_service, ) + return recovered async def _process_series_files( self, @@ -704,7 +868,7 @@ async def override_cv_id( async def _fetch_series_metadata_for_override(self, cv_id: int) -> SeriesMetadata: """Fetch ComicVine metadata for a manual imported-series override.""" - return await self._metadata_service._provider.get_series(str(cv_id)) + return await self._metadata_service.get_series_metadata(cv_id) async def rematch_imported_series_files( self, diff --git a/src/pullbox/services/import_service_file_operations.py b/src/pullbox/services/import_service_file_operations.py index d1fb9f8a..1f8e4cdb 100644 --- a/src/pullbox/services/import_service_file_operations.py +++ b/src/pullbox/services/import_service_file_operations.py @@ -232,6 +232,7 @@ async def _materialize_import_cbz_with_comicinfo_interruptible( comicinfo_payload: dict[str, Any], *, transfer_method: str, + temp_path: Path | None = None, progress_callback: Callable[[str, int, int, str], Awaitable[None] | None] | None = None, ) -> bool: """Run a killable combined CBZ materialization and ComicInfo write.""" @@ -242,6 +243,7 @@ async def _materialize_import_cbz_with_comicinfo_interruptible( target_path, comicinfo_payload, transfer_method=transfer_method, + temp_path=temp_path, progress_callback=progress_callback, raise_if_cancelled_immediately=self._raise_if_job_cancelled_immediately, ) diff --git a/src/pullbox/services/import_service_job_lifecycle.py b/src/pullbox/services/import_service_job_lifecycle.py index 440d2259..ae4c2176 100644 --- a/src/pullbox/services/import_service_job_lifecycle.py +++ b/src/pullbox/services/import_service_job_lifecycle.py @@ -19,6 +19,9 @@ copy_retry_import_settings, ) from pullbox.services.import_review_preview import get_preview as get_import_preview +from pullbox.services.story_arc_sync_queue import ( + retry_import_story_arc_sync_work as retry_import_story_arc_placements, +) if TYPE_CHECKING: from collections.abc import Awaitable, Callable @@ -36,7 +39,7 @@ async def rollback_import( job_id: int, *, progress_callback: Callable[[ImportProgressEvent], Awaitable[None]] | None = None, - ) -> None: ... + ) -> bool: ... async def _log_event( self, @@ -78,9 +81,20 @@ async def cancel_job( job_id, log_event=self._log_event, ) - await self.rollback_import(session, job_id) + rollback_completed = await self.rollback_import(session, job_id) + if not rollback_completed: + logger.info( + "import_job_delete_waiting_for_story_arc_rollback", + job_id=job_id, + ) + return "rollback_pending" reloaded = await session.get(ImportJob, job_id) if reloaded is not None: + if ( + reloaded.status == ImportJobStatus.FAILED + and dict(reloaded.progress_snapshot or {}).get("mode") == "rollback" + ): + return "rollback_incomplete" self._log_job_deleted(job_id, reloaded.status) await session.delete(reloaded) await session.flush() @@ -145,6 +159,23 @@ async def request_rollback( log_event=self._log_event, ) + async def retry_story_arc_placements( + self: ImportServiceJobLifecycleContext, + session: AsyncSession, + job_id: int, + ) -> tuple[ImportJob, int]: + """Requeue exact failed/cancelled placement work without replaying import.""" + job, retrying_count = await retry_import_story_arc_placements(session, job_id) + await self._log_event( + session, + job_id, + "INFO", + "import_story_arc_placements_retry_requested", + message=(f"Retrying {retrying_count} failed or cancelled Story Arc placements."), + retrying_count=retrying_count, + ) + return job, retrying_count + async def retry_job( self: ImportServiceJobLifecycleContext, session: AsyncSession, diff --git a/src/pullbox/services/import_service_matching.py b/src/pullbox/services/import_service_matching.py index cd9593c5..74bcce80 100644 --- a/src/pullbox/services/import_service_matching.py +++ b/src/pullbox/services/import_service_matching.py @@ -493,9 +493,13 @@ async def _run_file_matching( def _metadata_provider_for_job(self: ImportServiceMatchingContext, job_id: int) -> Any: """Return the job-scoped cached provider when Step 2 is actively scanning.""" + from pullbox.services.catalog.lookup import catalog_or_provider + if self._metadata_service is None: return None - return self._scan_provider_cache_by_job.get(job_id, self._metadata_service._provider) + return self._scan_provider_cache_by_job.get( + job_id, catalog_or_provider(self._metadata_service._provider) + ) async def _record_duplicate_copy_cluster( self: ImportServiceMatchingContext, diff --git a/src/pullbox/services/import_service_recovery.py b/src/pullbox/services/import_service_recovery.py index 821ddb8a..47ffbc0f 100644 --- a/src/pullbox/services/import_service_recovery.py +++ b/src/pullbox/services/import_service_recovery.py @@ -71,6 +71,8 @@ async def get_orphaned_series( page: int = 1, page_size: int = 25, sort: str = "file_count_desc", + *, + job_id: int | None = None, ) -> tuple[list[ImportedSeries], int]: """Return paginated ImportedSeries with status=NO_MATCH from COMPLETED jobs.""" return await get_import_orphaned_series( @@ -78,14 +80,17 @@ async def get_orphaned_series( page=page, page_size=page_size, sort=sort, + job_id=job_id, ) async def get_orphaned_count( self, session: AsyncSession, + *, + job_id: int | None = None, ) -> int: """Return total count of active unmatched series from COMPLETED jobs.""" - return await get_import_orphaned_count(session) + return await get_import_orphaned_count(session, job_id=job_id) async def _load_orphan_recovery_item( self, @@ -198,10 +203,13 @@ async def retry_failed_series( self: ImportServiceRecoveryContext, session: AsyncSession, job_id: int, + *, + file_ids: list[int] | None = None, ) -> tuple[ImportJob, int]: """Reset failed import rows/files for a job back to retryable states.""" return await retry_import_failed_series( session, job_id, log_event=self._log_event, + file_ids=file_ids, ) diff --git a/src/pullbox/services/import_service_review.py b/src/pullbox/services/import_service_review.py index 4b211fed..7dd2c32f 100644 --- a/src/pullbox/services/import_service_review.py +++ b/src/pullbox/services/import_service_review.py @@ -45,13 +45,21 @@ from pullbox.services.import_review_actions import ( update_series_selection as update_import_series_selection, ) +from pullbox.services.import_review_queries import ConflictGroupsPage from pullbox.services.import_review_queries import ( get_conflict_groups as get_import_conflict_groups, ) +from pullbox.services.import_review_queries import ( + get_conflict_groups_page as get_import_conflict_groups_page, +) from pullbox.services.import_review_queries import ( get_files_for_series as get_import_files_for_series, ) from pullbox.services.import_review_selection import load_import_review_selection_state +from pullbox.services.import_story_arc_review import ( + StoryArcReviewAction, + update_import_story_arc_decision, +) if TYPE_CHECKING: from typing import Protocol @@ -60,6 +68,7 @@ from pullbox.models.import_job import ImportedFile, ImportedFileStatus from pullbox.models.issue import Issue + from pullbox.models.story_arc_import import ImportedStoryArc class ImportServiceReviewContext(Protocol): _sync_import_file_source_metadata: Any @@ -133,6 +142,24 @@ async def get_conflict_groups( """Return all conflict groups for a job.""" return await get_import_conflict_groups(session, job_id) + async def get_conflict_groups_page( + self, + session: AsyncSession, + job_id: int, + *, + page: int = 1, + page_size: int = 25, + sort: str = "legacy", + ) -> ConflictGroupsPage: + """Return one bounded conflict-group page for request paths.""" + return await get_import_conflict_groups_page( + session, + job_id, + page=page, + page_size=page_size, + sort=sort, + ) + async def override_file_match( self: ImportServiceReviewContext, session: AsyncSession, @@ -279,6 +306,24 @@ async def update_series_selection( include_in_import=include_in_import, ) + async def update_story_arc_decision( + self, + session: AsyncSession, + job_id: int, + imported_story_arc_id: int, + *, + action: StoryArcReviewAction, + proposed_story_arc_id: int | None, + ) -> ImportedStoryArc: + """Persist one staged story-arc select/skip decision.""" + return await update_import_story_arc_decision( + session, + job_id, + imported_story_arc_id, + action=action, + proposed_story_arc_id=proposed_story_arc_id, + ) + async def allow_safety_blocked_file_once( self: ImportServiceReviewContext, session: AsyncSession, @@ -296,11 +341,6 @@ async def allow_safety_blocked_file_once( imported_series = await session.get(ImportedSeries, imp_file.import_series_id) if imported_series is None: raise NotFoundError("ImportedSeries", imp_file.import_series_id) - if imported_series.status in {ImportSeriesStatus.MATCHED, ImportSeriesStatus.DUPLICATE}: - diagnostics = dict(imported_series.diagnostics or {}) - diagnostics["rematch_pending"] = True - imported_series.diagnostics = diagnostics - await session.flush() return imported_series async def allow_safety_blocked_file_once_for_retry( @@ -348,6 +388,7 @@ async def skip_safety_blocked_file( (imported_series.files_matched or 0) <= 0 and (imported_series.files_conflict or 0) <= 0 and (imported_series.files_no_match or 0) <= 0 + and int((imported_series.diagnostics or {}).get("safety_blocked_files") or 0) <= 0 ): job = await session.get(ImportJob, job_id) if job is None: diff --git a/src/pullbox/services/import_source_metadata.py b/src/pullbox/services/import_source_metadata.py index a870d86a..9806bf4d 100644 --- a/src/pullbox/services/import_source_metadata.py +++ b/src/pullbox/services/import_source_metadata.py @@ -10,6 +10,7 @@ from sqlalchemy import select as sa_select +from pullbox.core.issue_numbers import format_issue_number from pullbox.core.name_matcher import NameMatcher from pullbox.core.release_parser import normalize_issue_number, parse_release_title from pullbox.core.source_metadata import ( @@ -18,6 +19,7 @@ SourceMetadataExtractor, volume_subtitle_hint_from_filename, ) +from pullbox.core.type_semantics import issue_type_family from pullbox.models.import_job import ImportedFile, ImportedFileStatus, ImportedSeries from pullbox.models.issue import IssueType @@ -157,7 +159,7 @@ def _archive_entry_issue_hint(diagnostics: dict[str, Any]) -> dict[str, Any] | N def _format_issue_number(issue_number: float | None) -> str: if issue_number is None: return "unknown" - return f"{issue_number:g}" + return format_issue_number(issue_number) def _issue_numbers_equal(left: float, right: float) -> bool: @@ -303,6 +305,7 @@ def source_metadata_for_import_file( year=imp_file.parsed_year or imp_series.raw_year, volume=_filename_parse_volume(diagnostics, imp_file.file_name), issue_type=_source_issue_type(raw_issue_type), + comicvine_series_id=_optional_int(diagnostics.get("comicvine_series_id")), comicvine_issue_id=imp_file.comicvine_issue_id, signals=_metadata_signals(diagnostics.get("metadata_signals")), diagnostics=source_diagnostics, @@ -325,11 +328,18 @@ async def load_archive_entry_issue_hint_for_import_file( if Path(metadata.source_path).suffix.lower() not in {".cbz", ".cbr", ".cb7"}: return metadata - hint = await asyncio.to_thread( - SourceMetadataExtractor.archive_entry_issue_hint_from_path, - metadata.source_path, - expected_series_name=metadata.series_name, - ) + archive_member_evidence = _archive_member_evidence_for_import_file(imp_file) + if ( + archive_member_evidence is not None + and archive_member_evidence.get("member_index_scanned") is True + ): + hint = SourceMetadataExtractor._archive_hint_from_member_evidence(archive_member_evidence) + else: + hint = await asyncio.to_thread( + SourceMetadataExtractor.archive_entry_issue_hint_from_path, + metadata.source_path, + expected_series_name=metadata.series_name, + ) source_diagnostics = {**metadata.diagnostics, "archive_entry_issue_hint_checked": True} persisted_diagnostics = dict(imp_file.diagnostics or {}) persisted_source_metadata = persisted_diagnostics.get("source_metadata") @@ -352,7 +362,13 @@ async def load_archive_entry_issue_hint_for_import_file( def import_file_has_deferred_archive_metadata(imp_file: ImportedFile) -> bool: """Return True when a file still has deferred archive metadata available.""" diagnostics = dict(imp_file.diagnostics or {}) - return _has_deferred_archive_metadata(diagnostics.get("source_metadata")) + source_metadata = diagnostics.get("source_metadata") + if not _has_deferred_archive_metadata(source_metadata): + return False + evidence = _archive_member_evidence_for_import_file(imp_file) + if evidence is not None and evidence.get("member_index_scanned") is True: + return int(evidence.get("comicinfo_entry_count") or 0) > 0 + return True async def load_deferred_source_metadata_for_import_file( @@ -366,10 +382,14 @@ async def load_deferred_source_metadata_for_import_file( if not import_file_has_deferred_archive_metadata(imp_file): return base_metadata + cached_sidecar = cached_mylar_sidecar_data(base_metadata.diagnostics) + archive_member_evidence = _archive_member_evidence_for_import_file(imp_file) loaded_metadata = await asyncio.to_thread( - SourceMetadataExtractor().from_archive_path, + SourceMetadataExtractor().from_path, imp_file.file_path, include_archive_entry_issue_hint=include_archive_entry_issue_hint, + sidecar_data=cached_sidecar, + archive_member_evidence=archive_member_evidence, ) update: dict[str, object] = {} if loaded_metadata.series_name is None and base_metadata.series_name is not None: @@ -378,9 +398,243 @@ async def load_deferred_source_metadata_for_import_file( update["issue_number"] = base_metadata.issue_number if loaded_metadata.year is None and base_metadata.year is not None: update["year"] = base_metadata.year + issue_identity_reconciliation = _corroborated_comicinfo_issue_reconciliation( + base_metadata, + loaded_metadata, + ) + identity_conflicts = _reconcile_loaded_exact_identities(base_metadata, loaded_metadata) + if issue_identity_reconciliation is not None: + recorded_issue_id = issue_identity_reconciliation["recorded_comicvine_issue_id"] + embedded_issue_id = issue_identity_reconciliation["embedded_comicvine_issue_id"] + identity_conflicts = [ + conflict + for conflict in identity_conflicts + if not ( + conflict.get("field") == "comicvine_issue_id" + and conflict.get("first") == recorded_issue_id + and conflict.get("conflicting") == embedded_issue_id + ) + ] + reconciliation = base_metadata.diagnostics.get("mylar3_path_reconciliation") + diagnostics = dict(loaded_metadata.diagnostics) + if issue_identity_reconciliation is not None: + diagnostics["mylar3_issue_identity_reconciliation"] = issue_identity_reconciliation + update["diagnostics"] = diagnostics + folder_scope_conflict = _deferred_mylar_folder_scope_conflict( + base_metadata, + loaded_metadata, + ) + if folder_scope_conflict is not None: + diagnostics["mylar3_folder_scope_conflict"] = folder_scope_conflict + update["diagnostics"] = diagnostics + if isinstance(reconciliation, dict): + diagnostics["mylar3_path_reconciliation"] = dict(reconciliation) + update["diagnostics"] = diagnostics + if identity_conflicts: + update["diagnostics"] = { + **diagnostics, + "identity_conflicts": identity_conflicts, + } + elif "identity_conflicts" in diagnostics: + diagnostics.pop("identity_conflicts", None) + update["diagnostics"] = diagnostics + base_series_signal = base_metadata.signals.get("comicvine_series_id") + if ( + base_metadata.comicvine_series_id is not None + and base_series_signal == MetadataSignal.MYLAR3 + ): + update["comicvine_series_id"] = base_metadata.comicvine_series_id + update["signals"] = { + **loaded_metadata.signals, + "comicvine_series_id": MetadataSignal.MYLAR3, + } + base_issue_signal = base_metadata.signals.get("comicvine_issue_id") + if ( + base_metadata.comicvine_issue_id is not None + and base_issue_signal == MetadataSignal.MYLAR3 + and issue_identity_reconciliation is None + ): + updated_signals = update.get("signals") + if not isinstance(updated_signals, dict): + updated_signals = loaded_metadata.signals + update["comicvine_issue_id"] = base_metadata.comicvine_issue_id + update["signals"] = { + **updated_signals, + "comicvine_issue_id": MetadataSignal.MYLAR3, + } return loaded_metadata.model_copy(update=update) if update else loaded_metadata +def _archive_member_evidence_for_import_file( + imp_file: ImportedFile, +) -> dict[str, Any] | None: + diagnostics = dict(imp_file.diagnostics or {}) + evidence = diagnostics.get("archive_member_evidence") + if not isinstance(evidence, dict): + source_metadata = diagnostics.get("source_metadata") + evidence = ( + source_metadata.get("archive_member_evidence") + if isinstance(source_metadata, dict) + else None + ) + return dict(evidence) if isinstance(evidence, dict) else None + + +def cached_mylar_sidecar_data(diagnostics: dict[str, object]) -> dict[str, Any] | None: + """Restore the normalized folder evidence Mylar already read once.""" + if diagnostics.get("mylar3_folder_metadata_scanned") is not True: + return None + snapshot = diagnostics.get("sidecar_snapshot") + if isinstance(snapshot, dict): + booktype = snapshot.get("booktype") + normalized_booktype: IssueType | None = None + if isinstance(booktype, str): + with contextlib.suppress(ValueError): + normalized_booktype = IssueType(booktype) + raw_conflicts = snapshot.get("identity_conflicts") + return { + "files_present": list(snapshot.get("files_present") or []), + "series_id": _optional_int(snapshot.get("series_id")), + "series_id_source": snapshot.get("series_id_source"), + "issue_id": _optional_int(snapshot.get("issue_id")), + "booktype": normalized_booktype, + "series_status": snapshot.get("series_status"), + "issue_count": _optional_int(snapshot.get("issue_count")), + "series_name": snapshot.get("series_name"), + "year": _optional_int(snapshot.get("year")), + "identity_conflicts": ( + [dict(item) for item in raw_conflicts if isinstance(item, dict)] + if isinstance(raw_conflicts, list) + else [] + ), + } + identity = diagnostics.get("sidecar_identity") + identity = identity if isinstance(identity, dict) else {} + raw_files_present = diagnostics.get("sidecar_files_present") + return { + "files_present": ( + [str(item) for item in raw_files_present] if isinstance(raw_files_present, list) else [] + ), + "series_id": _optional_int(identity.get("comicvine_series_id")), + "issue_id": _optional_int(identity.get("comicvine_issue_id")), + "booktype": None, + "series_status": None, + "issue_count": None, + "series_name": None, + "year": None, + "identity_conflicts": [], + } + + +def _reconcile_loaded_exact_identities( + base_metadata: SourceMetadata, + loaded_metadata: SourceMetadata, +) -> list[dict[str, object]]: + """Compare deferred archive identities with trusted scan-time identities.""" + raw_conflicts = loaded_metadata.diagnostics.get("identity_conflicts") + conflicts = ( + [dict(conflict) for conflict in raw_conflicts if isinstance(conflict, dict)] + if isinstance(raw_conflicts, list) + else [] + ) + for field_name, base_id, loaded_id in ( + ( + "comicvine_series_id", + base_metadata.comicvine_series_id, + loaded_metadata.comicvine_series_id, + ), + ( + "comicvine_issue_id", + base_metadata.comicvine_issue_id, + loaded_metadata.comicvine_issue_id, + ), + ): + if base_id is None or loaded_id is None or base_id == loaded_id: + continue + conflict = { + "field": field_name, + "first": base_id, + "conflicting": loaded_id, + } + if conflict not in conflicts: + conflicts.append(conflict) + return conflicts + + +def _corroborated_comicinfo_issue_reconciliation( + base_metadata: SourceMetadata, + loaded_metadata: SourceMetadata, +) -> dict[str, object] | None: + """Prefer an embedded issue ID when independent local evidence proves a stale Mylar ID.""" + recorded_issue_id = base_metadata.comicvine_issue_id + embedded_issue_id = loaded_metadata.comicvine_issue_id + base_issue_type_is_unqualified = ( + base_metadata.issue_type == IssueType.ISSUE and "issue_type" not in base_metadata.signals + ) + if ( + recorded_issue_id is None + or embedded_issue_id is None + or recorded_issue_id == embedded_issue_id + or base_metadata.signals.get("comicvine_issue_id") != MetadataSignal.MYLAR3 + or loaded_metadata.signals.get("comicvine_issue_id") != MetadataSignal.COMICINFO + or base_metadata.comicvine_series_id is None + or loaded_metadata.comicvine_series_id is None + or base_metadata.comicvine_series_id != loaded_metadata.comicvine_series_id + or loaded_metadata.signals.get("comicvine_series_id") != MetadataSignal.COMICINFO + or base_metadata.issue_number is None + or loaded_metadata.issue_number is None + or not _issue_numbers_equal(base_metadata.issue_number, loaded_metadata.issue_number) + or ( + not base_issue_type_is_unqualified + and issue_type_family(base_metadata.issue_type) + != issue_type_family(loaded_metadata.issue_type) + ) + or loaded_metadata.diagnostics.get("has_comicinfo") is not True + ): + return None + + raw_loaded_conflicts = loaded_metadata.diagnostics.get("identity_conflicts") + if isinstance(raw_loaded_conflicts, list) and any( + isinstance(conflict, dict) + and conflict.get("field") in {"comicvine_series_id", "comicvine_issue_id"} + for conflict in raw_loaded_conflicts + ): + return None + + return { + "recorded_comicvine_issue_id": int(recorded_issue_id), + "embedded_comicvine_issue_id": int(embedded_issue_id), + "comicvine_series_id": int(loaded_metadata.comicvine_series_id), + "issue_number": float(loaded_metadata.issue_number), + "method": "corroborated_embedded_comicinfo", + } + + +def _deferred_mylar_folder_scope_conflict( + base_metadata: SourceMetadata, + loaded_metadata: SourceMetadata, +) -> dict[str, object] | None: + """Detect a foreign embedded series on an unrecorded file in a Mylar folder.""" + raw_scope = base_metadata.diagnostics.get("mylar3_unrecorded_file") + if not isinstance(raw_scope, dict): + return None + signal = loaded_metadata.signals.get("series_name") + if signal not in {MetadataSignal.COMICINFO, MetadataSignal.SIDECAR}: + return None + expected_series = str(raw_scope.get("expected_series") or "").strip() + parsed_series = (loaded_metadata.series_name or "").strip() + if not expected_series or not parsed_series: + return None + if NameMatcher.normalize(expected_series) == NameMatcher.normalize(parsed_series): + return None + return { + "expected_series": expected_series, + "parsed_series": parsed_series, + "recorded_issue": False, + "signal": signal.value, + } + + def source_metadata_for_import_series(imp_series: ImportedSeries) -> SourceMetadata: """Build shared semantic metadata from persisted import series scan fields.""" diagnostics = dict(imp_series.diagnostics or {}) @@ -492,11 +746,15 @@ async def source_metadata_for_matching_series( should_probe_trusted_identity = ( probed_archive_count < trusted_identity_probe_limit and has_deferred_archive_metadata + and not identity_conflicts and ( trusted_series_id is None or ( trusted_series_id_signal - in {MetadataSignal.SIDECAR, MetadataSignal.PULLBOX_FOLDER} + in { + MetadataSignal.SIDECAR, + MetadataSignal.PULLBOX_FOLDER, + } and probed_archive_count == 0 ) ) @@ -758,6 +1016,43 @@ def build_import_metadata_conflict( target_issue_title: str | None, ) -> dict[str, Any] | None: """Explain why import refused an auto-match due to conflicting strong signals.""" + folder_scope_conflict = metadata.diagnostics.get("mylar3_folder_scope_conflict") + if isinstance(folder_scope_conflict, dict): + return { + "kind": "source_scope_review", + "reason": "mylar3_folder_scope_conflict", + "preserve_series_match": True, + "rejection_reason": ( + "This unrecorded file appears to belong to another series in the selected " + "Mylar folder." + ), + **folder_scope_conflict, + "target_series": target_series_title, + "target_series_year": target_series_year, + "target_issue_number": target_issue_number, + "target_issue_cv_id": target_issue_cv_id, + } + raw_identity_conflicts = metadata.diagnostics.get("identity_conflicts") + identity_conflicts = ( + [dict(conflict) for conflict in raw_identity_conflicts if isinstance(conflict, dict)] + if isinstance(raw_identity_conflicts, list) + else [] + ) + if identity_conflicts: + return { + "kind": "metadata_conflict", + "conflict_type": "trusted_source_identity_conflict", + "preserve_series_match": False, + "rejection_reason": "Trusted local metadata contains conflicting ComicVine IDs.", + "source_series": metadata.series_name, + "source_year": metadata.year, + "target_series": target_series_title, + "target_series_year": target_series_year, + "target_issue_number": target_issue_number, + "target_issue_cv_id": target_issue_cv_id, + "identity_conflicts": identity_conflicts, + } + archive_hint = _archive_entry_issue_hint({"source_metadata": metadata.diagnostics}) archive_issue_number = ( normalize_issue_number(archive_hint.get("issue_number")) if archive_hint else None diff --git a/src/pullbox/services/import_split_series.py b/src/pullbox/services/import_split_series.py new file mode 100644 index 00000000..d548c56e --- /dev/null +++ b/src/pullbox/services/import_split_series.py @@ -0,0 +1,380 @@ +"""Review-time detection for one logical series spanning library roots.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +from sqlalchemy import and_, or_, select + +from pullbox.core.exceptions import ValidationError +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportFileHandlingMode, + ImportSeriesStatus, +) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile, LibraryRoot +from pullbox.models.series import Series +from pullbox.services.library_root_management import validate_managed_library_root + +if TYPE_CHECKING: + from typing import Any, Protocol + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.models.import_job import ImportJob, ImportJobAction + + class RecordActionFunc(Protocol): + async def __call__( + self, + session: AsyncSession, + job: ImportJob, + *, + phase: str, + action_type: str, + payload: dict[str, Any], + ) -> ImportJobAction: ... + + +@dataclass(frozen=True, slots=True) +class SplitSeriesReviewItem: + """One canonical series whose selected files span multiple roots.""" + + imported_series_ids: tuple[int, ...] + canonical_series_id: int | None + comicvine_id: int | None + title: str + root_ids: tuple[int, ...] + root_names: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class SplitSeriesReview: + """Selected split-series rows that require one future destination.""" + + items: tuple[SplitSeriesReviewItem, ...] = () + + @property + def requires_preferred_destination(self) -> bool: + return bool(self.items) + + +@dataclass(frozen=True, slots=True) +class _RootBoundary: + root_id: int + name: str + lexical_path: Path + resolved_path: Path + + +@dataclass(slots=True) +class _ReviewAccumulator: + imported_series_ids: set[int] = field(default_factory=set) + canonical_series_id: int | None = None + comicvine_id: int | None = None + title: str = "Series" + root_ids: set[int] = field(default_factory=set) + + +async def load_selected_split_series_review( + session: AsyncSession, + job: ImportJob, +) -> SplitSeriesReview: + """Return selected in-place series that will span more than one root. + + The analysis is read-only. It groups import rows by canonical local series + identity (or ComicVine identity before creation), retains each selected + source file's containing root, and includes roots already used by a matched + local series. Existing paths are never rewritten by this workflow. + """ + if job.file_handling_mode != ImportFileHandlingMode.IN_PLACE: + return SplitSeriesReview() + + boundaries = await _load_root_boundaries(session) + if not boundaries: + return SplitSeriesReview() + + rows = await _load_selected_file_rows(session, int(job.id)) + if not rows: + return SplitSeriesReview() + + comicvine_ids = { + resolved_cv_id + for _item_id, _status, _series_id, cv_id, user_cv_id, _title, _path in rows + if (resolved_cv_id := _resolve_comicvine_id(cv_id, user_cv_id)) is not None + } + existing_series_by_cv = await _load_existing_series_by_comicvine_id( + session, + comicvine_ids, + ) + + accumulators: dict[tuple[str, int], _ReviewAccumulator] = {} + for item_id, _status, series_id, cv_id, user_cv_id, title, file_path in rows: + resolved_cv_id = _resolve_comicvine_id(cv_id, user_cv_id) + canonical_series_id = ( + int(series_id) + if series_id is not None + else existing_series_by_cv.get(resolved_cv_id) + if resolved_cv_id is not None + else None + ) + if canonical_series_id is not None: + key = ("series", canonical_series_id) + elif resolved_cv_id is not None: + key = ("comicvine", resolved_cv_id) + else: + key = ("import", int(item_id)) + accumulator = accumulators.setdefault( + key, + _ReviewAccumulator( + canonical_series_id=canonical_series_id, + comicvine_id=resolved_cv_id, + title=str(title or "Series"), + ), + ) + accumulator.imported_series_ids.add(int(item_id)) + root_id = _containing_root_id(str(file_path), boundaries) + if root_id is not None: + accumulator.root_ids.add(root_id) + + existing_root_ids = await _load_existing_series_root_ids( + session, + { + accumulator.canonical_series_id + for accumulator in accumulators.values() + if accumulator.canonical_series_id is not None + }, + ) + for accumulator in accumulators.values(): + if accumulator.canonical_series_id is not None: + accumulator.root_ids.update( + existing_root_ids.get(accumulator.canonical_series_id, set()) + ) + + root_names = {boundary.root_id: boundary.name for boundary in boundaries} + review_items = [ + SplitSeriesReviewItem( + imported_series_ids=tuple(sorted(accumulator.imported_series_ids)), + canonical_series_id=accumulator.canonical_series_id, + comicvine_id=accumulator.comicvine_id, + title=accumulator.title, + root_ids=tuple(sorted(accumulator.root_ids)), + root_names=tuple( + root_names.get(root_id, f"Library root {root_id}") + for root_id in sorted(accumulator.root_ids) + ), + ) + for accumulator in accumulators.values() + if len(accumulator.root_ids) > 1 + ] + review_items.sort( + key=lambda item: ( + item.title.casefold(), + item.canonical_series_id or 0, + item.comicvine_id or 0, + item.imported_series_ids, + ) + ) + return SplitSeriesReview(items=tuple(review_items)) + + +async def require_preferred_managed_root_for_selected_split_series( + session: AsyncSession, + job: ImportJob, + *, + preferred_library_root_id: int | None, +) -> SplitSeriesReview: + """Require and validate a future root only when selected series are split.""" + review = await load_selected_split_series_review(session, job) + if not review.requires_preferred_destination: + return review + if preferred_library_root_id is None: + raise ValidationError( + "Choose a preferred managed destination for future acquisitions before " + "importing a series that spans multiple library roots. Existing files will " + "remain in place." + ) + root = await session.get(LibraryRoot, preferred_library_root_id) + if root is None: + raise ValidationError("The selected preferred managed destination does not exist.") + await validate_managed_library_root(root) + return review + + +async def apply_import_preferred_series_root( + session: AsyncSession, + job: ImportJob, + *, + series_id: int, + record_action: RecordActionFunc, +) -> bool: + """Persist an explicit in-place future destination without moving files.""" + if ( + job.file_handling_mode != ImportFileHandlingMode.IN_PLACE + or job.target_library_root_id is None + ): + return False + root = await session.get(LibraryRoot, job.target_library_root_id) + if root is None: + raise ValidationError("The selected preferred managed destination does not exist.") + await validate_managed_library_root(root) + series = await session.get(Series, series_id) + if series is None: + raise ValidationError("The imported series no longer exists.") + if series.preferred_library_root_id == root.id: + return False + + old_root_id = series.preferred_library_root_id + series.preferred_library_root_id = root.id + await record_action( + session, + job, + phase="import", + action_type="series_preferred_root_updated", + payload={ + "series_id": int(series.id), + "old_preferred_library_root_id": old_root_id, + "new_preferred_library_root_id": int(root.id), + }, + ) + await session.flush() + return True + + +async def _load_root_boundaries(session: AsyncSession) -> tuple[_RootBoundary, ...]: + roots = list( + ( + await session.execute( + select(LibraryRoot) + .where( + LibraryRoot.enabled.is_(True), + LibraryRoot.allow_referenced_registrations.is_(True), + ) + .order_by(LibraryRoot.id.asc()) + ) + ) + .scalars() + .all() + ) + return tuple( + _RootBoundary( + root_id=int(root.id), + name=root.name, + lexical_path=Path(root.path).expanduser().absolute(), + resolved_path=Path(root.path).expanduser().resolve(strict=False), + ) + for root in roots + ) + + +async def _load_selected_file_rows( + session: AsyncSession, + job_id: int, +) -> list[tuple[int, ImportSeriesStatus, int | None, int | None, int | None, str, str]]: + result = await session.execute( + select( + ImportedSeries.id, + ImportedSeries.status, + ImportedSeries.series_id, + ImportedSeries.cv_id, + ImportedSeries.user_selected_cv_id, + ImportedSeries.raw_series_name, + ImportedFile.file_path, + ) + .join(ImportedFile, ImportedFile.import_series_id == ImportedSeries.id) + .where( + ImportedSeries.import_job_id == job_id, + ImportedFile.status.in_((ImportedFileStatus.MATCHED, ImportedFileStatus.CONFIRMED)), + or_( + and_( + ImportedSeries.status.in_( + (ImportSeriesStatus.MATCHED, ImportSeriesStatus.CONFIRMED) + ), + ImportedSeries.selected_for_import.is_(True), + ), + and_( + ImportedSeries.status == ImportSeriesStatus.DUPLICATE, + ImportedFile.include_in_import.is_(True), + ), + ), + ) + .order_by(ImportedSeries.id.asc(), ImportedFile.id.asc()) + ) + return [ + ( + int(item_id), + status, + int(series_id) if series_id is not None else None, + int(cv_id) if cv_id is not None else None, + int(user_cv_id) if user_cv_id is not None else None, + str(title), + str(file_path), + ) + for item_id, status, series_id, cv_id, user_cv_id, title, file_path in result.all() + ] + + +async def _load_existing_series_by_comicvine_id( + session: AsyncSession, + comicvine_ids: set[int], +) -> dict[int, int]: + if not comicvine_ids: + return {} + result = await session.execute( + select(Series.comicvine_id, Series.id).where(Series.comicvine_id.in_(comicvine_ids)) + ) + return { + int(comicvine_id): int(series_id) + for comicvine_id, series_id in result.all() + if comicvine_id is not None + } + + +async def _load_existing_series_root_ids( + session: AsyncSession, + series_ids: set[int], +) -> dict[int, set[int]]: + if not series_ids: + return {} + result = await session.execute( + select(Issue.series_id, LibraryFile.library_root_id) + .join(LibraryFile, LibraryFile.issue_id == Issue.id) + .where(Issue.series_id.in_(series_ids)) + ) + roots_by_series: dict[int, set[int]] = {} + for series_id, root_id in result.all(): + roots_by_series.setdefault(int(series_id), set()).add(int(root_id)) + return roots_by_series + + +def _containing_root_id( + raw_path: str, + boundaries: tuple[_RootBoundary, ...], +) -> int | None: + path = Path(raw_path).expanduser() + lexical_path = path.absolute() + resolved_path = path.resolve(strict=False) + candidates = [ + boundary + for boundary in boundaries + if ( + lexical_path == boundary.lexical_path + or lexical_path.is_relative_to(boundary.lexical_path) + ) + and ( + resolved_path == boundary.resolved_path + or resolved_path.is_relative_to(boundary.resolved_path) + ) + ] + if len(candidates) != 1: + return None + return candidates[0].root_id + + +def _resolve_comicvine_id(cv_id: int | None, user_cv_id: int | None) -> int | None: + value = user_cv_id if user_cv_id is not None else cv_id + return int(value) if value is not None else None diff --git a/src/pullbox/services/import_story_arc_detection.py b/src/pullbox/services/import_story_arc_detection.py new file mode 100644 index 00000000..5a408adc --- /dev/null +++ b/src/pullbox/services/import_story_arc_detection.py @@ -0,0 +1,189 @@ +"""Conservative, provider-free story-arc detection for folder imports.""" + +from __future__ import annotations + +import enum +import re +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation + +from pullbox.core.story_arc_identity import normalize_story_arc_name + + +class FolderArcClassification(enum.StrEnum): + """Folder-import classification without mutating or matching the library.""" + + NORMAL = "normal" + NORMAL_MIXED_FOLDER = "normal_mixed_folder" + STORY_ARC = "story_arc" + NEEDS_REVIEW = "needs_review" + + +@dataclass(frozen=True, slots=True) +class FolderArcFileEvidence: + """Sanitized identity/order evidence for one file in a candidate folder.""" + + relative_path: str + series: str | None + issue_number: str | None + story_arc: str | None = None + story_arc_number: str | None = None + story_arc_number_source: str | None = None + evidence_complete: bool = True + + +@dataclass(frozen=True, slots=True) +class FolderArcDetection: + """Pure classification result used by import preview and review.""" + + classification: FolderArcClassification + reason: str + proposed_name: str | None + file_count: int + series_count: int + ordered_file_count: int + provider_calls_required: bool = False + + +def detect_folder_story_arc( + *, + folder_label: str, + files: tuple[FolderArcFileEvidence, ...], + confirmed_order_pattern: bool = False, +) -> FolderArcDetection: + """Classify bounded folder evidence without title fuzzing or provider calls.""" + series_keys = { + item.series.strip().casefold() + for item in files + if item.series is not None and item.series.strip() + } + ordered = [ + item + for item in files + if item.story_arc_number is not None and item.story_arc_number.strip() + ] + named = [item for item in files if item.story_arc is not None and item.story_arc.strip()] + + normalized_names: dict[str, str] = {} + for item in named: + assert item.story_arc is not None + display_name = _collapse_whitespace(item.story_arc) + normalized_names.setdefault(normalize_story_arc_name(display_name), display_name) + + if len(normalized_names) > 1: + return _result( + FolderArcClassification.NEEDS_REVIEW, + "conflicting_exact_arc_names", + None, + files, + series_keys, + ordered, + ) + + ordered_mixed = len(series_keys) > 1 and len(ordered) == len(files) and bool(files) + has_strong_arc_evidence = bool(named) or ordered_mixed + + if any(not item.evidence_complete for item in files) and has_strong_arc_evidence: + return _result( + FolderArcClassification.NEEDS_REVIEW, + "incomplete_arc_evidence", + next(iter(normalized_names.values()), _clean_folder_label(folder_label)), + files, + series_keys, + ordered, + ) + + order_keys = [_order_key(item.story_arc_number or "") for item in ordered] + if has_strong_arc_evidence and len(order_keys) != len(set(order_keys)): + return _result( + FolderArcClassification.NEEDS_REVIEW, + "duplicate_arc_order", + next(iter(normalized_names.values()), _clean_folder_label(folder_label)), + files, + series_keys, + ordered, + ) + + if normalized_names: + return _result( + FolderArcClassification.STORY_ARC, + "consistent_exact_arc_name", + next(iter(normalized_names.values())), + files, + series_keys, + ordered, + ) + + if ordered_mixed and confirmed_order_pattern: + return _result( + FolderArcClassification.STORY_ARC, + "confirmed_ordered_mixed_folder", + _clean_folder_label(folder_label), + files, + series_keys, + ordered, + ) + if ordered_mixed: + return _result( + FolderArcClassification.NEEDS_REVIEW, + "ordered_mixed_folder_requires_confirmation", + _clean_folder_label(folder_label), + files, + series_keys, + ordered, + ) + if len(series_keys) > 1: + return _result( + FolderArcClassification.NORMAL_MIXED_FOLDER, + "mixed_series_without_strong_arc_evidence", + None, + files, + series_keys, + ordered, + ) + return _result( + FolderArcClassification.NORMAL, + "no_strong_arc_evidence", + None, + files, + series_keys, + ordered, + ) + + +def _result( + classification: FolderArcClassification, + reason: str, + proposed_name: str | None, + files: tuple[FolderArcFileEvidence, ...], + series_keys: set[str], + ordered: list[FolderArcFileEvidence], +) -> FolderArcDetection: + return FolderArcDetection( + classification=classification, + reason=reason, + proposed_name=proposed_name, + file_count=len(files), + series_count=len(series_keys), + ordered_file_count=len(ordered), + ) + + +def _clean_folder_label(value: str) -> str: + cleaned = _collapse_whitespace(value) + return cleaned or "Story Arc" + + +def _collapse_whitespace(value: str) -> str: + return re.sub(r"\s+", " ", value).strip() + + +def _order_key(value: str) -> tuple[str, str]: + normalized = value.strip() + try: + decimal = Decimal(normalized) + except InvalidOperation: + return ("text", normalized.casefold()) + if not decimal.is_finite(): + return ("text", normalized.casefold()) + return ("number", format(decimal.normalize(), "f")) diff --git a/src/pullbox/services/import_story_arc_materialization.py b/src/pullbox/services/import_story_arc_materialization.py new file mode 100644 index 00000000..14cf9b95 --- /dev/null +++ b/src/pullbox/services/import_story_arc_materialization.py @@ -0,0 +1,2431 @@ +"""Materialize confirmed story-arc staging rows into the logical domain.""" + +from __future__ import annotations + +import os +import stat +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath +from typing import TYPE_CHECKING, Literal, cast + +from sqlalchemy import func, or_, select, tuple_ +from sqlalchemy.orm import selectinload + +from pullbox.core.issue_numbers import normalize_issue_number_text +from pullbox.core.story_arc_naming import ( + validate_story_arc_file_template, + validate_story_arc_folder_template, +) +from pullbox.models.import_job import ( + ImportedFileStatus, + ImportJob, + ImportJobStatus, + ImportSourceType, +) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile, LibraryRoot +from pullbox.models.story_arc import ( + ImportedStoryArcStatus, + IssueStoryArc, + StoryArc, + StoryArcExternalIdentity, + StoryArcLifecycle, + StoryArcPlacement, + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcPlacementState, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.services.story_arc_service import StoryArcService, StoryArcServiceError +from pullbox.services.story_arc_sync_queue import ( + MAX_IMPORT_STORY_ARC_SYNC_ENQUEUE_BATCH_SIZE, + ImportStoryArcSyncProposal, + StoryArcImportSyncEnqueueResult, + enqueue_import_story_arc_sync_work_batch, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + + from pullbox.models.import_job import ImportJobAction + from pullbox.services.import_job_actions import ImportJobActionSpec + from pullbox.services.import_job_execution_types import RecordActionFunc, RecordActionsFunc + + +CancellationCheck = Callable[[], Awaitable[None]] +DurableCheckpoint = Callable[[], Awaitable[None]] +CountField = Literal[ + "arcs_created", + "arcs_merged", + "arcs_reused", + "arcs_failed", + "external_identities_created", + "external_identities_reused", + "memberships_created", + "memberships_reused", + "resolved_entries", + "unresolved_entries", + "entries_skipped", + "managed_placements_queued", + "managed_placements_reused", +] + +_POLICY_FLAGS = ("monitored", "search_missing", "include_upcoming", "sync_enabled") +_CONFIRMED_POLICY_KEYS = frozenset( + { + "schema_version", + "source", + "activation", + "monitored", + "search_missing", + "include_upcoming", + "sync_enabled", + "placement_policy", + } +) +_PLACEMENT_POLICY_KEYS = frozenset( + { + "schema_version", + "mode", + "target_library_root_id", + "destination_root", + "folder_template", + "file_template", + "symlink_style", + "synchronize", + } +) +_PLACEMENT_POLICY_MODES = frozenset({"logical", "reference_only", "copy", "hardlink", "symlink"}) +_SAFE_IMPORTED_FILE_STATES = frozenset( + {ImportedFileStatus.IMPORTED, ImportedFileStatus.ALREADY_OWNED} +) +_UNRESOLVED_STATES = frozenset( + { + StoryArcResolutionState.PENDING, + StoryArcResolutionState.MISSING, + StoryArcResolutionState.AMBIGUOUS, + StoryArcResolutionState.CONFLICT, + StoryArcResolutionState.SKIPPED, + } +) + + +@dataclass(frozen=True, slots=True) +class StoryArcMaterializationWarning: + """Sanitized warning tied to durable import staging identity.""" + + code: str + imported_story_arc_id: int + imported_story_arc_entry_id: int | None = None + + +@dataclass(frozen=True, slots=True) +class StoryArcMaterializationResult: + """Durable logical-materialization counters for progress and diagnostics.""" + + arcs_examined: int = 0 + arcs_created: int = 0 + arcs_merged: int = 0 + arcs_reused: int = 0 + arcs_failed: int = 0 + external_identities_created: int = 0 + external_identities_reused: int = 0 + memberships_created: int = 0 + memberships_reused: int = 0 + resolved_entries: int = 0 + unresolved_entries: int = 0 + entries_skipped: int = 0 + managed_placements_queued: int = 0 + managed_placements_reused: int = 0 + warnings: tuple[StoryArcMaterializationWarning, ...] = () + + +@dataclass(frozen=True, slots=True) +class _ValidatedPolicy: + activated: bool + snapshot: dict[str, object] + monitored: bool = False + search_missing: bool = False + include_upcoming: bool = False + sync_enabled: bool = False + target_library_root_id: int | None = None + warning_code: str | None = None + + +@dataclass(slots=True) +class _MutableCounts: + arcs_examined: int = 0 + arcs_created: int = 0 + arcs_merged: int = 0 + arcs_reused: int = 0 + arcs_failed: int = 0 + external_identities_created: int = 0 + external_identities_reused: int = 0 + memberships_created: int = 0 + memberships_reused: int = 0 + resolved_entries: int = 0 + unresolved_entries: int = 0 + entries_skipped: int = 0 + managed_placements_queued: int = 0 + managed_placements_reused: int = 0 + + def freeze( + self, + warnings: Sequence[StoryArcMaterializationWarning], + ) -> StoryArcMaterializationResult: + return StoryArcMaterializationResult( + arcs_examined=self.arcs_examined, + arcs_created=self.arcs_created, + arcs_merged=self.arcs_merged, + arcs_reused=self.arcs_reused, + arcs_failed=self.arcs_failed, + external_identities_created=self.external_identities_created, + external_identities_reused=self.external_identities_reused, + memberships_created=self.memberships_created, + memberships_reused=self.memberships_reused, + resolved_entries=self.resolved_entries, + unresolved_entries=self.unresolved_entries, + entries_skipped=self.entries_skipped, + managed_placements_queued=self.managed_placements_queued, + managed_placements_reused=self.managed_placements_reused, + warnings=tuple(warnings), + ) + + +@dataclass(slots=True) +class _MaterializationState: + arcs_by_id: dict[int, StoryArc] + job_arc_ids_by_import_identity: dict[ + tuple[StoryArcSourceKind, str], + int | None, + ] + identities_by_key: dict[tuple[str, str, str], int] + loaded_identity_keys: set[tuple[str, str, str]] + + +@dataclass(slots=True) +class _MaterializationBatch: + state: _MaterializationState + library_root_ids: set[int] + identity_evidence: dict[ + int, + tuple[list[tuple[str, str, str]], str | None], + ] + + +@dataclass(slots=True) +class _ArcMaterializationContext: + staged_arc: ImportedStoryArc + arc: StoryArc + policy: _ValidatedPolicy + counts: _MutableCounts + + +@dataclass(frozen=True, slots=True) +class _ManagedPlacementRequest: + proposal: ImportStoryArcSyncProposal + context: _ArcMaterializationContext + + +@dataclass(slots=True) +class _EntryPageLookups: + issues_by_id: dict[int, Issue] + library_files_by_issue_id: dict[int, LibraryFile] + memberships_by_id: dict[int, IssueStoryArc] + memberships_by_arc_issue: dict[tuple[int, int], IssueStoryArc] + memberships_by_arc_source: dict[ + tuple[int, StoryArcSourceKind, str], + IssueStoryArc, + ] + reference_candidates_by_entry_id: dict[int, _ReferenceCandidateResolution] + placements_by_path: dict[str, StoryArcPlacement] + placements_by_membership: dict[int, list[StoryArcPlacement]] + + +@dataclass(frozen=True, slots=True) +class _ReferencePathCandidate: + path: Path + trusted_root: Path + + +@dataclass(frozen=True, slots=True) +class _ReferenceCandidateResolution: + candidate: _ReferencePathCandidate | None = None + warning_code: str | None = None + + +@dataclass(frozen=True, slots=True) +class _ReferencePathInspection: + fingerprint: dict[str, object] | None = None + warning_code: str | None = None + + +@dataclass(slots=True) +class _ReferencePageCheckpoint: + cancellation_check: CancellationCheck | None + checked: bool = False + + async def ensure_checked(self) -> None: + """Check cancellation once before a bounded page touches reference evidence.""" + if self.checked: + return + await _checkpoint(self.cancellation_check) + self.checked = True + + +async def materialize_confirmed_story_arcs( + session: AsyncSession, + *, + import_job_id: int, + batch_size: int = 100, + entry_checkpoint_size: int = 250, + cancellation_check: CancellationCheck | None = None, + durable_checkpoint: DurableCheckpoint | None = None, + record_action: RecordActionFunc | None = None, + record_actions: RecordActionsFunc | None = None, +) -> StoryArcMaterializationResult: + """Create or explicitly merge confirmed logical story arcs for one job. + + This Step 4 service consumes database staging and canonical issue rows. It + never invokes a provider, opens archive content, or mutates a source + artifact. Confirmed existing Mylar/folder artifacts may be attached as + referenced placements after metadata-only, no-follow root validation. The + runtime supplies a durable checkpoint that commits each flushed entry/arc + page and rechecks job control before later pages. A caller that omits it + retains ownership of the surrounding transaction. + """ + if isinstance(batch_size, bool) or batch_size <= 0: + raise ValueError("Story-arc materialization batch size must be positive") + if isinstance(entry_checkpoint_size, bool) or entry_checkpoint_size <= 0: + raise ValueError("Story-arc entry checkpoint size must be positive") + + counts = _MutableCounts() + warnings: list[StoryArcMaterializationWarning] = [] + job = await session.get(ImportJob, import_job_id) + if job is None: + raise ValueError(f"Import job {import_job_id} was not found") + progress_snapshot = dict(job.progress_snapshot or {}) + progress_snapshot.update( + { + "status": ImportJobStatus.IMPORTING.value, + "mode": "import", + "phase": "story_arcs", + } + ) + job.progress_snapshot = progress_snapshot + await _checkpoint(cancellation_check) + state = _MaterializationState( + arcs_by_id={}, + job_arc_ids_by_import_identity={}, + identities_by_key={}, + loaded_identity_keys=set(), + ) + await _load_job_arc_recovery_index( + session, + import_job_id=import_job_id, + batch_size=batch_size, + state=state, + cancellation_check=cancellation_check, + ) + last_id = 0 + + while True: + staged_arcs = list( + ( + await session.scalars( + select(ImportedStoryArc) + .where( + ImportedStoryArc.import_job_id == import_job_id, + ImportedStoryArc.status == ImportedStoryArcStatus.CONFIRMED, + ImportedStoryArc.selected_for_import.is_(True), + ImportedStoryArc.id > last_id, + ) + .order_by(ImportedStoryArc.id) + .limit(batch_size) + ) + ).all() + ) + if not staged_arcs: + break + + await _checkpoint(cancellation_check) + batch_warning_start = len(warnings) + identity_evidence = await _load_batch_external_identity_evidence( + session, + staged_arcs=staged_arcs, + entry_page_size=entry_checkpoint_size, + cancellation_check=cancellation_check, + ) + batch = await _prepare_materialization_batch( + session, + import_job_id=import_job_id, + staged_arcs=staged_arcs, + state=state, + identity_evidence=identity_evidence, + ) + contexts: dict[int, _ArcMaterializationContext] = {} + for staged_arc in staged_arcs: + counts.arcs_examined += 1 + context = await _materialize_one_arc( + session, + import_job_id=import_job_id, + job=job, + staged_arc=staged_arc, + counts=counts, + warnings=warnings, + record_action=record_action, + batch=batch, + ) + if context is not None: + contexts[int(staged_arc.id)] = context + + await _materialize_entry_pages( + session, + import_job_id=import_job_id, + job=job, + contexts=contexts, + counts=counts, + warnings=warnings, + record_action=record_action, + record_actions=record_actions, + entry_page_size=entry_checkpoint_size, + cancellation_check=cancellation_check, + durable_checkpoint=durable_checkpoint, + ) + batch_warning_codes = _warning_codes_by_arc(warnings[batch_warning_start:]) + for context in contexts.values(): + context.staged_arc.status = ImportedStoryArcStatus.IMPORTED + _persist_arc_materialization_diagnostics( + context.staged_arc, + story_arc_id=int(context.arc.id), + status="imported", + counts=_count_snapshot(context.counts), + warning_codes=batch_warning_codes.get(int(context.staged_arc.id), []), + ) + await session.flush() + if durable_checkpoint is not None: + await durable_checkpoint() + last_id = int(staged_arcs[-1].id) + + return counts.freeze(warnings) + + +async def _load_job_arc_recovery_index( + session: AsyncSession, + *, + import_job_id: int, + batch_size: int, + state: _MaterializationState, + cancellation_check: CancellationCheck | None, +) -> None: + """Build an O(1) restart index from bounded scalar-only pages.""" + last_id = 0 + while True: + rows = ( + await session.execute( + select( + StoryArc.id, + StoryArc.source_kind, + StoryArc.source_import_job_id, + StoryArc.diagnostics, + ) + .where( + StoryArc.source_import_job_id == import_job_id, + StoryArc.id > last_id, + ) + .order_by(StoryArc.id) + .limit(batch_size) + ) + ).all() + if not rows: + return + for arc_id, source_kind, source_import_job_id, diagnostics in rows: + _index_job_arc_values( + state, + arc_id=int(arc_id), + source_kind=source_kind, + source_import_job_id=int(source_import_job_id), + diagnostics=diagnostics, + ) + last_id = int(rows[-1].id) + await _checkpoint(cancellation_check) + + +async def _load_batch_external_identity_evidence( + session: AsyncSession, + *, + staged_arcs: Sequence[ImportedStoryArc], + entry_page_size: int, + cancellation_check: CancellationCheck | None, +) -> dict[int, tuple[list[tuple[str, str, str]], str | None]]: + """Scan provider-neutral entry evidence without materializing entry ORM rows.""" + staged_arc_ids = [int(staged_arc.id) for staged_arc in staged_arcs] + comicvine_ids: dict[int, set[str]] = {arc_id: set() for arc_id in staged_arc_ids} + row_count = 0 + result = await session.stream( + select( + ImportedStoryArcEntry.imported_story_arc_id, + ImportedStoryArcEntry.evidence, + ) + .where( + ImportedStoryArcEntry.imported_story_arc_id.in_(staged_arc_ids), + ImportedStoryArcEntry.selected_for_import.is_(True), + ) + .order_by(ImportedStoryArcEntry.id) + .execution_options(yield_per=entry_page_size) + ) + try: + async for imported_story_arc_id, evidence in result: + values = comicvine_ids[int(imported_story_arc_id)] + cv_arc_id = _mapping(evidence).get("cv_arc_id") + if cv_arc_id is not None and str(cv_arc_id).strip() and len(values) < 2: + values.add(str(cv_arc_id)) + row_count += 1 + if row_count % entry_page_size == 0: + await _checkpoint(cancellation_check) + finally: + await result.close() + + loaded: dict[int, tuple[list[tuple[str, str, str]], str | None]] = {} + for staged_arc in staged_arcs: + identities: list[tuple[str, str, str]] = [] + if staged_arc.source_arc_id: + identities.append((staged_arc.source_kind.value, "story_arc", staged_arc.source_arc_id)) + values = comicvine_ids[int(staged_arc.id)] + warning_code = None + if len(values) == 1: + identities.append(("comicvine", "story_arc", next(iter(values)))) + elif len(values) > 1: + warning_code = "conflicting_external_identity_evidence" + loaded[int(staged_arc.id)] = (identities, warning_code) + return loaded + + +async def _prepare_materialization_batch( + session: AsyncSession, + *, + import_job_id: int, + staged_arcs: Sequence[ImportedStoryArc], + state: _MaterializationState, + identity_evidence: dict[ + int, + tuple[list[tuple[str, str, str]], str | None], + ], +) -> _MaterializationBatch: + """Prefetch all canonical lookups needed by one bounded staging page.""" + state.arcs_by_id.clear() + identity_keys: set[tuple[str, str, str]] = set() + target_arc_ids: set[int] = set() + library_root_ids: set[int] = set() + + for staged_arc in staged_arcs: + identities, _warning = identity_evidence[int(staged_arc.id)] + identity_keys.update(identities) + if staged_arc.materialized_story_arc_id is not None: + target_arc_ids.add(int(staged_arc.materialized_story_arc_id)) + if staged_arc.proposed_story_arc_id is not None: + target_arc_ids.add(int(staged_arc.proposed_story_arc_id)) + raw_policy = staged_arc.proposed_policy_snapshot + if isinstance(raw_policy, dict): + placement_policy = raw_policy.get("placement_policy") + target_root_id = ( + placement_policy.get("target_library_root_id") + if isinstance(placement_policy, dict) + else None + ) + if ( + isinstance(target_root_id, int) + and not isinstance(target_root_id, bool) + and target_root_id > 0 + ): + library_root_ids.add(target_root_id) + + missing_identity_keys = identity_keys - state.loaded_identity_keys + if missing_identity_keys: + loaded_identities = list( + ( + await session.scalars( + select(StoryArcExternalIdentity).where( + tuple_( + StoryArcExternalIdentity.source, + StoryArcExternalIdentity.namespace, + StoryArcExternalIdentity.external_id, + ).in_(missing_identity_keys) + ) + ) + ).all() + ) + for loaded_identity in loaded_identities: + state.identities_by_key[_identity_key(loaded_identity)] = int( + loaded_identity.story_arc_id + ) + state.loaded_identity_keys.update(missing_identity_keys) + for key in identity_keys: + identity_arc_id = state.identities_by_key.get(key) + if identity_arc_id is not None: + target_arc_ids.add(identity_arc_id) + + for staged_arc in staged_arcs: + recovered_arc_id = state.job_arc_ids_by_import_identity.get( + (staged_arc.source_kind, staged_arc.source_key) + ) + if recovered_arc_id is not None: + target_arc_ids.add(recovered_arc_id) + + missing_arc_ids = target_arc_ids - state.arcs_by_id.keys() + if missing_arc_ids: + arcs = list( + (await session.scalars(select(StoryArc).where(StoryArc.id.in_(missing_arc_ids)))).all() + ) + for arc in arcs: + state.arcs_by_id[int(arc.id)] = arc + if arc.source_import_job_id == import_job_id: + _index_job_arc(state, arc) + + existing_root_ids = ( + { + int(root_id) + for root_id in ( + await session.scalars( + select(LibraryRoot.id).where( + LibraryRoot.id.in_(library_root_ids), + LibraryRoot.enabled.is_(True), + ) + ) + ).all() + } + if library_root_ids + else set() + ) + return _MaterializationBatch( + state=state, + library_root_ids=existing_root_ids, + identity_evidence=identity_evidence, + ) + + +async def _materialize_one_arc( + session: AsyncSession, + *, + import_job_id: int, + job: ImportJob, + staged_arc: ImportedStoryArc, + counts: _MutableCounts, + warnings: list[StoryArcMaterializationWarning], + record_action: RecordActionFunc | None, + batch: _MaterializationBatch, +) -> _ArcMaterializationContext | None: + arc_warning_start = len(warnings) + arc_counts = _MutableCounts() + policy = _validate_policy(staged_arc, library_root_ids=batch.library_root_ids) + if policy.warning_code is not None: + _warn(warnings, policy.warning_code, staged_arc) + + identities, identity_warning = batch.identity_evidence[int(staged_arc.id)] + if identity_warning is not None: + _warn(warnings, identity_warning, staged_arc) + _increment_counts(counts, arc_counts, "arcs_failed") + staged_arc.status = ImportedStoryArcStatus.FAILED + _persist_arc_materialization_diagnostics( + staged_arc, + story_arc_id=None, + status="failed", + counts=_count_snapshot(arc_counts), + warning_codes=[warning.code for warning in warnings[arc_warning_start:]], + ) + return None + + arc, outcome, failure_code = await _resolve_or_create_arc( + session, + import_job_id=import_job_id, + staged_arc=staged_arc, + policy=policy, + identities=identities, + batch=batch, + ) + if arc is None: + _increment_counts(counts, arc_counts, "arcs_failed") + staged_arc.status = ImportedStoryArcStatus.FAILED + _warn(warnings, failure_code or "canonical_story_arc_unavailable", staged_arc) + _persist_arc_materialization_diagnostics( + staged_arc, + story_arc_id=None, + status="failed", + counts=_count_snapshot(arc_counts), + warning_codes=[warning.code for warning in warnings[arc_warning_start:]], + ) + return None + + if outcome == "created": + _increment_counts(counts, arc_counts, "arcs_created") + elif outcome == "merged": + _increment_counts(counts, arc_counts, "arcs_merged") + else: + _increment_counts(counts, arc_counts, "arcs_reused") + + policy_before = _story_arc_policy_state(arc) + if policy.activated: + policy_changed = _apply_policy( + arc, + policy, + increment_revision=outcome != "created", + ) + else: + policy_changed = False + staged_arc.materialized_story_arc_id = arc.id + + if outcome == "created" and record_action is not None: + await record_action( + session, + job, + phase="story_arcs", + action_type="story_arc_created", + payload={ + "story_arc_id": int(arc.id), + "imported_story_arc_id": int(staged_arc.id), + "expected_after": _story_arc_created_state(arc), + }, + ) + elif policy_changed and record_action is not None: + await session.flush() + await record_action( + session, + job, + phase="story_arcs", + action_type="story_arc_policy_updated", + payload={ + "story_arc_id": int(arc.id), + "imported_story_arc_id": int(staged_arc.id), + "restore_before": policy_before, + "expected_after": _story_arc_policy_state(arc), + }, + ) + + for identity in identities: + identity_outcome, materialized_identity = await _materialize_external_identity( + session, + arc=arc, + staged_arc=staged_arc, + identity=identity, + state=batch.state, + ) + if identity_outcome == "created": + _increment_counts(counts, arc_counts, "external_identities_created") + if record_action is not None and materialized_identity is not None: + await record_action( + session, + job, + phase="story_arcs", + action_type="story_arc_external_identity_created", + payload={ + "external_identity_id": int(materialized_identity.id), + "story_arc_id": int(arc.id), + "imported_story_arc_id": int(staged_arc.id), + "expected_after": _external_identity_state(materialized_identity), + }, + ) + elif identity_outcome == "reused": + _increment_counts(counts, arc_counts, "external_identities_reused") + else: + _warn(warnings, "external_identity_conflict", staged_arc) + + return _ArcMaterializationContext( + staged_arc=staged_arc, + arc=arc, + policy=policy, + counts=arc_counts, + ) + + +async def _resolve_or_create_arc( + session: AsyncSession, + *, + import_job_id: int, + staged_arc: ImportedStoryArc, + policy: _ValidatedPolicy, + identities: Sequence[tuple[str, str, str]], + batch: _MaterializationBatch, +) -> tuple[StoryArc | None, str | None, str | None]: + if staged_arc.materialized_story_arc_id is not None: + materialized = batch.state.arcs_by_id.get(int(staged_arc.materialized_story_arc_id)) + if materialized is not None: + return materialized, "reused", None + + if staged_arc.proposed_story_arc_id is not None: + proposed = batch.state.arcs_by_id.get(int(staged_arc.proposed_story_arc_id)) + if proposed is None: + return None, None, "explicit_merge_target_missing" + if proposed.lifecycle == StoryArcLifecycle.ARCHIVED: + return None, None, "explicit_merge_target_archived" + if _identity_targets_another_arc(batch.state, identities, int(proposed.id)): + return None, None, "external_identity_requires_explicit_merge_review" + return proposed, "merged", None + + recovered = _recover_arc_created_by_this_job( + batch.state, + import_job_id=import_job_id, + staged_arc=staged_arc, + identities=identities, + ) + if recovered is not None: + return recovered, "reused", None + if _any_external_identity_exists(batch.state, identities): + return None, None, "external_identity_requires_explicit_merge_review" + if staged_arc.name is None or not staged_arc.name.strip(): + return None, None, "canonical_story_arc_name_missing" + + service = StoryArcService() + try: + arc = await service.create( + session, + name=staged_arc.name, + description=staged_arc.description, + monitored=policy.monitored if policy.activated else False, + search_missing=policy.search_missing if policy.activated else False, + include_upcoming=policy.include_upcoming if policy.activated else False, + sync_enabled=policy.sync_enabled if policy.activated else False, + source_kind=staged_arc.source_kind, + ) + except (StoryArcServiceError, ValueError): + return None, None, "canonical_story_arc_validation_failed" + arc.source_import_job_id = import_job_id + arc.diagnostics = { + "schema_version": 1, + "import_identity": { + "import_job_id": import_job_id, + "source_key": staged_arc.source_key, + }, + } + if policy.activated: + _apply_policy(arc, policy, increment_revision=False) + await session.flush() + batch.state.arcs_by_id[int(arc.id)] = arc + _index_job_arc(batch.state, arc) + return arc, "created", None + + +def _recover_arc_created_by_this_job( + state: _MaterializationState, + *, + import_job_id: int, + staged_arc: ImportedStoryArc, + identities: Sequence[tuple[str, str, str]], +) -> StoryArc | None: + identity_arc_ids = _external_identity_arc_ids(state, identities) + if len(identity_arc_ids) == 1: + arc = state.arcs_by_id.get(next(iter(identity_arc_ids))) + if arc is not None and _arc_matches_import_identity(arc, import_job_id, staged_arc): + return arc + + recovered_arc_id = state.job_arc_ids_by_import_identity.get( + (staged_arc.source_kind, staged_arc.source_key) + ) + return state.arcs_by_id.get(recovered_arc_id) if recovered_arc_id is not None else None + + +def _arc_matches_import_identity( + arc: StoryArc, + import_job_id: int, + staged_arc: ImportedStoryArc, +) -> bool: + identity = _mapping(_mapping(arc.diagnostics).get("import_identity")) + return ( + arc.source_import_job_id == import_job_id + and identity.get("import_job_id") == import_job_id + and identity.get("source_key") == staged_arc.source_key + ) + + +def _index_job_arc(state: _MaterializationState, arc: StoryArc) -> None: + _index_job_arc_values( + state, + arc_id=int(arc.id), + source_kind=arc.source_kind, + source_import_job_id=arc.source_import_job_id, + diagnostics=arc.diagnostics, + ) + + +def _index_job_arc_values( + state: _MaterializationState, + *, + arc_id: int, + source_kind: StoryArcSourceKind, + source_import_job_id: int | None, + diagnostics: object, +) -> None: + identity = _mapping(_mapping(diagnostics).get("import_identity")) + source_key = identity.get("source_key") + if ( + not isinstance(source_key, str) + or not source_key + or identity.get("import_job_id") != source_import_job_id + ): + return + key = (source_kind, source_key) + if key not in state.job_arc_ids_by_import_identity: + state.job_arc_ids_by_import_identity[key] = arc_id + return + existing_arc_id = state.job_arc_ids_by_import_identity[key] + if existing_arc_id is None or existing_arc_id != arc_id: + state.job_arc_ids_by_import_identity[key] = None + + +async def _materialize_entry_pages( + session: AsyncSession, + *, + import_job_id: int, + job: ImportJob, + contexts: Mapping[int, _ArcMaterializationContext], + counts: _MutableCounts, + warnings: list[StoryArcMaterializationWarning], + record_action: RecordActionFunc | None, + record_actions: RecordActionsFunc | None, + entry_page_size: int, + cancellation_check: CancellationCheck | None, + durable_checkpoint: DurableCheckpoint | None, +) -> None: + if not contexts: + return + staged_arc_ids = list(contexts) + last_entry_id = 0 + while True: + await _checkpoint(cancellation_check) + selected_entries = list( + ( + await session.scalars( + select(ImportedStoryArcEntry) + .where( + ImportedStoryArcEntry.imported_story_arc_id.in_(staged_arc_ids), + ImportedStoryArcEntry.selected_for_import.is_(True), + ImportedStoryArcEntry.id > last_entry_id, + ) + .options(selectinload(ImportedStoryArcEntry.import_file)) + .order_by(ImportedStoryArcEntry.id) + .limit(entry_page_size) + ) + ).all() + ) + if not selected_entries: + return + lookups = await _prepare_entry_page_lookups( + session, + import_job_id=import_job_id, + job=job, + contexts=contexts, + entries=selected_entries, + ) + reference_checkpoint = _ReferencePageCheckpoint(cancellation_check) + managed_placement_requests: list[_ManagedPlacementRequest] = [] + for entry in selected_entries: + context = contexts[int(entry.imported_story_arc_id)] + await _materialize_entry( + session, + import_job_id=import_job_id, + job=job, + context=context, + entry=entry, + counts=counts, + warnings=warnings, + record_action=record_action, + lookups=lookups, + reference_checkpoint=reference_checkpoint, + managed_placement_requests=managed_placement_requests, + managed_placement_journal_available=( + record_actions is not None or record_action is not None + ), + ) + await session.flush() + await _enqueue_managed_placement_requests( + session, + job=job, + requests=managed_placement_requests, + counts=counts, + record_action=record_action, + record_actions=record_actions, + cancellation_check=cancellation_check, + durable_checkpoint=durable_checkpoint, + ) + if not managed_placement_requests and durable_checkpoint is not None: + await durable_checkpoint() + last_entry_id = int(selected_entries[-1].id) + + +async def _prepare_entry_page_lookups( + session: AsyncSession, + *, + import_job_id: int, + job: ImportJob, + contexts: Mapping[int, _ArcMaterializationContext], + entries: Sequence[ImportedStoryArcEntry], +) -> _EntryPageLookups: + issue_ids = _candidate_issue_ids(entries, import_job_id=import_job_id) + issues = ( + { + int(issue.id): issue + for issue in (await session.scalars(select(Issue).where(Issue.id.in_(issue_ids)))).all() + } + if issue_ids + else {} + ) + canonical_library_files = ( + select( + LibraryFile.issue_id.label("issue_id"), + func.min(LibraryFile.id).label("library_file_id"), + ) + .where(LibraryFile.issue_id.in_(issue_ids)) + .group_by(LibraryFile.issue_id) + .subquery() + ) + library_files_by_issue_id = ( + { + int(library_file.issue_id): library_file + for library_file in ( + await session.scalars( + select(LibraryFile) + .join( + canonical_library_files, + LibraryFile.id == canonical_library_files.c.library_file_id, + ) + .order_by(LibraryFile.issue_id) + ) + ).all() + if library_file.issue_id is not None + } + if issue_ids + else {} + ) + pointer_ids = { + int(entry.materialized_membership_id) + for entry in entries + if entry.materialized_membership_id is not None + } + arc_issue_pairs: set[tuple[int, int]] = set() + arc_source_pairs: set[tuple[int, StoryArcSourceKind, str]] = set() + for entry in entries: + arc_id = int(contexts[int(entry.imported_story_arc_id)].arc.id) + for issue_id in _candidate_issue_ids((entry,), import_job_id=import_job_id): + arc_issue_pairs.add((arc_id, issue_id)) + if entry.source_entry_id is not None: + arc_source_pairs.add((arc_id, entry.source_kind, entry.source_entry_id)) + + filters: list[ColumnElement[bool]] = [] + if pointer_ids: + filters.append(IssueStoryArc.id.in_(pointer_ids)) + if arc_issue_pairs: + filters.append( + tuple_(IssueStoryArc.story_arc_id, IssueStoryArc.issue_id).in_(arc_issue_pairs) + ) + if arc_source_pairs: + filters.append( + tuple_( + IssueStoryArc.story_arc_id, + IssueStoryArc.source_kind, + IssueStoryArc.source_entry_id, + ).in_(arc_source_pairs) + ) + memberships = ( + list( + ( + await session.scalars( + select(IssueStoryArc).where(or_(*filters)).order_by(IssueStoryArc.id) + ) + ).all() + ) + if filters + else [] + ) + reference_candidates = { + int(entry.id): _resolve_reference_candidate(job, entry) + for entry in entries + if entry.source_location is not None + } + candidate_paths = { + str(resolution.candidate.path) + for resolution in reference_candidates.values() + if resolution.candidate is not None + } + membership_ids = {int(membership.id) for membership in memberships} + placement_filters: list[ColumnElement[bool]] = [] + if candidate_paths: + placement_filters.append(StoryArcPlacement.placement_path.in_(candidate_paths)) + if membership_ids: + placement_filters.append(StoryArcPlacement.issue_story_arc_id.in_(membership_ids)) + placements = ( + list( + ( + await session.scalars( + select(StoryArcPlacement) + .where(or_(*placement_filters)) + .order_by(StoryArcPlacement.id) + ) + ).all() + ) + if placement_filters + else [] + ) + placements_by_membership: dict[int, list[StoryArcPlacement]] = {} + for placement in placements: + placements_by_membership.setdefault(int(placement.issue_story_arc_id), []).append(placement) + return _EntryPageLookups( + issues_by_id=issues, + library_files_by_issue_id=library_files_by_issue_id, + memberships_by_id={int(membership.id): membership for membership in memberships}, + memberships_by_arc_issue={ + (int(membership.story_arc_id), int(membership.issue_id)): membership + for membership in memberships + if membership.issue_id is not None + }, + memberships_by_arc_source={ + ( + int(membership.story_arc_id), + membership.source_kind, + membership.source_entry_id, + ): membership + for membership in memberships + if membership.source_entry_id is not None + }, + reference_candidates_by_entry_id=reference_candidates, + placements_by_path={placement.placement_path: placement for placement in placements}, + placements_by_membership=placements_by_membership, + ) + + +async def _materialize_entry( + session: AsyncSession, + *, + import_job_id: int, + job: ImportJob, + context: _ArcMaterializationContext, + entry: ImportedStoryArcEntry, + counts: _MutableCounts, + warnings: list[StoryArcMaterializationWarning], + record_action: RecordActionFunc | None, + lookups: _EntryPageLookups, + reference_checkpoint: _ReferencePageCheckpoint, + managed_placement_requests: list[_ManagedPlacementRequest], + managed_placement_journal_available: bool, +) -> None: + staged_arc = context.staged_arc + arc = context.arc + entry_warning_start = len(warnings) + issue_id, issue_warning = _resolved_issue_id( + entry, + issues=lookups.issues_by_id, + import_job_id=import_job_id, + ) + if issue_warning is not None: + _warn(warnings, issue_warning, staged_arc, entry) + if entry.resolution_state in { + StoryArcResolutionState.AMBIGUOUS, + StoryArcResolutionState.CONFLICT, + StoryArcResolutionState.SKIPPED, + }: + issue_id = None + + arc_id = int(arc.id) + membership = _membership_from_pointer(lookups.memberships_by_id, entry, arc_id) + if membership is None and issue_id is not None: + membership = lookups.memberships_by_arc_issue.get((arc_id, issue_id)) + source_identity_conflict = False + if membership is None and entry.source_entry_id is not None: + membership = lookups.memberships_by_arc_source.get( + (arc_id, entry.source_kind, entry.source_entry_id) + ) + if ( + membership is not None + and issue_id is not None + and membership.issue_id is not None + and membership.issue_id != issue_id + ): + _warn(warnings, "source_entry_identity_conflict", staged_arc, entry) + membership = None + issue_id = None + source_identity_conflict = True + + if membership is not None: + membership_before = _membership_state(membership) + arc_revision_before = int(arc.revision) + same_source_entry = ( + membership.source_kind == entry.source_kind + and membership.source_entry_id == entry.source_entry_id + ) + if issue_id is not None and membership.issue_id is None: + duplicate = lookups.memberships_by_arc_issue.get((arc_id, issue_id)) + if duplicate is None or duplicate.id == membership.id: + membership.issue_id = issue_id + membership.resolution_state = StoryArcResolutionState.RESOLVED + membership.sync_eligible = bool(arc.sync_enabled) + lookups.memberships_by_arc_issue[(arc_id, issue_id)] = membership + arc.revision += 1 + membership_changed = membership_before != _membership_state(membership) + if membership_changed and record_action is not None: + await session.flush() + await record_action( + session, + job, + phase="story_arcs", + action_type="story_arc_membership_updated", + payload={ + "membership_id": int(membership.id), + "story_arc_id": arc_id, + "imported_story_arc_entry_id": int(entry.id), + "restore_before": membership_before, + "expected_after": _membership_state(membership), + "arc_revision_before": arc_revision_before, + "arc_revision_after": int(arc.revision), + }, + ) + if not same_source_entry and issue_id is not None: + _warn(warnings, "duplicate_issue_membership_reused", staged_arc, entry) + entry.materialized_membership_id = membership.id + _increment_counts(counts, context.counts, "memberships_reused") + else: + arc_revision_before = int(arc.revision) + issue = lookups.issues_by_id.get(issue_id) if issue_id is not None else None + exact_number, exact_warning = _exact_issue_number(entry, issue) + if exact_warning is not None: + _warn(warnings, exact_warning, staged_arc, entry) + resolution_state = ( + StoryArcResolutionState.CONFLICT + if source_identity_conflict + else _materialized_resolution_state(entry, issue_id) + ) + sequence_number = ( + int(entry.reading_order) + if entry.reading_order is not None + else int(entry.source_ordinal) + ) + if entry.reading_order is None: + _warn(warnings, "reading_order_defaulted_to_source_ordinal", staged_arc, entry) + membership = IssueStoryArc( + story_arc_id=arc.id, + issue_id=issue_id, + sequence_number=sequence_number, + source_ordinal=int(entry.source_ordinal), + legacy_sequence_was_null=entry.reading_order is None, + resolution_state=resolution_state, + source_kind=entry.source_kind, + source_entry_id=entry.source_entry_id, + source_arc_id=entry.source_arc_id, + source_issue_id=entry.source_issue_id, + source_series_id=entry.source_series_id, + source_issue_number_text=exact_number, + source_series_name=entry.source_series_name, + source_issue_title=entry.source_issue_title, + source_publisher=entry.source_publisher, + source_release_date_text=entry.source_release_date_text, + source_issue_date_text=entry.source_issue_date_text, + resolution_confidence=entry.resolution_confidence, + resolution_method=entry.resolution_method, + evidence=dict(entry.evidence or {}), + sync_eligible=( + issue_id is not None + and resolution_state == StoryArcResolutionState.RESOLVED + and bool(arc.sync_enabled) + ), + ) + session.add(membership) + await session.flush() + entry.materialized_membership_id = membership.id + lookups.memberships_by_id[int(membership.id)] = membership + if membership.issue_id is not None: + lookups.memberships_by_arc_issue[(arc_id, int(membership.issue_id))] = membership + if membership.source_entry_id is not None: + lookups.memberships_by_arc_source[ + (arc_id, membership.source_kind, membership.source_entry_id) + ] = membership + _increment_counts(counts, context.counts, "memberships_created") + arc.revision += 1 + if record_action is not None: + await session.flush() + await record_action( + session, + job, + phase="story_arcs", + action_type="story_arc_membership_created", + payload={ + "membership_id": int(membership.id), + "story_arc_id": arc_id, + "imported_story_arc_entry_id": int(entry.id), + "expected_after": _membership_state(membership), + "arc_revision_before": arc_revision_before, + "arc_revision_after": int(arc.revision), + }, + ) + + if membership.resolution_state == StoryArcResolutionState.RESOLVED: + _increment_counts(counts, context.counts, "resolved_entries") + else: + _increment_counts(counts, context.counts, "unresolved_entries") + if membership.resolution_state == StoryArcResolutionState.SKIPPED: + _increment_counts(counts, context.counts, "entries_skipped") + await _materialize_referenced_placement( + session, + job=job, + staged_arc=staged_arc, + entry=entry, + membership=membership, + warnings=warnings, + record_action=record_action, + lookups=lookups, + reference_checkpoint=reference_checkpoint, + ) + _prepare_managed_placement_request( + context=context, + entry=entry, + membership=membership, + warnings=warnings, + lookups=lookups, + journal_available=managed_placement_journal_available, + requests=managed_placement_requests, + ) + _persist_entry_materialization_diagnostics( + entry, + membership_id=int(membership.id), + warning_codes=[warning.code for warning in warnings[entry_warning_start:]], + ) + + +def _prepare_managed_placement_request( + *, + context: _ArcMaterializationContext, + entry: ImportedStoryArcEntry, + membership: IssueStoryArc, + warnings: list[StoryArcMaterializationWarning], + lookups: _EntryPageLookups, + journal_available: bool, + requests: list[_ManagedPlacementRequest], +) -> None: + """Prepare one exact placement request without writing its journal or outbox.""" + policy = context.policy + mode = policy.snapshot.get("mode") if policy.activated else None + if mode not in {"copy", "hardlink", "symlink"}: + return + if ( + membership.issue_id is None + or membership.resolution_state is not StoryArcResolutionState.RESOLVED + ): + return + library_file = lookups.library_files_by_issue_id.get(int(membership.issue_id)) + if library_file is None: + _warn( + warnings, + "story_arc_managed_placement_canonical_file_missing", + context.staged_arc, + entry, + ) + return + if not journal_available: + _warn( + warnings, + "story_arc_managed_placement_journal_unavailable", + context.staged_arc, + entry, + ) + return + + requests.append( + _ManagedPlacementRequest( + proposal=ImportStoryArcSyncProposal( + library_file=library_file, + membership=membership, + story_arc=context.arc, + imported_story_arc_id=int(context.staged_arc.id), + imported_story_arc_entry_id=int(entry.id), + ), + context=context, + ) + ) + + +async def _enqueue_managed_placement_requests( + session: AsyncSession, + *, + job: ImportJob, + requests: Sequence[_ManagedPlacementRequest], + counts: _MutableCounts, + record_actions: RecordActionsFunc | None, + cancellation_check: CancellationCheck | None, + durable_checkpoint: DurableCheckpoint | None = None, + record_action: RecordActionFunc | None = None, +) -> None: + """Publish SQL-bounded batches with a durable control checkpoint after each.""" + if not requests: + return + + batch_recorder = record_actions + if batch_recorder is None: + single_recorder = record_action + if single_recorder is None: + raise RuntimeError("Managed Story Arc placement journal is unavailable") + + async def record_actions_adapter( + callback_session: AsyncSession, + callback_job: ImportJob, + specs: Sequence[ImportJobActionSpec], + ) -> list[ImportJobAction]: + return [ + await single_recorder( + callback_session, + callback_job, + phase=spec.phase, + action_type=spec.action_type, + payload=spec.payload, + ) + for spec in specs + ] + + batch_recorder = cast("RecordActionsFunc", record_actions_adapter) + + if batch_recorder is None: + raise RuntimeError("Managed Story Arc placement journal is unavailable") + + for start in range(0, len(requests), MAX_IMPORT_STORY_ARC_SYNC_ENQUEUE_BATCH_SIZE): + await _checkpoint(cancellation_check) + request_batch = requests[start : start + MAX_IMPORT_STORY_ARC_SYNC_ENQUEUE_BATCH_SIZE] + results: list[ + StoryArcImportSyncEnqueueResult + ] = await enqueue_import_story_arc_sync_work_batch( + session, + job=job, + proposals=[request.proposal for request in request_batch], + record_actions=batch_recorder, + ) + if len(results) != len(request_batch): + raise RuntimeError("Managed Story Arc placement batch result count changed") + for request, result in zip(request_batch, results, strict=True): + if result.classification == "created": + _increment_counts( + counts, + request.context.counts, + "managed_placements_queued", + ) + continue + if result.classification in { + "in_call_duplicate", + "in_call_membership_duplicate", + "existing_import_membership_duplicate", + "existing_import_work_pending", + }: + continue + if result.classification in { + "existing_import_work_completed", + "existing_non_origin_placement", + "existing_managed_placement", + "existing_referenced_placement", + }: + _increment_counts( + counts, + request.context.counts, + "managed_placements_reused", + ) + continue + raise RuntimeError(f"Managed Story Arc placement returned {result.classification!r}") + await session.flush() + if durable_checkpoint is not None: + await durable_checkpoint() + + +async def _materialize_referenced_placement( + session: AsyncSession, + *, + job: ImportJob, + staged_arc: ImportedStoryArc, + entry: ImportedStoryArcEntry, + membership: IssueStoryArc, + warnings: list[StoryArcMaterializationWarning], + record_action: RecordActionFunc | None, + lookups: _EntryPageLookups, + reference_checkpoint: _ReferencePageCheckpoint, +) -> None: + """Attach one confirmed pre-existing artifact without opening or mutating it.""" + resolution = lookups.reference_candidates_by_entry_id.get(int(entry.id)) + if resolution is None: + return + await reference_checkpoint.ensure_checked() + if resolution.warning_code is not None or resolution.candidate is None: + _warn( + warnings, + resolution.warning_code or "story_arc_reference_path_invalid", + staged_arc, + entry, + ) + return + + candidate = resolution.candidate + placement_path = str(candidate.path) + existing = lookups.placements_by_path.get(placement_path) + if existing is not None and not _is_same_import_reference( + existing, + job=job, + entry=entry, + membership=membership, + ): + _warn(warnings, "story_arc_reference_path_collision", staged_arc, entry) + return + if existing is None: + prior_for_membership = [ + placement + for placement in lookups.placements_by_membership.get(int(membership.id), ()) + if _is_same_import_reference( + placement, + job=job, + entry=entry, + membership=membership, + ) + ] + if prior_for_membership: + _warn(warnings, "story_arc_reference_location_changed", staged_arc, entry) + return + + inspection = _inspect_reference_path(candidate) + if existing is not None: + if existing.creating_action_id is None: + _warn(warnings, "story_arc_reference_provenance_incomplete", staged_arc, entry) + return + _refresh_referenced_placement( + existing, + inspection=inspection, + staged_arc=staged_arc, + entry=entry, + warnings=warnings, + ) + return + if inspection.warning_code is not None or inspection.fingerprint is None: + _warn( + warnings, + inspection.warning_code or "story_arc_reference_path_invalid", + staged_arc, + entry, + ) + return + if record_action is None: + _warn(warnings, "story_arc_reference_journal_unavailable", staged_arc, entry) + return + + prepared_payload: dict[str, object] = { + "schema_version": 1, + "journal_state": "prepared", + "placement_id": None, + "issue_story_arc_id": int(membership.id), + "imported_story_arc_entry_id": int(entry.id), + "placement_path": placement_path, + "source_kind": entry.source_kind.value, + "source_import_job_id": int(job.id), + "expected_after": None, + } + action = await record_action( + session, + job, + phase="story_arcs", + action_type="story_arc_referenced_placement_attached", + payload=prepared_payload, + ) + now = datetime.now(UTC) + last_result = _reference_last_result( + code="reference_current", + baseline=inspection.fingerprint, + observed=inspection.fingerprint, + ) + imported_file = entry.import_file + library_file_id = ( + int(imported_file.library_file_id) + if imported_file is not None + and imported_file.import_job_id == job.id + and imported_file.library_file_id is not None + else None + ) + placement = StoryArcPlacement( + issue_story_arc_id=int(membership.id), + library_file_id=library_file_id, + placement_path=placement_path, + mode=StoryArcPlacementMode.REFERENCE_ONLY, + ownership=StoryArcPlacementOwnership.REFERENCED, + symlink_style=None, + source_kind=entry.source_kind, + source_import_job_id=int(job.id), + creating_action_id=int(action.id), + rendered_reading_order=int(membership.sequence_number), + source_fingerprint={}, + state=StoryArcPlacementState.CURRENT, + last_result=last_result, + last_checked_at=now, + ) + session.add(placement) + await session.flush() + action.payload = { + **prepared_payload, + "journal_state": "completed", + "placement_id": int(placement.id), + "expected_after": _referenced_placement_state(placement), + } + await session.flush() + lookups.placements_by_path[placement_path] = placement + lookups.placements_by_membership.setdefault(int(membership.id), []).append(placement) + + +def _refresh_referenced_placement( + placement: StoryArcPlacement, + *, + inspection: _ReferencePathInspection, + staged_arc: ImportedStoryArc, + entry: ImportedStoryArcEntry, + warnings: list[StoryArcMaterializationWarning], +) -> None: + baseline = _reference_baseline_fingerprint(placement) + now = datetime.now(UTC) + placement.last_checked_at = now + if inspection.warning_code is not None or inspection.fingerprint is None: + missing = inspection.warning_code == "story_arc_reference_missing" + placement.state = ( + StoryArcPlacementState.MISSING if missing else StoryArcPlacementState.DRIFTED + ) + placement.last_result = _reference_last_result( + code="reference_missing" if missing else "reference_unsafe", + baseline=baseline, + observed=None, + warning_code=inspection.warning_code, + ) + _warn( + warnings, + inspection.warning_code or "story_arc_reference_path_invalid", + staged_arc, + entry, + ) + return + if baseline is None: + placement.state = StoryArcPlacementState.DRIFTED + placement.last_result = _reference_last_result( + code="reference_provenance_incomplete", + baseline=None, + observed=inspection.fingerprint, + ) + _warn(warnings, "story_arc_reference_provenance_incomplete", staged_arc, entry) + return + if baseline != inspection.fingerprint: + placement.state = StoryArcPlacementState.DRIFTED + placement.last_result = _reference_last_result( + code="reference_drifted", + baseline=baseline, + observed=inspection.fingerprint, + ) + _warn(warnings, "story_arc_reference_drifted", staged_arc, entry) + return + placement.state = StoryArcPlacementState.CURRENT + placement.last_result = _reference_last_result( + code="reference_current", + baseline=baseline, + observed=inspection.fingerprint, + ) + + +def _is_same_import_reference( + placement: StoryArcPlacement, + *, + job: ImportJob, + entry: ImportedStoryArcEntry, + membership: IssueStoryArc, +) -> bool: + return ( + placement.issue_story_arc_id == membership.id + and placement.mode is StoryArcPlacementMode.REFERENCE_ONLY + and placement.ownership is StoryArcPlacementOwnership.REFERENCED + and placement.source_kind is entry.source_kind + and placement.source_import_job_id == job.id + ) + + +def _reference_baseline_fingerprint( + placement: StoryArcPlacement, +) -> dict[str, object] | None: + baseline = _mapping(placement.last_result).get("baseline_fingerprint") + return dict(baseline) if isinstance(baseline, dict) else None + + +def _reference_last_result( + *, + code: str, + baseline: Mapping[str, object] | None, + observed: Mapping[str, object] | None, + warning_code: str | None = None, +) -> dict[str, object]: + return { + "schema_version": 1, + "code": code, + "baseline_fingerprint": dict(baseline) if baseline is not None else None, + "observed_fingerprint": dict(observed) if observed is not None else None, + "warning_code": warning_code, + } + + +def _referenced_placement_state(placement: StoryArcPlacement) -> dict[str, object]: + return { + "issue_story_arc_id": int(placement.issue_story_arc_id), + "library_file_id": placement.library_file_id, + "placement_path": placement.placement_path, + "mode": placement.mode.value, + "ownership": placement.ownership.value, + "symlink_style": None, + "source_kind": placement.source_kind.value, + "source_import_job_id": placement.source_import_job_id, + "creating_action_id": placement.creating_action_id, + "rendered_reading_order": placement.rendered_reading_order, + "source_fingerprint": dict(placement.source_fingerprint or {}), + "state": placement.state.value, + "last_result": dict(placement.last_result or {}), + } + + +def _resolve_reference_candidate( + job: ImportJob, + entry: ImportedStoryArcEntry, +) -> _ReferenceCandidateResolution: + raw_location = entry.source_location + if raw_location is None: + return _ReferenceCandidateResolution() + if _unsafe_path_text(raw_location): + return _ReferenceCandidateResolution(warning_code="story_arc_reference_path_invalid") + if ( + job.source_type is ImportSourceType.FILESYSTEM + and entry.source_kind is StoryArcSourceKind.FOLDER + ): + return _candidate_under_trusted_root( + raw_location=raw_location, + trusted_root=job.source_path, + ) + if ( + job.source_type is ImportSourceType.MYLAR3 + and entry.source_kind is StoryArcSourceKind.MYLAR3 + ): + return _mapped_mylar_reference_candidate( + raw_location=raw_location, + path_map=job.mylar3_path_map, + ) + return _ReferenceCandidateResolution(warning_code="story_arc_reference_source_mismatch") + + +def _candidate_under_trusted_root( + *, + raw_location: str, + trusted_root: str, +) -> _ReferenceCandidateResolution: + if _unsafe_path_text(trusted_root): + return _ReferenceCandidateResolution(warning_code="story_arc_reference_root_untrusted") + root = Path(trusted_root) + candidate = Path(raw_location) + if ( + not root.is_absolute() + or not candidate.is_absolute() + or root == Path(root.anchor) + or ".." in root.parts + or ".." in candidate.parts + ): + return _ReferenceCandidateResolution(warning_code="story_arc_reference_path_invalid") + normalized_root = Path(os.path.abspath(root)) + normalized_candidate = Path(os.path.abspath(candidate)) + try: + normalized_candidate.relative_to(normalized_root) + except ValueError: + return _ReferenceCandidateResolution( + warning_code="story_arc_reference_outside_trusted_root" + ) + if normalized_candidate == normalized_root: + return _ReferenceCandidateResolution(warning_code="story_arc_reference_not_regular_file") + if len(str(normalized_candidate)) > 1000: + return _ReferenceCandidateResolution(warning_code="story_arc_reference_path_invalid") + return _ReferenceCandidateResolution( + candidate=_ReferencePathCandidate( + path=normalized_candidate, + trusted_root=normalized_root, + ) + ) + + +def _mapped_mylar_reference_candidate( + *, + raw_location: str, + path_map: object, +) -> _ReferenceCandidateResolution: + if not isinstance(path_map, dict) or not path_map: + return _ReferenceCandidateResolution(warning_code="story_arc_reference_root_untrusted") + mapping_items: list[tuple[PurePath, str, Path]] = [] + for raw_remote_root, raw_host_root in path_map.items(): + if not isinstance(raw_remote_root, str) or not isinstance(raw_host_root, str): + continue + if _unsafe_path_text(raw_remote_root) or _unsafe_path_text(raw_host_root): + continue + remote_root = _pure_absolute_path(raw_remote_root) + host_root = Path(raw_host_root) + if ( + remote_root is None + or not host_root.is_absolute() + or host_root == Path(host_root.anchor) + or ".." in host_root.parts + ): + continue + mapping_items.append((remote_root, raw_host_root, Path(os.path.abspath(host_root)))) + if not mapping_items: + return _ReferenceCandidateResolution(warning_code="story_arc_reference_root_untrusted") + + direct_candidates: list[_ReferenceCandidateResolution] = [] + for _remote_root, raw_host_root, _normalized_host_root in mapping_items: + direct = _candidate_under_trusted_root( + raw_location=raw_location, + trusted_root=raw_host_root, + ) + if direct.candidate is not None: + direct_candidates.append(direct) + if direct_candidates: + return max( + direct_candidates, + key=lambda item: ( + len(item.candidate.trusted_root.parts) if item.candidate is not None else 0 + ), + ) + + remote_location = _pure_absolute_path(raw_location) + if remote_location is None or ".." in remote_location.parts: + return _ReferenceCandidateResolution(warning_code="story_arc_reference_path_invalid") + for remote_root, _raw_host_root, host_root in sorted( + mapping_items, + key=lambda item: len(item[0].parts), + reverse=True, + ): + if type(remote_location) is not type(remote_root): + continue + try: + relative = remote_location.relative_to(remote_root) + except ValueError: + continue + if not relative.parts or ".." in relative.parts: + return _ReferenceCandidateResolution( + warning_code="story_arc_reference_not_regular_file" + ) + candidate = host_root.joinpath(*relative.parts) + if len(str(candidate)) > 1000: + return _ReferenceCandidateResolution(warning_code="story_arc_reference_path_invalid") + return _ReferenceCandidateResolution( + candidate=_ReferencePathCandidate( + path=candidate, + trusted_root=host_root, + ) + ) + return _ReferenceCandidateResolution(warning_code="story_arc_reference_outside_trusted_root") + + +def _pure_absolute_path(value: str) -> PurePath | None: + windows_path = PureWindowsPath(value) + if windows_path.is_absolute(): + return windows_path + posix_path = PurePosixPath(value.replace("\\", "/")) + return posix_path if posix_path.is_absolute() else None + + +def _unsafe_path_text(value: str) -> bool: + return not value or any(ord(character) < 32 for character in value) + + +def _inspect_reference_path( + candidate: _ReferencePathCandidate, +) -> _ReferencePathInspection: + if not _secure_reference_inspection_supported(): + return _ReferencePathInspection( + warning_code="story_arc_reference_secure_inspection_unavailable" + ) + try: + before = _secure_reference_stat(candidate) + after = _secure_reference_stat(candidate) + except _ReferencePathValidationError as exc: + return _ReferencePathInspection(warning_code=exc.code) + if _reference_stat_identity(before) != _reference_stat_identity(after): + return _ReferencePathInspection( + warning_code="story_arc_reference_changed_during_inspection" + ) + return _ReferencePathInspection(fingerprint=_reference_metadata_fingerprint(after)) + + +class _ReferencePathValidationError(ValueError): + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +def _secure_reference_inspection_supported() -> bool: + return bool( + os.name == "posix" + and hasattr(os, "O_NOFOLLOW") + and hasattr(os, "O_DIRECTORY") + and os.open in os.supports_dir_fd + and os.stat in os.supports_dir_fd + and os.stat in os.supports_follow_symlinks + ) + + +def _secure_reference_stat(candidate: _ReferencePathCandidate) -> os.stat_result: + root = candidate.trusted_root + try: + relative = candidate.path.relative_to(root) + except ValueError as exc: # pragma: no cover - candidate constructor invariant + raise _ReferencePathValidationError("story_arc_reference_outside_trusted_root") from exc + if not relative.parts: + raise _ReferencePathValidationError("story_arc_reference_not_regular_file") + try: + root_stat = root.lstat() + except FileNotFoundError as exc: + raise _ReferencePathValidationError("story_arc_reference_missing") from exc + except OSError as exc: + raise _ReferencePathValidationError("story_arc_reference_unavailable") from exc + if stat.S_ISLNK(root_stat.st_mode): + raise _ReferencePathValidationError("story_arc_reference_symlink") + if not stat.S_ISDIR(root_stat.st_mode): + raise _ReferencePathValidationError("story_arc_reference_root_untrusted") + + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + descriptors: list[int] = [] + try: + root_fd = os.open(root, flags) + descriptors.append(root_fd) + opened_root = os.fstat(root_fd) + if _reference_stat_node(root_stat) != _reference_stat_node(opened_root): + raise _ReferencePathValidationError("story_arc_reference_changed_during_inspection") + parent_fd = root_fd + for part in relative.parts[:-1]: + child_stat = _stat_child_nofollow(parent_fd, part) + if stat.S_ISLNK(child_stat.st_mode): + raise _ReferencePathValidationError("story_arc_reference_symlink") + if not stat.S_ISDIR(child_stat.st_mode): + raise _ReferencePathValidationError("story_arc_reference_not_regular_file") + try: + child_fd = os.open(part, flags, dir_fd=parent_fd) + except FileNotFoundError as exc: + raise _ReferencePathValidationError("story_arc_reference_missing") from exc + except OSError as exc: + raise _ReferencePathValidationError( + "story_arc_reference_changed_during_inspection" + ) from exc + descriptors.append(child_fd) + opened_child = os.fstat(child_fd) + if _reference_stat_node(child_stat) != _reference_stat_node(opened_child): + raise _ReferencePathValidationError("story_arc_reference_changed_during_inspection") + parent_fd = child_fd + target_stat = _stat_child_nofollow(parent_fd, relative.parts[-1]) + if stat.S_ISLNK(target_stat.st_mode): + raise _ReferencePathValidationError("story_arc_reference_symlink") + if not stat.S_ISREG(target_stat.st_mode): + raise _ReferencePathValidationError("story_arc_reference_not_regular_file") + return target_stat + except _ReferencePathValidationError: + raise + except FileNotFoundError as exc: + raise _ReferencePathValidationError("story_arc_reference_missing") from exc + except OSError as exc: + raise _ReferencePathValidationError("story_arc_reference_unavailable") from exc + finally: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def _stat_child_nofollow(parent_fd: int, name: str) -> os.stat_result: + try: + return os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + except FileNotFoundError as exc: + raise _ReferencePathValidationError("story_arc_reference_missing") from exc + except OSError as exc: + raise _ReferencePathValidationError("story_arc_reference_unavailable") from exc + + +def _reference_stat_node(value: os.stat_result) -> tuple[int, int, int]: + return value.st_dev, value.st_ino, value.st_mode + + +def _reference_stat_identity(value: os.stat_result) -> tuple[int, int, int, int, int, int]: + return ( + value.st_dev, + value.st_ino, + value.st_mode, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def _reference_metadata_fingerprint(value: os.stat_result) -> dict[str, object]: + return { + "schema_version": 1, + "kind": "regular_metadata", + "size": int(value.st_size), + "mtime_ns": int(value.st_mtime_ns), + "ctime_ns": int(value.st_ctime_ns), + "device": int(value.st_dev), + "inode": int(value.st_ino), + } + + +def _candidate_issue_ids( + entries: Sequence[ImportedStoryArcEntry], + *, + import_job_id: int, +) -> set[int]: + result: set[int] = set() + for entry in entries: + if entry.matched_issue_id is not None: + result.add(int(entry.matched_issue_id)) + imported_file = entry.import_file + if ( + imported_file is not None + and imported_file.import_job_id == import_job_id + and imported_file.status in _SAFE_IMPORTED_FILE_STATES + and imported_file.matched_issue_id is not None + ): + result.add(int(imported_file.matched_issue_id)) + return result + + +def _resolved_issue_id( + entry: ImportedStoryArcEntry, + *, + issues: Mapping[int, Issue], + import_job_id: int, +) -> tuple[int | None, str | None]: + direct_id = int(entry.matched_issue_id) if entry.matched_issue_id is not None else None + imported_file = entry.import_file + file_id: int | None = None + if imported_file is not None: + if imported_file.import_job_id != import_job_id: + return direct_id if direct_id in issues else None, "import_file_job_mismatch" + if ( + imported_file.status in _SAFE_IMPORTED_FILE_STATES + and imported_file.matched_issue_id is not None + ): + file_id = int(imported_file.matched_issue_id) + elif direct_id is None and imported_file.matched_issue_id is not None: + return None, "import_file_match_not_materialized" + + if direct_id is not None and file_id is not None and direct_id != file_id: + return None, "matched_issue_identity_conflict" + candidate = direct_id if direct_id is not None else file_id + if candidate is None: + return None, None + if candidate not in issues: + return None, "matched_issue_missing" + return candidate, None + + +def _membership_from_pointer( + memberships_by_id: Mapping[int, IssueStoryArc], + entry: ImportedStoryArcEntry, + story_arc_id: int, +) -> IssueStoryArc | None: + if entry.materialized_membership_id is None: + return None + membership = memberships_by_id.get(int(entry.materialized_membership_id)) + if membership is None or membership.story_arc_id != story_arc_id: + return None + return membership + + +def _exact_issue_number( + entry: ImportedStoryArcEntry, + issue: Issue | None, +) -> tuple[str | None, str | None]: + if entry.source_issue_number_text is not None: + try: + return normalize_issue_number_text(entry.source_issue_number_text), None + except ValueError: + return ( + issue.effective_issue_number_text if issue is not None else None, + "source_issue_number_invalid", + ) + if issue is not None: + return issue.effective_issue_number_text, None + return None, "source_issue_number_missing" + + +def _materialized_resolution_state( + entry: ImportedStoryArcEntry, + issue_id: int | None, +) -> StoryArcResolutionState: + if issue_id is not None: + return StoryArcResolutionState.RESOLVED + if entry.resolution_state == StoryArcResolutionState.RESOLVED: + return StoryArcResolutionState.MISSING + if entry.resolution_state in _UNRESOLVED_STATES: + return entry.resolution_state + return StoryArcResolutionState.PENDING + + +def _validate_policy( + staged_arc: ImportedStoryArc, + *, + library_root_ids: set[int], +) -> _ValidatedPolicy: + raw = staged_arc.proposed_policy_snapshot + if not isinstance(raw, dict) or not raw: + return _ValidatedPolicy(activated=False, snapshot={}) + if raw.get("activation") != "confirmed": + return _ValidatedPolicy( + activated=False, + snapshot={}, + warning_code="policy_not_activated", + ) + if ( + set(raw) != _CONFIRMED_POLICY_KEYS + or isinstance(raw.get("schema_version"), bool) + or raw.get("schema_version") != 1 + or raw.get("source") != staged_arc.source_kind.value + or any(not isinstance(raw.get(key), bool) for key in _POLICY_FLAGS) + or ( + raw.get("monitored") is False + and (raw.get("search_missing") is True or raw.get("include_upcoming") is True) + ) + ): + return _ValidatedPolicy( + activated=False, + snapshot={}, + warning_code="policy_validation_failed", + ) + placement_snapshot, target_root_id, placement_warning = _validate_placement_policy_snapshot( + raw.get("placement_policy"), + library_root_ids=library_root_ids, + ) + if placement_warning is not None or placement_snapshot is None: + return _ValidatedPolicy( + activated=False, + snapshot={}, + warning_code=placement_warning or "policy_validation_failed", + ) + if raw["sync_enabled"] != placement_snapshot["synchronize"]: + return _ValidatedPolicy( + activated=False, + snapshot={}, + warning_code="policy_validation_failed", + ) + return _ValidatedPolicy( + activated=True, + snapshot=placement_snapshot, + monitored=bool(raw["monitored"]), + search_missing=bool(raw["search_missing"]), + include_upcoming=bool(raw["include_upcoming"]), + sync_enabled=bool(raw["sync_enabled"]), + target_library_root_id=target_root_id, + ) + + +def _validate_placement_policy_snapshot( + value: object, + *, + library_root_ids: set[int], +) -> tuple[dict[str, object] | None, int | None, str | None]: + if not isinstance(value, dict) or set(value) != _PLACEMENT_POLICY_KEYS: + return None, None, "policy_validation_failed" + if isinstance(value.get("schema_version"), bool) or value.get("schema_version") != 1: + return None, None, "policy_validation_failed" + mode = value.get("mode") + if not isinstance(mode, str) or mode not in _PLACEMENT_POLICY_MODES: + return None, None, "policy_validation_failed" + synchronize = value.get("synchronize") + if not isinstance(synchronize, bool): + return None, None, "policy_validation_failed" + folder_template = value.get("folder_template") + file_template = value.get("file_template") + if ( + not isinstance(folder_template, str) + or not isinstance(file_template, str) + or len(folder_template.encode("utf-8")) > 1024 + or len(file_template.encode("utf-8")) > 1024 + ): + return None, None, "policy_validation_failed" + try: + validate_story_arc_folder_template(folder_template) + validate_story_arc_file_template(file_template) + except ValueError: + return None, None, "policy_validation_failed" + + symlink_style = value.get("symlink_style") + if mode == "symlink": + if symlink_style not in {"absolute", "relative"}: + return None, None, "policy_validation_failed" + elif symlink_style is not None: + return None, None, "policy_validation_failed" + + target_root_id_raw = value.get("target_library_root_id") + destination_root = value.get("destination_root") + if mode == "logical": + if target_root_id_raw is not None or destination_root is not None or synchronize: + return None, None, "policy_validation_failed" + target_root_id = None + else: + if ( + isinstance(target_root_id_raw, bool) + or not isinstance(target_root_id_raw, int) + or target_root_id_raw <= 0 + or not isinstance(destination_root, str) + or not destination_root.strip() + or len(destination_root) > 1000 + or _unsafe_path_text(destination_root) + or not Path(destination_root).is_absolute() + ): + return None, None, "policy_validation_failed" + if target_root_id_raw not in library_root_ids: + return None, None, "policy_target_root_missing" + target_root_id = target_root_id_raw + return dict(value), target_root_id, None + + +def _apply_policy( + arc: StoryArc, + policy: _ValidatedPolicy, + *, + increment_revision: bool, +) -> bool: + changed = any( + ( + arc.monitored != policy.monitored, + arc.search_missing != policy.search_missing, + arc.include_upcoming != policy.include_upcoming, + arc.sync_enabled != policy.sync_enabled, + arc.target_library_root_id != policy.target_library_root_id, + arc.policy_schema_version != 1, + arc.policy_snapshot != policy.snapshot, + ) + ) + arc.monitored = policy.monitored + arc.search_missing = policy.search_missing + arc.include_upcoming = policy.include_upcoming + arc.sync_enabled = policy.sync_enabled + arc.target_library_root_id = policy.target_library_root_id + arc.policy_schema_version = 1 + arc.policy_snapshot = dict(policy.snapshot) + if changed and increment_revision: + arc.revision += 1 + return changed + + +async def _materialize_external_identity( + session: AsyncSession, + *, + arc: StoryArc, + staged_arc: ImportedStoryArc, + identity: tuple[str, str, str], + state: _MaterializationState, +) -> tuple[str, StoryArcExternalIdentity | None]: + source, namespace, external_id = identity + existing_arc_id = state.identities_by_key.get(identity) + if existing_arc_id is not None: + return ("reused", None) if existing_arc_id == arc.id else ("conflict", None) + created = StoryArcExternalIdentity( + story_arc_id=arc.id, + source=source, + namespace=namespace, + external_id=external_id, + evidence={ + "schema_version": 1, + "import_job_id": staged_arc.import_job_id, + "imported_story_arc_id": staged_arc.id, + }, + ) + session.add(created) + await session.flush() + state.identities_by_key[identity] = int(arc.id) + state.loaded_identity_keys.add(identity) + return "created", created + + +def _identity_targets_another_arc( + state: _MaterializationState, + identities: Sequence[tuple[str, str, str]], + expected_story_arc_id: int, +) -> bool: + return any( + arc_id != expected_story_arc_id for arc_id in _external_identity_arc_ids(state, identities) + ) + + +def _any_external_identity_exists( + state: _MaterializationState, + identities: Sequence[tuple[str, str, str]], +) -> bool: + return bool(_external_identity_arc_ids(state, identities)) + + +def _external_identity_arc_ids( + state: _MaterializationState, + identities: Sequence[tuple[str, str, str]], +) -> set[int]: + return { + identity_arc_id + for key in identities + if (identity_arc_id := state.identities_by_key.get(key)) is not None + } + + +def _identity_key(identity: StoryArcExternalIdentity) -> tuple[str, str, str]: + return (identity.source, identity.namespace, identity.external_id) + + +def _warn( + warnings: list[StoryArcMaterializationWarning], + code: str, + staged_arc: ImportedStoryArc, + entry: ImportedStoryArcEntry | None = None, +) -> None: + warnings.append( + StoryArcMaterializationWarning( + code=code, + imported_story_arc_id=int(staged_arc.id), + imported_story_arc_entry_id=int(entry.id) if entry is not None else None, + ) + ) + + +def _story_arc_policy_state(arc: StoryArc) -> dict[str, object]: + return { + "monitored": bool(arc.monitored), + "search_missing": bool(arc.search_missing), + "include_upcoming": bool(arc.include_upcoming), + "sync_enabled": bool(arc.sync_enabled), + "target_library_root_id": arc.target_library_root_id, + "policy_schema_version": arc.policy_schema_version, + "policy_snapshot": dict(arc.policy_snapshot or {}), + "revision": int(arc.revision), + } + + +def _story_arc_created_state(arc: StoryArc) -> dict[str, object]: + return { + "name": arc.name, + "normalized_name": arc.normalized_name, + "description": arc.description, + "comicvine_id": arc.comicvine_id, + "publisher_id": arc.publisher_id, + "comicvine_url": arc.comicvine_url, + "source_kind": arc.source_kind.value, + "lifecycle": arc.lifecycle.value, + "source_import_job_id": arc.source_import_job_id, + "diagnostics": dict(arc.diagnostics or {}), + **_story_arc_policy_state(arc), + } + + +def _membership_state(membership: IssueStoryArc) -> dict[str, object]: + return { + "story_arc_id": int(membership.story_arc_id), + "issue_id": membership.issue_id, + "sequence_number": int(membership.sequence_number), + "source_ordinal": int(membership.source_ordinal), + "legacy_sequence_was_null": bool(membership.legacy_sequence_was_null), + "resolution_state": membership.resolution_state.value, + "source_kind": membership.source_kind.value, + "source_entry_id": membership.source_entry_id, + "source_arc_id": membership.source_arc_id, + "source_issue_id": membership.source_issue_id, + "source_series_id": membership.source_series_id, + "source_issue_number_text": membership.source_issue_number_text, + "source_series_name": membership.source_series_name, + "source_issue_title": membership.source_issue_title, + "source_publisher": membership.source_publisher, + "source_release_date_text": membership.source_release_date_text, + "source_issue_date_text": membership.source_issue_date_text, + "resolution_confidence": membership.resolution_confidence, + "resolution_method": membership.resolution_method, + "evidence": dict(membership.evidence or {}), + "sync_eligible": bool(membership.sync_eligible), + "last_materialization_result": dict(membership.last_materialization_result or {}), + } + + +def _external_identity_state(identity: StoryArcExternalIdentity) -> dict[str, object]: + return { + "story_arc_id": int(identity.story_arc_id), + "source": identity.source, + "namespace": identity.namespace, + "external_id": identity.external_id, + "source_url": identity.source_url, + "evidence": dict(identity.evidence or {}), + } + + +def _persist_arc_materialization_diagnostics( + staged_arc: ImportedStoryArc, + *, + story_arc_id: int | None, + status: str, + counts: Mapping[str, int], + warning_codes: Sequence[str], +) -> None: + diagnostics = dict(staged_arc.diagnostics or {}) + diagnostics["materialization"] = { + "schema_version": 1, + "status": status, + "story_arc_id": story_arc_id, + "counts": dict(counts), + "warning_codes": list(dict.fromkeys(warning_codes)), + } + staged_arc.diagnostics = diagnostics + + +def _persist_entry_materialization_diagnostics( + entry: ImportedStoryArcEntry, + *, + membership_id: int, + warning_codes: Sequence[str], +) -> None: + diagnostics = dict(entry.diagnostics or {}) + diagnostics["materialization"] = { + "schema_version": 1, + "status": "imported", + "membership_id": membership_id, + "warning_codes": list(dict.fromkeys(warning_codes)), + } + entry.diagnostics = diagnostics + + +def _mapping(value: object) -> Mapping[str, object]: + return value if isinstance(value, dict) else {} + + +def _count_snapshot(counts: _MutableCounts) -> dict[str, int]: + return { + "arcs_created": counts.arcs_created, + "arcs_merged": counts.arcs_merged, + "arcs_reused": counts.arcs_reused, + "arcs_failed": counts.arcs_failed, + "external_identities_created": counts.external_identities_created, + "external_identities_reused": counts.external_identities_reused, + "memberships_created": counts.memberships_created, + "memberships_reused": counts.memberships_reused, + "resolved_entries": counts.resolved_entries, + "unresolved_entries": counts.unresolved_entries, + "entries_skipped": counts.entries_skipped, + "managed_placements_queued": counts.managed_placements_queued, + "managed_placements_reused": counts.managed_placements_reused, + } + + +def _increment_counts( + total: _MutableCounts, + arc: _MutableCounts, + field: CountField, +) -> None: + setattr(total, field, int(getattr(total, field)) + 1) + setattr(arc, field, int(getattr(arc, field)) + 1) + + +def _warning_codes_by_arc( + warnings: Sequence[StoryArcMaterializationWarning], +) -> dict[int, list[str]]: + result: dict[int, list[str]] = {} + for warning in warnings: + result.setdefault(warning.imported_story_arc_id, []).append(warning.code) + return result + + +async def _checkpoint(callback: CancellationCheck | None) -> None: + if callback is not None: + await callback() diff --git a/src/pullbox/services/import_story_arc_placement_completion.py b/src/pullbox/services/import_story_arc_placement_completion.py new file mode 100644 index 00000000..88c304d2 --- /dev/null +++ b/src/pullbox/services/import_story_arc_placement_completion.py @@ -0,0 +1,705 @@ +"""Truthful database-only completion for import-owned Story Arc placements.""" + +from __future__ import annotations + +import enum +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any, NoReturn, cast + +from sqlalchemy import and_, case, func, or_, select, update + +from pullbox.core.exceptions import ( + JobCancelledError, + JobPausedError, + NotFoundError, + ValidationError, +) +from pullbox.models.import_job import ( + ImportControlRequest, + ImportJob, + ImportJobAction, + ImportJobActionStatus, + ImportJobStatus, +) +from pullbox.models.library import LibraryFile +from pullbox.models.story_arc import ( + ImportedStoryArcStatus, + IssueStoryArc, + StoryArcPlacement, + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcPlacementState, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.models.story_arc_sync import StoryArcSyncWork, StoryArcSyncWorkState + +if TYPE_CHECKING: + from sqlalchemy.engine import CursorResult + from sqlalchemy.ext.asyncio import AsyncSession + + +_PHASE = "story_arc_placements" +_ACTION_TYPE = "story_arc_managed_placement_requested" +_PAYLOAD_KEYS = frozenset( + { + "schema_version", + "sync_work_id", + "membership_id", + "desired_generation", + "imported_story_arc_id", + "imported_story_arc_entry_id", + "source_import_job_id", + } +) +_ORIGIN_PAGE_SIZE = 1_000 +_ACTIVE_STATUSES = frozenset({ImportJobStatus.IMPORTING, ImportJobStatus.STALLED}) +_PENDING_STATES = frozenset( + { + StoryArcSyncWorkState.QUEUED, + StoryArcSyncWorkState.RUNNING, + StoryArcSyncWorkState.RETRY_WAIT, + } +) +_MANAGED_MODES = frozenset( + { + StoryArcPlacementMode.COPY, + StoryArcPlacementMode.HARDLINK, + StoryArcPlacementMode.SYMLINK, + } +) + + +class ImportStoryArcPlacementCompletionState(enum.StrEnum): + """One import-placement completion evaluation result.""" + + PENDING = "pending" + STALLED = "stalled" + COMPLETED = "completed" + + +@dataclass(frozen=True, slots=True) +class ImportStoryArcPlacementCounts: + """Sanitized aggregate counts for every durable work state.""" + + queued: int = 0 + running: int = 0 + retry_wait: int = 0 + failed: int = 0 + completed: int = 0 + cancelled: int = 0 + + @property + def total(self) -> int: + return sum(self.as_dict().values()) + + def as_dict(self) -> dict[str, int]: + return { + "queued": self.queued, + "running": self.running, + "retry_wait": self.retry_wait, + "failed": self.failed, + "completed": self.completed, + "cancelled": self.cancelled, + } + + +@dataclass(frozen=True, slots=True) +class ImportStoryArcPlacementCompletionOutcome: + """Database-only finalizer outcome safe for logs and progress responses.""" + + job_id: int + state: ImportStoryArcPlacementCompletionState + counts: ImportStoryArcPlacementCounts + error_code: str | None = None + + +@dataclass(frozen=True, slots=True) +class _OriginAggregate: + action_count: int + invalid_count: int + counts: ImportStoryArcPlacementCounts + + +_ERRORS = { + "story_arc_placement_origin_invalid": ( + "Story-arc placement recovery data is incomplete. Roll back the import." + ), + "story_arc_placement_work_failed": ( + "One or more story-arc placements failed. Retry the placement work or roll back the import." + ), + "story_arc_placement_work_cancelled": ( + "Story-arc placement work ended unexpectedly. Retry the placement work or " + "roll back the import." + ), + "story_arc_placement_evidence_invalid": ( + "Story-arc placement verification is incomplete. Roll back the import." + ), +} + + +async def finalize_import_story_arc_placements( + session: AsyncSession, + job_id: int, + *, + now: datetime | None = None, +) -> ImportStoryArcPlacementCompletionOutcome: + """Project one import's durable placement work into a truthful job state. + + The caller owns the surrounding transaction. This function locks and flushes + the job row, but never commits and never reads provider or filesystem data. + """ + job = await session.scalar(select(ImportJob).where(ImportJob.id == job_id).with_for_update()) + if job is None: + raise NotFoundError("ImportJob", job_id) + + snapshot = dict(job.progress_snapshot or {}) + if job.status is ImportJobStatus.COMPLETED and snapshot.get("phase") == "done": + return ImportStoryArcPlacementCompletionOutcome( + job_id=job.id, + state=ImportStoryArcPlacementCompletionState.COMPLETED, + counts=_counts_from_snapshot(snapshot), + ) + if job.status not in _ACTIVE_STATUSES or snapshot.get("phase") != _PHASE: + raise ValidationError("Import job is not awaiting story-arc placements.") + if job.control_request is not ImportControlRequest.NONE: + raise ValidationError("Import job has an active control request.") + + expected_total = _positive_int(snapshot.get("story_arc_placements_total")) + origin = await _aggregate_origin_work(session, job.id) + invalid_origin = ( + expected_total is None + or origin.action_count != expected_total + or origin.invalid_count != 0 + or origin.counts.total != origin.action_count + ) + if invalid_origin: + return await _stall( + session, + job, + snapshot, + origin.counts, + expected_total=expected_total, + error_code="story_arc_placement_origin_invalid", + ) + assert expected_total is not None + if origin.counts.failed: + return await _stall( + session, + job, + snapshot, + origin.counts, + expected_total=expected_total, + error_code="story_arc_placement_work_failed", + ) + if origin.counts.cancelled: + return await _stall( + session, + job, + snapshot, + origin.counts, + expected_total=expected_total, + error_code="story_arc_placement_work_cancelled", + ) + if any(origin.counts.as_dict()[state.value] for state in _PENDING_STATES): + return await _mark_pending( + session, + job, + snapshot, + origin.counts, + expected_total=expected_total, + ) + invalid_placements = await _count_invalid_completed_placements(session, job.id) + if invalid_placements: + return await _stall( + session, + job, + snapshot, + origin.counts, + expected_total=expected_total, + error_code="story_arc_placement_evidence_invalid", + ) + if origin.counts.completed != expected_total: + return await _stall( + session, + job, + snapshot, + origin.counts, + expected_total=expected_total, + error_code="story_arc_placement_origin_invalid", + ) + return await _complete( + session, + job, + snapshot, + origin.counts, + completed_at=now or datetime.now(UTC), + ) + + +async def inspect_import_story_arc_placement_origin( + session: AsyncSession, + job_id: int, +) -> ImportStoryArcPlacementCounts: + """Return authoritative typed work counts before publishing the wait phase.""" + origin = await _aggregate_origin_work(session, job_id) + if origin.invalid_count or origin.counts.total != origin.action_count: + raise ValidationError("Import Story Arc placement origin evidence is incomplete.") + return origin.counts + + +async def seal_import_story_arc_placement_origin( + session: AsyncSession, + job_id: int, +) -> ImportStoryArcPlacementCounts: + """Make fully journaled import work claimable in the caller's final transaction.""" + fence = cast( + "CursorResult[Any]", + await session.execute( + update(ImportJob) + .where( + ImportJob.id == job_id, + ImportJob.status == ImportJobStatus.IMPORTING, + ImportJob.control_request == ImportControlRequest.NONE, + ImportJob.progress_snapshot["phase"].as_string() == "story_arcs", + ) + .values(status=ImportJobStatus.IMPORTING) + .execution_options(synchronize_session=False) + ), + ) + if fence.rowcount != 1: + await _raise_seal_fence_control(session, job_id) + + counts = await inspect_import_story_arc_placement_origin(session, job_id) + invalid_held = int( + await session.scalar( + select(func.count(StoryArcSyncWork.id)).where( + StoryArcSyncWork.origin_import_job_id == job_id, + StoryArcSyncWork.claimable.is_(False), + StoryArcSyncWork.state != StoryArcSyncWorkState.QUEUED, + ) + ) + or 0 + ) + if invalid_held: + raise ValidationError("Held import Story Arc work changed before it was sealed.") + await session.execute( + update(StoryArcSyncWork) + .where( + StoryArcSyncWork.origin_import_job_id == job_id, + StoryArcSyncWork.claimable.is_(False), + StoryArcSyncWork.state == StoryArcSyncWorkState.QUEUED, + ) + .values(claimable=True) + ) + return counts + + +async def _raise_seal_fence_control(session: AsyncSession, job_id: int) -> NoReturn: + """Translate a lost final-transition CAS into the cooperative runner path.""" + row = ( + await session.execute( + select( + ImportJob.status, + ImportJob.control_request, + ImportJob.progress_snapshot, + ).where(ImportJob.id == job_id) + ) + ).one_or_none() + if row is None: + raise JobCancelledError(f"Import job {job_id} was cancelled.") + status, control_request, snapshot = row + if control_request is ImportControlRequest.CANCEL or status in { + ImportJobStatus.CANCELLING, + ImportJobStatus.CANCELLED, + ImportJobStatus.ROLLING_BACK, + ImportJobStatus.ROLLED_BACK, + }: + raise JobCancelledError(f"Import job {job_id} was cancelled.") + if control_request is ImportControlRequest.PAUSE or status in { + ImportJobStatus.PAUSING, + ImportJobStatus.PAUSED, + }: + raise JobPausedError(f"Import job {job_id} was paused.") + phase = str(dict(snapshot or {}).get("phase") or "") + raise ValidationError( + "Import job cannot publish Story Arc placements from " + f"status={status.value}, phase={phase or 'unknown'}." + ) + + +async def _aggregate_origin_work(session: AsyncSession, job_id: int) -> _OriginAggregate: + action_count = invalid_count = 0 + state_counts = {state: 0 for state in StoryArcSyncWorkState} + after_action_id = 0 + while True: + rows = ( + await session.execute( + _origin_work_page_statement( + job_id, + after_action_id=after_action_id, + limit=_ORIGIN_PAGE_SIZE, + ) + ) + ).all() + if not rows: + break + for row in rows: + action_count += 1 + if row.work_state in state_counts: + state_counts[row.work_state] += 1 + if not _origin_row_is_valid(row, job_id=job_id): + invalid_count += 1 + after_action_id = int(rows[-1].action_id) + + return _OriginAggregate( + action_count=action_count, + invalid_count=invalid_count, + counts=ImportStoryArcPlacementCounts( + queued=state_counts[StoryArcSyncWorkState.QUEUED], + running=state_counts[StoryArcSyncWorkState.RUNNING], + retry_wait=state_counts[StoryArcSyncWorkState.RETRY_WAIT], + failed=state_counts[StoryArcSyncWorkState.FAILED], + completed=state_counts[StoryArcSyncWorkState.COMPLETED], + cancelled=state_counts[StoryArcSyncWorkState.CANCELLED], + ), + ) + + +def _origin_work_page_statement(job_id: int, *, after_action_id: int, limit: int) -> Any: + """Build a bounded origin page without casting untrusted JSON in SQL.""" + candidate = or_( + ImportJobAction.phase == _PHASE, + ImportJobAction.action_type == _ACTION_TYPE, + ) + return ( + select( + ImportJobAction.id.label("action_id"), + ImportJobAction.phase.label("action_phase"), + ImportJobAction.action_type.label("action_type"), + ImportJobAction.status.label("action_status"), + ImportJobAction.payload.label("action_payload"), + StoryArcSyncWork.id.label("work_id"), + StoryArcSyncWork.origin_import_action_id.label("work_action_id"), + StoryArcSyncWork.origin_import_job_id.label("work_job_id"), + StoryArcSyncWork.origin_imported_story_arc_id.label("work_imported_arc_id"), + StoryArcSyncWork.origin_imported_story_arc_entry_id.label("work_imported_entry_id"), + StoryArcSyncWork.issue_story_arc_id.label("work_membership_id"), + StoryArcSyncWork.desired_generation.label("work_generation"), + StoryArcSyncWork.state.label("work_state"), + IssueStoryArc.story_arc_id.label("membership_arc_id"), + LibraryFile.issue_id.label("library_issue_id"), + ImportedStoryArc.id.label("staged_arc_id"), + ImportedStoryArc.import_job_id.label("staged_arc_job_id"), + ImportedStoryArc.status.label("staged_arc_status"), + ImportedStoryArc.materialized_story_arc_id.label("staged_materialized_arc_id"), + ImportedStoryArcEntry.id.label("staged_entry_id"), + ImportedStoryArcEntry.imported_story_arc_id.label("staged_entry_arc_id"), + ImportedStoryArcEntry.materialized_membership_id.label("staged_entry_membership_id"), + ImportedStoryArcEntry.matched_issue_id.label("staged_entry_issue_id"), + ImportedStoryArcEntry.resolution_state.label("staged_entry_resolution_state"), + ) + .select_from(ImportJobAction) + .outerjoin( + StoryArcSyncWork, + StoryArcSyncWork.origin_import_action_id == ImportJobAction.id, + ) + .outerjoin( + IssueStoryArc, + IssueStoryArc.id == StoryArcSyncWork.issue_story_arc_id, + ) + .outerjoin( + LibraryFile, + LibraryFile.id == StoryArcSyncWork.library_file_id, + ) + .outerjoin( + ImportedStoryArc, + ImportedStoryArc.id == StoryArcSyncWork.origin_imported_story_arc_id, + ) + .outerjoin( + ImportedStoryArcEntry, + ImportedStoryArcEntry.id == StoryArcSyncWork.origin_imported_story_arc_entry_id, + ) + .where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.id > after_action_id, + candidate, + ) + .order_by(ImportJobAction.id.asc()) + .limit(limit) + ) + + +def _origin_row_is_valid(row: Any, *, job_id: int) -> bool: + """Validate exact typed provenance and the journal payload in Python.""" + values = row._mapping + payload_value = values["action_payload"] + payload = payload_value if isinstance(payload_value, dict) else {} + work_id = _positive_int(values["work_id"]) + membership_id = _positive_int(values["work_membership_id"]) + imported_arc_id = _positive_int(values["work_imported_arc_id"]) + imported_entry_id = _positive_int(values["work_imported_entry_id"]) + generation = values["work_generation"] + return bool( + values["action_phase"] == _PHASE + and values["action_type"] == _ACTION_TYPE + and values["action_status"] is ImportJobActionStatus.COMPLETED + and work_id is not None + and values["work_action_id"] == values["action_id"] + and values["work_job_id"] == job_id + and membership_id is not None + and imported_arc_id is not None + and imported_entry_id is not None + and isinstance(generation, str) + and len(generation) == 64 + and set(payload) == _PAYLOAD_KEYS + and _positive_int(payload.get("schema_version")) == 1 + and _positive_int(payload.get("sync_work_id")) == work_id + and _positive_int(payload.get("membership_id")) == membership_id + and payload.get("desired_generation") == generation + and _positive_int(payload.get("imported_story_arc_id")) == imported_arc_id + and _positive_int(payload.get("imported_story_arc_entry_id")) == imported_entry_id + and _positive_int(payload.get("source_import_job_id")) == job_id + and values["staged_arc_id"] == imported_arc_id + and values["staged_arc_job_id"] == job_id + and values["staged_arc_status"] is ImportedStoryArcStatus.IMPORTED + and values["staged_materialized_arc_id"] == values["membership_arc_id"] + and values["staged_entry_id"] == imported_entry_id + and values["staged_entry_arc_id"] == imported_arc_id + and values["staged_entry_membership_id"] == membership_id + and values["staged_entry_issue_id"] == values["library_issue_id"] + and values["staged_entry_resolution_state"] is StoryArcResolutionState.RESOLVED + ) + + +async def _count_invalid_completed_placements(session: AsyncSession, job_id: int) -> int: + valid_placement = and_( + StoryArcPlacement.source_import_job_id == job_id, + StoryArcPlacement.creating_action_id == ImportJobAction.id, + StoryArcPlacement.issue_story_arc_id == StoryArcSyncWork.issue_story_arc_id, + StoryArcPlacement.library_file_id == StoryArcSyncWork.library_file_id, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.MANAGED, + StoryArcPlacement.mode.in_(_MANAGED_MODES), + StoryArcPlacement.state == StoryArcPlacementState.CURRENT, + StoryArcPlacement.source_kind == StoryArcSourceKind.PULLBOX, + StoryArcPlacement.policy_schema_version == StoryArcSyncWork.policy_schema_version, + StoryArcPlacement.rendered_reading_order == StoryArcSyncWork.membership_sequence, + StoryArcPlacement.operation_token.is_(None), + StoryArcPlacement.last_result["status"].as_string() == "complete", + ) + per_action = ( + select( + ImportJobAction.id.label("action_id"), + func.count(StoryArcPlacement.id).label("placement_count"), + func.coalesce( + func.sum(case((valid_placement, 1), else_=0)), + 0, + ).label("valid_count"), + ) + .select_from(ImportJobAction) + .join( + StoryArcSyncWork, + StoryArcSyncWork.origin_import_action_id == ImportJobAction.id, + ) + .outerjoin( + StoryArcPlacement, + StoryArcPlacement.creating_action_id == ImportJobAction.id, + ) + .where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.phase == _PHASE, + ImportJobAction.action_type == _ACTION_TYPE, + StoryArcSyncWork.origin_import_job_id == job_id, + StoryArcSyncWork.state == StoryArcSyncWorkState.COMPLETED, + ) + .group_by(ImportJobAction.id) + .subquery() + ) + return int( + await session.scalar( + select( + func.coalesce( + func.sum( + case( + ( + or_( + per_action.c.placement_count != 1, + per_action.c.valid_count != 1, + ), + 1, + ), + else_=0, + ) + ), + 0, + ) + ).select_from(per_action) + ) + or 0 + ) + + +async def _mark_pending( + session: AsyncSession, + job: ImportJob, + snapshot: dict[str, object], + counts: ImportStoryArcPlacementCounts, + *, + expected_total: int, +) -> ImportStoryArcPlacementCompletionOutcome: + job.story_arc_placement_followup_pending = False + updated = _with_counts(snapshot, counts, expected_total=expected_total) + updated.update( + { + "status": ImportJobStatus.IMPORTING.value, + "mode": "import", + "phase": _PHASE, + "progress": 99, + "message": "Creating the approved story-arc copies and links...", + } + ) + await _apply_job_state( + session, + job, + status=ImportJobStatus.IMPORTING, + snapshot=updated, + error_message=None, + import_completed_at=None, + ) + return ImportStoryArcPlacementCompletionOutcome( + job_id=job.id, + state=ImportStoryArcPlacementCompletionState.PENDING, + counts=counts, + ) + + +async def _stall( + session: AsyncSession, + job: ImportJob, + snapshot: dict[str, object], + counts: ImportStoryArcPlacementCounts, + *, + expected_total: int | None, + error_code: str, +) -> ImportStoryArcPlacementCompletionOutcome: + job.story_arc_placement_followup_pending = False + message = _ERRORS[error_code] + updated = _with_counts( + snapshot, + counts, + expected_total=expected_total if expected_total is not None else counts.total, + ) + updated.update( + { + "status": ImportJobStatus.STALLED.value, + "mode": "import", + "phase": _PHASE, + "progress": 99, + "message": message, + } + ) + await _apply_job_state( + session, + job, + status=ImportJobStatus.STALLED, + snapshot=updated, + error_message=message, + import_completed_at=None, + ) + return ImportStoryArcPlacementCompletionOutcome( + job_id=job.id, + state=ImportStoryArcPlacementCompletionState.STALLED, + counts=counts, + error_code=error_code, + ) + + +async def _complete( + session: AsyncSession, + job: ImportJob, + snapshot: dict[str, object], + counts: ImportStoryArcPlacementCounts, + *, + completed_at: datetime, +) -> ImportStoryArcPlacementCompletionOutcome: + job.story_arc_placement_followup_pending = True + updated = _with_counts(snapshot, counts, expected_total=counts.total) + updated.update( + { + "status": ImportJobStatus.COMPLETED.value, + "mode": "import", + "phase": "done", + "progress": 100, + "message": "Import completed.", + "story_arc_placement_followup_pending": True, + } + ) + await _apply_job_state( + session, + job, + status=ImportJobStatus.COMPLETED, + snapshot=updated, + error_message=None, + import_completed_at=completed_at, + ) + return ImportStoryArcPlacementCompletionOutcome( + job_id=job.id, + state=ImportStoryArcPlacementCompletionState.COMPLETED, + counts=counts, + ) + + +async def _apply_job_state( + session: AsyncSession, + job: ImportJob, + *, + status: ImportJobStatus, + snapshot: dict[str, object], + error_message: str | None, + import_completed_at: datetime | None, +) -> None: + changed = ( + job.status is not status + or dict(job.progress_snapshot or {}) != snapshot + or job.error_message != error_message + or job.import_completed_at != import_completed_at + ) + if not changed: + return + job.status = status + job.progress_snapshot = snapshot + job.error_message = error_message + job.import_completed_at = import_completed_at + job.progress_revision = int(job.progress_revision or 0) + 1 + await session.flush() + + +def _with_counts( + snapshot: dict[str, object], + counts: ImportStoryArcPlacementCounts, + *, + expected_total: int, +) -> dict[str, object]: + updated = dict(snapshot) + updated["story_arc_placements_total"] = expected_total + for state, count in counts.as_dict().items(): + updated[f"story_arc_placements_{state}"] = count + return updated + + +def _counts_from_snapshot(snapshot: dict[str, object]) -> ImportStoryArcPlacementCounts: + values: dict[str, int] = {} + for state in StoryArcSyncWorkState: + value = snapshot.get(f"story_arc_placements_{state.value}") + values[state.value] = ( + value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else 0 + ) + return ImportStoryArcPlacementCounts(**values) + + +def _positive_int(value: object) -> int | None: + if not isinstance(value, int) or isinstance(value, bool) or value <= 0: + return None + return value diff --git a/src/pullbox/services/import_story_arc_policy_confirmation.py b/src/pullbox/services/import_story_arc_policy_confirmation.py new file mode 100644 index 00000000..214e5fe3 --- /dev/null +++ b/src/pullbox/services/import_story_arc_policy_confirmation.py @@ -0,0 +1,392 @@ +"""Explicit Step 3 confirmation for one staged story-arc policy.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import re +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from sqlalchemy import select + +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.story_arc_naming import ( + DEFAULT_STORY_ARC_FILE_TEMPLATE, + DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + StoryArcNamingValues, + render_story_arc_relative_path, +) +from pullbox.models.import_job import ImportJob, ImportJobStatus +from pullbox.models.story_arc import ImportedStoryArcStatus +from pullbox.models.story_arc_import import ImportedStoryArc +from pullbox.services.story_arc_placement_integration import ( + StoryArcPlacementIntegrationError, + StoryArcPlacementPolicyInput, + StoryArcPlacementPolicyMode, + validate_story_arc_placement_policy_input, +) + +if TYPE_CHECKING: + from collections.abc import Mapping + + from sqlalchemy.ext.asyncio import AsyncSession + +_POLICY_DIGEST_RE = re.compile(r"[0-9a-f]{64}") +_MUTABLE_REVIEW_STATUSES = frozenset( + { + ImportedStoryArcStatus.DETECTED, + ImportedStoryArcStatus.NEEDS_REVIEW, + ImportedStoryArcStatus.READY, + } +) +_SETTING_LABELS = { + "STORYARCDIR": "Separate story-arc directory", + "STORYARC_LOCATION": "Destination", + "COPY2ARCDIR": "Copy to arc directory", + "ARC_FOLDERFORMAT": "Folder format", + "ARC_FILEOPS": "File operation", + "ARC_FILEOPS_SOFTLINK_RELATIVE": "Relative symlinks", + "UPCOMING_STORYARCS": "Include upcoming issues", + "SEARCH_STORYARCS": "Search for missing issues", + "READ2FILENAME": "Reading-order prefix", +} +_REVIEW_WARNING_RE = re.compile(r"[^A-Za-z0-9_.:-]+") + + +@dataclass(frozen=True, slots=True) +class ImportStoryArcPolicyConfirmationResult: + """Safe API-facing result without private destination details.""" + + imported_story_arc_id: int + activation: str + materialize_filesystem: bool + mode: str + monitored: bool + search_missing: bool + include_upcoming: bool + sync_enabled: bool + policy_digest: str + + +@dataclass(frozen=True, slots=True) +class ImportStoryArcSourceSettingReview: + """One sanitized source-setting summary without private path values.""" + + key: str + label: str + value: str + + +@dataclass(frozen=True, slots=True) +class ImportStoryArcPolicyReview: + """Safe Step 3 presentation of one draft or confirmed policy.""" + + policy_digest: str + activation: str + confirmed: bool + warnings: tuple[str, ...] + source_settings: tuple[ImportStoryArcSourceSettingReview, ...] + monitored: bool + search_missing: bool + include_upcoming: bool + materialize_filesystem: bool + mode: str + target_library_root_id: int | None + destination_configured: bool + folder_template: str + file_template: str + symlink_style: str | None + synchronize: bool + example_relative_path: str | None + + +def story_arc_policy_digest(snapshot: Mapping[str, object]) -> str: + """Return the stable optimistic token for one complete JSON policy snapshot.""" + encoded = json.dumps( + dict(snapshot), + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def build_import_story_arc_policy_review( + snapshot: Mapping[str, object], + source_settings_snapshot: Mapping[str, object], + diagnostics: Mapping[str, object] | None = None, +) -> ImportStoryArcPolicyReview: + """Build a bounded, path-redacted review view without source or filesystem I/O.""" + raw = dict(snapshot) + placement = _mapping(raw.get("placement_policy")) + activation = str(raw.get("activation") or "requires_confirmation") + mode = _safe_string_choice( + placement.get("mode"), + {"logical", "reference_only", "copy", "hardlink", "symlink"}, + "logical", + ) + folder_template = _bounded_template( + placement.get("folder_template"), + DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + ) + file_template = _bounded_template( + placement.get("file_template"), + DEFAULT_STORY_ARC_FILE_TEMPLATE, + ) + raw_symlink_style = placement.get("symlink_style") + symlink_style = ( + raw_symlink_style + if isinstance(raw_symlink_style, str) and raw_symlink_style in {"absolute", "relative"} + else None + ) + if mode != "symlink": + symlink_style = None + root_id_raw = placement.get("target_library_root_id") + root_id = ( + root_id_raw + if isinstance(root_id_raw, int) and not isinstance(root_id_raw, bool) and root_id_raw > 0 + else None + ) + destination = placement.get("destination_root") + destination_configured = isinstance(destination, str) and bool(destination.strip()) + warnings = _review_warning_codes(raw, source_settings_snapshot, diagnostics or {}) + return ImportStoryArcPolicyReview( + policy_digest=story_arc_policy_digest(raw), + activation=activation, + confirmed=activation == "confirmed", + warnings=warnings, + source_settings=_source_setting_reviews(source_settings_snapshot), + monitored=raw.get("monitored") is True, + search_missing=raw.get("search_missing") is True, + include_upcoming=raw.get("include_upcoming") is True, + materialize_filesystem=mode != "logical", + mode=mode, + target_library_root_id=root_id, + destination_configured=destination_configured, + folder_template=folder_template, + file_template=file_template, + symlink_style=symlink_style, + synchronize=placement.get("synchronize") is True, + example_relative_path=_policy_example(folder_template, file_template), + ) + + +async def confirm_import_story_arc_policy( + session: AsyncSession, + *, + job_id: int, + imported_story_arc_id: int, + expected_policy_digest: str, + explicit_confirmation: bool, + materialize_filesystem: bool, + monitored: bool, + search_missing: bool, + include_upcoming: bool, + placement_policy: StoryArcPlacementPolicyInput, +) -> ImportStoryArcPolicyConfirmationResult: + """Validate and freeze one staged policy without creating arcs or files.""" + if explicit_confirmation is not True: + raise ValidationError("You must explicitly confirm this story arc policy") + if not isinstance(expected_policy_digest, str) or not _POLICY_DIGEST_RE.fullmatch( + expected_policy_digest + ): + raise ValidationError("Story arc policy review token is invalid") + if not all( + isinstance(value, bool) + for value in ( + materialize_filesystem, + monitored, + search_missing, + include_upcoming, + ) + ): + raise ValidationError("Story arc policy choices must be true or false") + if not monitored and (search_missing or include_upcoming): + raise ValidationError( + "Story arc search and upcoming automation require monitoring to be enabled" + ) + + job = await session.scalar(select(ImportJob).where(ImportJob.id == job_id).with_for_update()) + if job is None: + raise NotFoundError("ImportJob", job_id) + if job.status != ImportJobStatus.REVIEW or job.import_started_at is not None: + raise ValidationError("Job must be in REVIEW state to confirm story arc policy") + + staged_arc = await session.scalar( + select(ImportedStoryArc) + .where( + ImportedStoryArc.id == imported_story_arc_id, + ImportedStoryArc.import_job_id == job_id, + ) + .with_for_update() + ) + if staged_arc is None: + raise NotFoundError("ImportedStoryArc", imported_story_arc_id) + if staged_arc.status not in _MUTABLE_REVIEW_STATUSES: + raise ValidationError("Story arc policy can only change during active Step 3 review") + + current_snapshot = dict(staged_arc.proposed_policy_snapshot or {}) + current_source = current_snapshot.get("source") + if current_source is not None and current_source != staged_arc.source_kind.value: + raise ValidationError("Staged story arc policy source does not match its evidence") + current_digest = story_arc_policy_digest(current_snapshot) + if not hmac.compare_digest(current_digest, expected_policy_digest): + raise ValidationError("Story arc policy changed; review the latest draft and try again") + + logical = _proposal_is_logical(placement_policy) + if materialize_filesystem == logical: + raise ValidationError( + "Story arc filesystem materialization choice does not match the placement policy" + ) + + try: + validated_policy = await validate_story_arc_placement_policy_input( + session, + placement_policy, + revision=1, + ) + except StoryArcPlacementIntegrationError as exc: + raise ValidationError(str(exc)) from exc + + canonical_snapshot = validated_policy.snapshot + if materialize_filesystem is False and canonical_snapshot["mode"] != "logical": + raise ValidationError("Logical story arc import cannot activate a filesystem policy") + if materialize_filesystem is True and canonical_snapshot["mode"] == "logical": + raise ValidationError("Filesystem materialization requires a placement mode") + + durable_warnings = _review_warning_codes( + current_snapshot, + staged_arc.source_settings_snapshot or {}, + staged_arc.diagnostics or {}, + ) + confirmed_snapshot: dict[str, object] = { + "schema_version": 1, + "source": staged_arc.source_kind.value, + "activation": "confirmed", + "monitored": monitored, + "search_missing": search_missing, + "include_upcoming": include_upcoming, + "sync_enabled": validated_policy.synchronize, + "placement_policy": canonical_snapshot, + } + diagnostics = dict(staged_arc.diagnostics or {}) + diagnostics["story_arc_policy_review"] = { + "schema_version": 1, + "warning_codes": list(durable_warnings), + } + staged_arc.diagnostics = diagnostics + staged_arc.proposed_policy_snapshot = confirmed_snapshot + await session.flush() + return ImportStoryArcPolicyConfirmationResult( + imported_story_arc_id=int(staged_arc.id), + activation="confirmed", + materialize_filesystem=materialize_filesystem, + mode=validated_policy.mode.value, + monitored=monitored, + search_missing=search_missing, + include_upcoming=include_upcoming, + sync_enabled=validated_policy.synchronize, + policy_digest=story_arc_policy_digest(confirmed_snapshot), + ) + + +def _proposal_is_logical(policy: StoryArcPlacementPolicyInput) -> bool: + try: + return StoryArcPlacementPolicyMode(policy.mode) is StoryArcPlacementPolicyMode.LOGICAL + except ValueError: + return False + + +def _mapping(value: object) -> Mapping[str, object]: + if isinstance(value, dict): + return value + return {} + + +def _safe_string_choice(value: object, allowed: set[str], default: str) -> str: + return value if isinstance(value, str) and value in allowed else default + + +def _bounded_template(value: object, default: str) -> str: + if not isinstance(value, str) or not value or len(value.encode("utf-8")) > 1024: + return default + return value + + +def _review_warning_codes( + snapshot: Mapping[str, object], + source_settings_snapshot: Mapping[str, object], + diagnostics: Mapping[str, object], +) -> tuple[str, ...]: + values: list[object] = [] + draft_warnings = snapshot.get("review_warnings") + if isinstance(draft_warnings, list | tuple): + values.extend(draft_warnings[:20]) + parse_warnings = source_settings_snapshot.get("parse_warnings") + if isinstance(parse_warnings, list | tuple): + values.extend(parse_warnings[:20]) + durable_review = _mapping(diagnostics.get("story_arc_policy_review")) + durable_warnings = durable_review.get("warning_codes") + if isinstance(durable_warnings, list | tuple): + values.extend(durable_warnings[:20]) + result: list[str] = [] + for value in values: + normalized = _REVIEW_WARNING_RE.sub("_", str(value)[:200]).strip("_") + if normalized and normalized not in result: + result.append(normalized) + return tuple(result[:20]) + + +def _source_setting_reviews( + source_settings_snapshot: Mapping[str, object], +) -> tuple[ImportStoryArcSourceSettingReview, ...]: + raw_values = _mapping(source_settings_snapshot.get("values")) + reviews: list[ImportStoryArcSourceSettingReview] = [] + for key, label in _SETTING_LABELS.items(): + setting = _mapping(raw_values.get(key)) + if not setting: + continue + value = setting.get("value") + if key == "STORYARC_LOCATION": + display = "Configured" if isinstance(value, str) and bool(value.strip()) else "Not set" + elif isinstance(value, bool): + display = "Enabled" if value else "Disabled" + elif key == "ARC_FILEOPS" and isinstance(value, str): + normalized = value.strip().casefold() + display = ( + normalized + if normalized in {"copy", "move", "hardlink", "softlink"} + else "Needs review" + ) + elif key == "ARC_FOLDERFORMAT" and isinstance(value, str): + display = "Detected" if value.strip() else "Not set" + else: + display = "Detected" if value is not None else "Not set" + reviews.append(ImportStoryArcSourceSettingReview(key=key, label=label, value=display)) + return tuple(reviews) + + +def _policy_example(folder_template: str, file_template: str) -> str | None: + try: + rendered = render_story_arc_relative_path( + StoryArcNamingValues( + story_arc="Example Arc", + reading_order=1, + series="Example Series", + publisher="Example Publisher", + issue_number="1", + issue_title="Example Issue", + year=2026, + start_year=2025, + end_year=2026, + extension="cbz", + original_filename="Example Series 001.cbz", + ), + folder_template=folder_template, + file_template=file_template, + ) + except ValueError: + return None + return str(rendered) diff --git a/src/pullbox/services/import_story_arc_preflight.py b/src/pullbox/services/import_story_arc_preflight.py new file mode 100644 index 00000000..181ef988 --- /dev/null +++ b/src/pullbox/services/import_story_arc_preflight.py @@ -0,0 +1,307 @@ +"""Bounded, read-only Story Arc evidence analysis for Import Step 1.""" + +from __future__ import annotations + +import asyncio +import os +import time +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from pullbox.core.collection_scanner import COMIC_EXTENSIONS, IGNORE_DIRS +from pullbox.core.filesystem_policy import is_sensitive_path, resolve_preview_source +from pullbox.core.mylar3_reader import Mylar3Reader +from pullbox.core.mylar_story_arc_policy import build_mylar_story_arc_policy_draft +from pullbox.core.naming import parse_filename +from pullbox.core.story_arc_naming import ( + DEFAULT_STORY_ARC_FILE_TEMPLATE, + DEFAULT_STORY_ARC_FOLDER_TEMPLATE, +) +from pullbox.core.story_arc_ordering import extract_story_arc_order_prefix +from pullbox.models.import_job import ImportSourceType +from pullbox.schemas.import_story_arc_preflight import ( + StoryArcEvidenceExample, + StoryArcPolicyPreview, + StoryArcPreflightResponse, + StoryArcResolutionPreview, + StoryArcSettingPreview, +) + + +@dataclass(frozen=True, slots=True) +class StoryArcPreflightBudget: + """Server-owned path limits for a Step 1 folder sample.""" + + max_directories: int = 2_000 + max_files: int = 5_000 + max_examples: int = 5 + deadline_seconds: float = 2.0 + + +@dataclass(frozen=True, slots=True) +class _FolderCandidate: + folder: str + examples: tuple[StoryArcEvidenceExample, ...] + entry_count: int + duplicate_count: int + + +class StoryArcPreflightAnalyzer: + """Expose source evidence without creating jobs, calling providers, or writing files.""" + + async def analyze( + self, + source_path: str | Path, + *, + source_type: ImportSourceType, + budget: StoryArcPreflightBudget | None = None, + ) -> StoryArcPreflightResponse: + """Return a typed Mylar or folder preflight response.""" + path = resolve_preview_source(source_path) + if source_type is ImportSourceType.MYLAR3: + database = path / "mylar.db" if path.is_dir() else path + # Validate the target without relocating the selected folder's config.ini. + resolve_preview_source(database) + return await self._analyze_mylar(database) + if not path.is_dir(): + raise ValueError("Filesystem Story Arc analysis requires a directory") + return await self._analyze_folder(path, budget or StoryArcPreflightBudget()) + + async def _analyze_mylar(self, database: Path) -> StoryArcPreflightResponse: + reader = Mylar3Reader(database) + snapshot = await reader.read_story_arc_preflight() + draft = build_mylar_story_arc_policy_draft(snapshot.arc_settings) + placement_value = draft.get("placement_policy") + placement = ( + cast("dict[str, object]", placement_value) if isinstance(placement_value, dict) else {} + ) + configured_settings = tuple( + StoryArcSettingPreview( + key=setting.key, + value=self._safe_setting_value(setting.key, setting.value), + used_default=setting.used_default, + ) + for setting in snapshot.arc_settings.values + if not setting.used_default + ) + examples = [StoryArcEvidenceExample.model_validate(item) for item in snapshot.examples] + missing = min(snapshot.missing_count, snapshot.entries_count) + evidence_detected = snapshot.arcs_count > 0 + warnings = list(snapshot.warnings) + review_warnings = draft.get("review_warnings") + if isinstance(review_warnings, list): + warnings.extend(str(item) for item in review_warnings if item) + return StoryArcPreflightResponse( + source_type=ImportSourceType.MYLAR3, + evidence_detected=evidence_detected, + arcs_detected=snapshot.arcs_count, + entries_detected=snapshot.entries_count, + resolution=StoryArcResolutionPreview( + pending=max(snapshot.entries_count - missing, 0), + missing=missing, + duplicates=snapshot.duplicate_count, + ), + existing_arc_files_detected=snapshot.existing_location_count > 0, + existing_arc_folders_detected=snapshot.existing_location_count > 0, + pattern_summary=( + "Mylar Story Arc rows and saved ordering" if evidence_detected else None + ), + settings=list(configured_settings), + examples=examples, + provider_calls_required=False, + provider_call_summary="No provider calls are needed for trusted Mylar data.", + proposed_policy=StoryArcPolicyPreview( + mode=str(placement.get("mode") or "logical"), + destination_root_configured=bool(placement.get("destination_root")), + folder_template=str( + placement.get("folder_template") or DEFAULT_STORY_ARC_FOLDER_TEMPLATE + ), + file_template=str( + placement.get("file_template") or DEFAULT_STORY_ARC_FILE_TEMPLATE + ), + reading_order_prefix="{ReadingOrder" in str(placement.get("file_template") or ""), + synchronize=bool(placement.get("synchronize")), + ), + readlist_present=snapshot.readlist_present, + readlist_count=snapshot.readlist_count, + readlist_import_state=("deferred_v1.5.0" if snapshot.readlist_present else None), + warnings=self._unique(warnings), + ) + + async def _analyze_folder( + self, + root: Path, + budget: StoryArcPreflightBudget, + ) -> StoryArcPreflightResponse: + if ( + budget.max_directories < 1 + or budget.max_files < 1 + or budget.max_examples < 1 + or budget.deadline_seconds < 0 + ): + raise ValueError("Story Arc preflight budget values are invalid") + candidates, partial, warnings = await asyncio.to_thread( + self._scan_folder_candidates, + root, + budget, + ) + entries = sum(candidate.entry_count for candidate in candidates) + duplicates = sum(candidate.duplicate_count for candidate in candidates) + examples = [example for candidate in candidates for example in candidate.examples][ + : budget.max_examples + ] + detected = bool(candidates) + if detected: + warnings.append("full_scan_may_find_additional_comicinfo_evidence") + return StoryArcPreflightResponse( + source_type=ImportSourceType.FILESYSTEM, + evidence_detected=detected, + arcs_detected=len(candidates), + entries_detected=entries, + resolution=StoryArcResolutionPreview( + pending=entries, + duplicates=duplicates, + ), + existing_arc_files_detected=detected, + existing_arc_folders_detected=detected, + pattern_summary=("Reading-order prefixes across multiple series" if detected else None), + examples=examples, + provider_calls_required=True, + provider_call_summary=( + "The normal matching workflow may use providers after local evidence is exhausted." + ), + proposed_policy=StoryArcPolicyPreview( + mode="reference_only" if detected else "logical", + destination_root_configured=detected, + folder_template=DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + file_template=DEFAULT_STORY_ARC_FILE_TEMPLATE, + reading_order_prefix=True, + synchronize=False, + ), + archive_probes=0, + partial=partial or detected, + warnings=self._unique(warnings), + ) + + def _scan_folder_candidates( + self, + root: Path, + budget: StoryArcPreflightBudget, + ) -> tuple[list[_FolderCandidate], bool, list[str]]: + deadline = time.monotonic() + budget.deadline_seconds + directories = 0 + files = 0 + partial = False + warnings: list[str] = [] + groups: dict[str, list[StoryArcEvidenceExample]] = {} + orders: dict[str, list[int]] = {} + series: dict[str, set[str]] = {} + comic_counts: dict[str, int] = {} + + def on_error(_error: OSError) -> None: + nonlocal partial + partial = True + warnings.append("unreadable_path_skipped") + + for current_root, dir_names, file_names in os.walk( + root, + topdown=True, + onerror=on_error, + followlinks=False, + ): + if time.monotonic() >= deadline: + partial = True + warnings.append("deadline_reached") + break + if directories >= budget.max_directories: + partial = True + warnings.append("directory_limit_reached") + break + current = Path(current_root) + if is_sensitive_path(current): + dir_names.clear() + partial = True + warnings.append("sensitive_directory_skipped") + continue + if any(is_sensitive_path(current / name) for name in dir_names): + partial = True + warnings.append("sensitive_directory_skipped") + dir_names[:] = [ + name + for name in sorted(dir_names) + if name not in IGNORE_DIRS + and not name.startswith(".") + and not (current / name).is_symlink() + and not is_sensitive_path(current / name) + ] + directories += 1 + relative_folder = current.relative_to(root).as_posix() + for file_name in sorted(file_names): + path = current / file_name + if path.is_symlink() or path.suffix.lower() not in COMIC_EXTENSIONS: + continue + if files >= budget.max_files: + partial = True + warnings.append("file_limit_reached") + break + files += 1 + comic_counts[relative_folder] = comic_counts.get(relative_folder, 0) + 1 + prefix = extract_story_arc_order_prefix(file_name) + if prefix is None: + continue + parsed = parse_filename(prefix.residual_file_name) + parsed_series = parsed.series if parsed is not None else None + relative_path = path.relative_to(root).as_posix() + groups.setdefault(relative_folder, []).append( + StoryArcEvidenceExample( + story_arc=current.name, + series=parsed_series, + issue_number=( + str(int(parsed.issue_number)) + if parsed is not None and parsed.issue_number.is_integer() + else str(parsed.issue_number) + if parsed is not None + else None + ), + issue_title=None, + reading_order=prefix.reading_order_raw, + status=None, + relative_path=relative_path, + ) + ) + orders.setdefault(relative_folder, []).append(prefix.reading_order) + if parsed_series: + series.setdefault(relative_folder, set()).add(parsed_series.casefold()) + if partial and "file_limit_reached" in warnings: + break + + candidates: list[_FolderCandidate] = [] + for folder in sorted(groups): + folder_examples = groups[folder] + if ( + len(folder_examples) < 2 + or len(folder_examples) != comic_counts.get(folder, 0) + or len(series.get(folder, set())) < 2 + ): + continue + folder_orders = orders.get(folder, []) + candidates.append( + _FolderCandidate( + folder=folder, + examples=tuple(folder_examples[: budget.max_examples]), + entry_count=len(folder_examples), + duplicate_count=len(folder_orders) - len(set(folder_orders)), + ) + ) + return candidates, partial, self._unique(warnings) + + def _safe_setting_value(self, key: str, value: bool | str | None) -> bool | str | None: + if key == "STORYARC_LOCATION": + return "Configured" if isinstance(value, str) and value.strip() else "Not configured" + if isinstance(value, str): + return " ".join(value.split())[:200] + return value + + def _unique(self, values: list[str]) -> list[str]: + return list(dict.fromkeys(value for value in values if value)) diff --git a/src/pullbox/services/import_story_arc_resolution.py b/src/pullbox/services/import_story_arc_resolution.py new file mode 100644 index 00000000..5197ca76 --- /dev/null +++ b/src/pullbox/services/import_story_arc_resolution.py @@ -0,0 +1,539 @@ +"""Resolve staged story-arc entries from trusted local import evidence.""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from sqlalchemy import or_, select + +from pullbox.models.import_job import ImportedFile, ImportedFileStatus +from pullbox.models.issue import Issue +from pullbox.models.story_arc import StoryArcResolutionState +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +CancellationCheck = Callable[[], Awaitable[None]] +DurableCheckpoint = Callable[[], Awaitable[None]] + + +@dataclass(frozen=True, slots=True) +class StoryArcResolutionResult: + """Bounded reconciliation counts for review and diagnostics.""" + + entries_examined: int = 0 + resolved: int = 0 + pending: int = 0 + missing: int = 0 + ambiguous: int = 0 + conflicts: int = 0 + skipped: int = 0 + linked_files: int = 0 + + +@dataclass(frozen=True, slots=True) +class _ResolutionIndexes: + files_by_id: dict[int, ImportedFile] + files_by_path: dict[str, tuple[ImportedFile, ...]] + files_by_cv_id: dict[int, tuple[ImportedFile, ...]] + issues_by_id: dict[int, Issue] + issues_by_cv_id: dict[int, Issue] + + +async def resolve_staged_story_arc_entries( + session: AsyncSession, + *, + import_job_id: int, + batch_size: int = 200, + cancellation_check: CancellationCheck | None = None, + durable_checkpoint: DurableCheckpoint | None = None, +) -> StoryArcResolutionResult: + """Reconcile staged entries without providers, source I/O, or canonical writes. + + Exact staged/import associations and trusted ComicVine issue identities are + authoritative. Names, titles, order, and issue-number similarity are never + used to manufacture a match. The runtime supplies a durable checkpoint that + commits each flushed page and rechecks job control before the next page. A + caller that omits it retains ownership of the surrounding transaction. + """ + if batch_size <= 0: + msg = "Story-arc resolution batch size must be positive." + raise ValueError(msg) + + await _checkpoint(cancellation_check) + last_entry_id = 0 + counts = {state: 0 for state in StoryArcResolutionState} + entries_examined = 0 + linked_files = 0 + + while True: + entries = list( + ( + await session.scalars( + select(ImportedStoryArcEntry) + .join( + ImportedStoryArc, + ImportedStoryArc.id == ImportedStoryArcEntry.imported_story_arc_id, + ) + .where( + ImportedStoryArc.import_job_id == import_job_id, + ImportedStoryArcEntry.id > last_entry_id, + ) + .order_by(ImportedStoryArcEntry.id) + .limit(batch_size) + ) + ).all() + ) + if not entries: + break + + await _checkpoint(cancellation_check) + indexes = await _load_resolution_indexes( + session, + import_job_id=import_job_id, + entries=entries, + ) + for entry in entries: + previous_file_id = entry.import_file_id + _resolve_entry(entry, indexes) + if previous_file_id is None and entry.import_file_id is not None: + linked_files += 1 + counts[entry.resolution_state] += 1 + entries_examined += 1 + + await session.flush() + if durable_checkpoint is not None: + await durable_checkpoint() + last_entry_id = entries[-1].id + + return StoryArcResolutionResult( + entries_examined=entries_examined, + resolved=counts[StoryArcResolutionState.RESOLVED], + pending=counts[StoryArcResolutionState.PENDING], + missing=counts[StoryArcResolutionState.MISSING], + ambiguous=counts[StoryArcResolutionState.AMBIGUOUS], + conflicts=counts[StoryArcResolutionState.CONFLICT], + skipped=counts[StoryArcResolutionState.SKIPPED], + linked_files=linked_files, + ) + + +async def refresh_story_arc_entries_for_import_files( + session: AsyncSession, + *, + import_job_id: int, + import_file_ids: Sequence[int], +) -> int: + """Refresh derived arc state after authoritative import-file mutations.""" + normalized_file_ids = tuple(dict.fromkeys(int(value) for value in import_file_ids)) + if not normalized_file_ids: + return 0 + + entries = list( + ( + await session.scalars( + select(ImportedStoryArcEntry) + .join( + ImportedStoryArc, + ImportedStoryArc.id == ImportedStoryArcEntry.imported_story_arc_id, + ) + .where( + ImportedStoryArc.import_job_id == import_job_id, + ImportedStoryArcEntry.import_file_id.in_(normalized_file_ids), + ) + .order_by(ImportedStoryArcEntry.id) + ) + ).all() + ) + if not entries: + return 0 + + indexes = await _load_resolution_indexes( + session, + import_job_id=import_job_id, + entries=entries, + ) + for entry in entries: + _resolve_entry(entry, indexes) + + await _refresh_arc_safety_diagnostics( + session, + arc_ids={int(entry.imported_story_arc_id) for entry in entries}, + ) + await session.flush() + return len(entries) + + +async def _load_resolution_indexes( + session: AsyncSession, + *, + import_job_id: int, + entries: Sequence[ImportedStoryArcEntry], +) -> _ResolutionIndexes: + direct_file_ids = { + entry.import_file_id for entry in entries if entry.import_file_id is not None + } + source_locations = { + entry.source_location for entry in entries if entry.source_location is not None + } + source_cv_ids = { + source_cv_id + for entry in entries + if (source_cv_id := _parse_provider_id(entry.source_issue_id)) is not None + } + + file_filters = [] + if direct_file_ids: + file_filters.append(ImportedFile.id.in_(direct_file_ids)) + if source_locations: + file_filters.append(ImportedFile.file_path.in_(source_locations)) + if source_cv_ids: + file_filters.extend( + ( + ImportedFile.comicvine_issue_id.in_(source_cv_ids), + ImportedFile.matched_issue_cv_id.in_(source_cv_ids), + ) + ) + + files: list[ImportedFile] = [] + if file_filters: + files = list( + ( + await session.scalars( + select(ImportedFile) + .where( + ImportedFile.import_job_id == import_job_id, + or_(*file_filters), + ) + .order_by(ImportedFile.id) + ) + ).all() + ) + + issue_ids = {entry.matched_issue_id for entry in entries if entry.matched_issue_id is not None} + issue_ids.update(item.matched_issue_id for item in files if item.matched_issue_id is not None) + issue_cv_ids = set(source_cv_ids) + issue_cv_ids.update( + item.matched_issue_cv_id for item in files if item.matched_issue_cv_id is not None + ) + issue_cv_ids.update( + item.comicvine_issue_id for item in files if item.comicvine_issue_id is not None + ) + + issue_filters = [] + if issue_ids: + issue_filters.append(Issue.id.in_(issue_ids)) + if issue_cv_ids: + issue_filters.append(Issue.comicvine_id.in_(issue_cv_ids)) + issues: list[Issue] = [] + if issue_filters: + issues = list((await session.scalars(select(Issue).where(or_(*issue_filters)))).all()) + + files_by_path_mutable: defaultdict[str, list[ImportedFile]] = defaultdict(list) + files_by_cv_id_mutable: defaultdict[int, list[ImportedFile]] = defaultdict(list) + for item in files: + files_by_path_mutable[item.file_path].append(item) + for provider_id in _trusted_file_provider_ids(item): + files_by_cv_id_mutable[provider_id].append(item) + + return _ResolutionIndexes( + files_by_id={item.id: item for item in files}, + files_by_path={key: tuple(value) for key, value in files_by_path_mutable.items()}, + files_by_cv_id={key: tuple(value) for key, value in files_by_cv_id_mutable.items()}, + issues_by_id={issue.id: issue for issue in issues}, + issues_by_cv_id={ + issue.comicvine_id: issue for issue in issues if issue.comicvine_id is not None + }, + ) + + +def _resolve_entry(entry: ImportedStoryArcEntry, indexes: _ResolutionIndexes) -> None: + if entry.resolution_state == StoryArcResolutionState.SKIPPED: + return + + source_cv_id = _parse_provider_id(entry.source_issue_id) + existing_issue = ( + indexes.issues_by_id.get(entry.matched_issue_id) + if entry.matched_issue_id is not None + else None + ) + if existing_issue is not None: + if _issue_conflicts_with_source(existing_issue, source_cv_id): + _mark_review( + entry, + StoryArcResolutionState.CONFLICT, + "conflicting_exact_issue_identity", + ) + return + _mark_resolved(entry, existing_issue, method="existing_staged_issue") + return + + candidate, candidate_conflict = _select_candidate_file( + entry, + source_cv_id=source_cv_id, + indexes=indexes, + ) + if candidate_conflict: + _mark_review( + entry, + StoryArcResolutionState.CONFLICT, + "conflicting_exact_issue_identity", + ) + return + + if candidate is not None: + entry.import_file_id = candidate.id + if candidate.status == ImportedFileStatus.SKIPPED: + _mark_skipped(entry, reason="source_file_skipped") + return + if candidate.status == ImportedFileStatus.CONFLICT: + _mark_review( + entry, + StoryArcResolutionState.CONFLICT, + "source_file_identity_conflict", + ) + return + if candidate.status == ImportedFileStatus.SAFETY_BLOCKED: + _mark_safety_review( + entry, + candidate, + ) + return + + stale_safety_review = _review_reason(entry) == "source_file_safety_blocked" + _clear_current_safety_diagnostics(entry) + if entry.resolution_state == StoryArcResolutionState.AMBIGUOUS and stale_safety_review: + entry.resolution_state = StoryArcResolutionState.PENDING + + candidate_issue = ( + indexes.issues_by_id.get(candidate.matched_issue_id) + if candidate.matched_issue_id is not None + else None + ) + if candidate_issue is not None: + if _issue_conflicts_with_source(candidate_issue, source_cv_id): + _mark_review( + entry, + StoryArcResolutionState.CONFLICT, + "conflicting_exact_issue_identity", + ) + return + method = ( + "linked_import_file" + if entry.import_file_id == candidate.id and source_cv_id is None + else "exact_source_issue_id" + ) + _mark_resolved(entry, candidate_issue, method=method) + return + + candidate_cv_id = candidate.matched_issue_cv_id or candidate.comicvine_issue_id + if candidate_cv_id is not None: + issue_by_file_identity = indexes.issues_by_cv_id.get(candidate_cv_id) + if issue_by_file_identity is not None: + _mark_resolved(entry, issue_by_file_identity, method="exact_import_file_identity") + return + + if source_cv_id is not None: + issue_by_source_identity = indexes.issues_by_cv_id.get(source_cv_id) + if issue_by_source_identity is not None: + _mark_resolved(entry, issue_by_source_identity, method="exact_source_issue_id") + return + + if entry.resolution_state not in { + StoryArcResolutionState.MISSING, + StoryArcResolutionState.AMBIGUOUS, + StoryArcResolutionState.CONFLICT, + }: + entry.resolution_state = StoryArcResolutionState.PENDING + + +def _select_candidate_file( + entry: ImportedStoryArcEntry, + *, + source_cv_id: int | None, + indexes: _ResolutionIndexes, +) -> tuple[ImportedFile | None, bool]: + if entry.import_file_id is not None: + direct = indexes.files_by_id.get(entry.import_file_id) + if direct is not None: + return direct, _file_conflicts_with_source(direct, source_cv_id) + + if entry.source_location is not None: + by_path = indexes.files_by_path.get(entry.source_location, ()) + selected, conflict = _select_convergent_file(by_path) + if conflict: + return None, True + if selected is not None: + return selected, _file_conflicts_with_source(selected, source_cv_id) + + if source_cv_id is not None: + return _select_convergent_file(indexes.files_by_cv_id.get(source_cv_id, ())) + return None, False + + +def _select_convergent_file( + candidates: Sequence[ImportedFile], +) -> tuple[ImportedFile | None, bool]: + if not candidates: + return None, False + matched_issue_ids = { + candidate.matched_issue_id + for candidate in candidates + if candidate.matched_issue_id is not None + } + matched_cv_ids = { + candidate.matched_issue_cv_id + for candidate in candidates + if candidate.matched_issue_cv_id is not None + } + if len(matched_issue_ids) > 1 or len(matched_cv_ids) > 1: + return None, True + return min(candidates, key=lambda item: item.id), False + + +def _file_conflicts_with_source(item: ImportedFile, source_cv_id: int | None) -> bool: + trusted_ids = _trusted_file_provider_ids(item) + return source_cv_id is not None and bool(trusted_ids) and source_cv_id not in trusted_ids + + +def _issue_conflicts_with_source(issue: Issue, source_cv_id: int | None) -> bool: + return ( + source_cv_id is not None + and issue.comicvine_id is not None + and issue.comicvine_id != source_cv_id + ) + + +def _trusted_file_provider_ids(item: ImportedFile) -> set[int]: + return { + value for value in (item.comicvine_issue_id, item.matched_issue_cv_id) if value is not None + } + + +def _mark_resolved(entry: ImportedStoryArcEntry, issue: Issue, *, method: str) -> None: + entry.matched_issue_id = issue.id + entry.resolution_state = StoryArcResolutionState.RESOLVED + entry.resolution_confidence = 1.0 + entry.resolution_method = method + diagnostics = dict(entry.diagnostics or {}) + diagnostics.pop("review_reason", None) + _archive_and_remove_safety_code(diagnostics) + diagnostics["resolution_evidence"] = "trusted_local_exact_identity" + entry.diagnostics = diagnostics + + +def _mark_review( + entry: ImportedStoryArcEntry, + state: StoryArcResolutionState, + reason: str, +) -> None: + entry.matched_issue_id = None + entry.resolution_state = state + entry.resolution_confidence = None + entry.resolution_method = None + diagnostics = dict(entry.diagnostics or {}) + diagnostics["review_reason"] = reason + entry.diagnostics = diagnostics + + +def _mark_safety_review(entry: ImportedStoryArcEntry, item: ImportedFile) -> None: + _mark_review( + entry, + StoryArcResolutionState.AMBIGUOUS, + "source_file_safety_blocked", + ) + diagnostics = dict(entry.diagnostics or {}) + safety_block = item.diagnostics.get("safety_block") if item.diagnostics else None + safety_code = safety_block.get("code") if isinstance(safety_block, dict) else None + if isinstance(safety_code, str) and safety_code.strip(): + diagnostics["safety_code"] = safety_code.strip() + entry.diagnostics = diagnostics + + +def _mark_skipped(entry: ImportedStoryArcEntry, *, reason: str) -> None: + entry.matched_issue_id = None + entry.resolution_state = StoryArcResolutionState.SKIPPED + entry.resolution_confidence = None + entry.resolution_method = None + entry.selected_for_import = False + diagnostics = dict(entry.diagnostics or {}) + _archive_and_remove_safety_code(diagnostics) + diagnostics["review_reason"] = reason + entry.diagnostics = diagnostics + + +def _clear_current_safety_diagnostics(entry: ImportedStoryArcEntry) -> None: + diagnostics = dict(entry.diagnostics or {}) + _archive_and_remove_safety_code(diagnostics) + if diagnostics.get("review_reason") == "source_file_safety_blocked": + diagnostics.pop("review_reason", None) + entry.diagnostics = diagnostics + + +def _archive_and_remove_safety_code(diagnostics: dict[str, object]) -> None: + safety_code = diagnostics.pop("safety_code", None) + if isinstance(safety_code, str) and safety_code.strip(): + diagnostics.setdefault("scan_safety_code", safety_code.strip()) + + +def _review_reason(entry: ImportedStoryArcEntry) -> str | None: + reason = (entry.diagnostics or {}).get("review_reason") + return reason if isinstance(reason, str) else None + + +async def _refresh_arc_safety_diagnostics( + session: AsyncSession, + *, + arc_ids: set[int], +) -> None: + if not arc_ids: + return + blocked_arc_ids = { + int(arc_id) + for arc_id in ( + await session.scalars( + select(ImportedStoryArcEntry.imported_story_arc_id) + .join(ImportedFile, ImportedFile.id == ImportedStoryArcEntry.import_file_id) + .where( + ImportedStoryArcEntry.imported_story_arc_id.in_(arc_ids), + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + ) + .distinct() + ) + ).all() + } + arcs = list( + ( + await session.scalars(select(ImportedStoryArc).where(ImportedStoryArc.id.in_(arc_ids))) + ).all() + ) + for arc in arcs: + diagnostics = dict(arc.diagnostics or {}) + if diagnostics.get("safety_incomplete") is True: + diagnostics.setdefault("scan_safety_incomplete", True) + if diagnostics.get("safety_blocked") is True: + diagnostics.setdefault("scan_safety_blocked", True) + has_current_block = int(arc.id) in blocked_arc_ids + diagnostics["safety_incomplete"] = has_current_block + diagnostics["safety_blocked"] = has_current_block + arc.diagnostics = diagnostics + + +def _parse_provider_id(value: str | None) -> int | None: + if value is None: + return None + normalized = value.strip() + if not normalized.isdecimal(): + return None + parsed = int(normalized) + return parsed if 0 < parsed <= 2**63 - 1 else None + + +async def _checkpoint(callback: CancellationCheck | None) -> None: + if callback is not None: + await callback() diff --git a/src/pullbox/services/import_story_arc_review.py b/src/pullbox/services/import_story_arc_review.py new file mode 100644 index 00000000..98454ec3 --- /dev/null +++ b/src/pullbox/services/import_story_arc_review.py @@ -0,0 +1,688 @@ +"""Step 3 review queries and decisions for staged story arcs.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal + +from sqlalchemy import func, or_, select +from sqlalchemy.orm import selectinload + +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.core.story_arc_naming import ( + DEFAULT_STORY_ARC_FILE_TEMPLATE, + DEFAULT_STORY_ARC_FOLDER_TEMPLATE, +) +from pullbox.models.import_job import ImportedFile, ImportedFileStatus, ImportJob, ImportJobStatus +from pullbox.models.story_arc import ( + ImportedStoryArcStatus, + StoryArc, + StoryArcLifecycle, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.services.import_story_arc_policy_confirmation import ( + ImportStoryArcPolicyReview, + build_import_story_arc_policy_review, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +@dataclass(frozen=True, slots=True) +class StoryArcMergeCandidate: + """One bounded exact-name target offered for an explicit merge.""" + + id: int + name: str + + +@dataclass(frozen=True, slots=True) +class ImportedStoryArcReviewRow: + """Bounded Step 3 presentation row for one staged story arc.""" + + id: int + name: str + source_kind: StoryArcSourceKind + source_ordinal: int + status: ImportedStoryArcStatus + selected_for_import: bool + proposed_story_arc_id: int | None + proposed_story_arc_name: str | None + merge_candidates: tuple[StoryArcMergeCandidate, ...] + entries_total: int + entries_resolved: int + entries_missing: int + entries_ambiguous: int + entries_conflict: int + entries_pending: int + entries_skipped: int + selection_blocked: bool + selection_block_reason: str | None + policy_review: ImportStoryArcPolicyReview + + +@dataclass(frozen=True, slots=True) +class ImportedStoryArcReviewPage: + """Paginated staged-arc review results.""" + + items: tuple[ImportedStoryArcReviewRow, ...] + total: int + page: int + page_size: int + + +StoryArcReviewAction = Literal["select", "skip"] +StoryArcDecisionTuple = tuple[int, StoryArcReviewAction, int | None] + + +async def auto_confirm_trusted_logical_story_arcs( + session: AsyncSession, + job_id: int, + *, + batch_size: int = 100, +) -> int: + """Confirm exact local arc evidence for logical post-import creation only.""" + if isinstance(batch_size, bool) or batch_size <= 0: + raise ValidationError("Story arc auto-confirmation batch size must be positive") + + confirmed = 0 + last_id = 0 + while True: + arcs = list( + ( + await session.scalars( + select(ImportedStoryArc) + .where( + ImportedStoryArc.import_job_id == job_id, + ImportedStoryArc.id > last_id, + ImportedStoryArc.status.in_( + ( + ImportedStoryArcStatus.DETECTED, + ImportedStoryArcStatus.NEEDS_REVIEW, + ) + ), + ImportedStoryArc.selected_for_import.is_(False), + ) + .options(selectinload(ImportedStoryArc.entries)) + .order_by(ImportedStoryArc.id) + .limit(batch_size) + ) + ).all() + ) + if not arcs: + break + + blocked_arc_ids = await _load_arc_ids_with_current_safety( + session, + [int(arc.id) for arc in arcs], + ) + for arc in arcs: + if int(arc.id) in blocked_arc_ids or not _has_trusted_complete_arc_evidence(arc): + continue + arc.proposed_policy_snapshot = _logical_auto_policy(arc.source_kind) + diagnostics = dict(arc.diagnostics or {}) + diagnostics["auto_confirmation"] = { + "schema_version": 1, + "scope": "logical_membership_only", + "evidence": "trusted_local_exact_identity", + } + arc.diagnostics = diagnostics + arc.status = ImportedStoryArcStatus.CONFIRMED + arc.selected_for_import = True + for entry in arc.entries: + entry.selected_for_import = ( + entry.resolution_state != StoryArcResolutionState.SKIPPED + ) + confirmed += 1 + await session.flush() + last_id = int(arcs[-1].id) + return confirmed + + +async def load_import_story_arc_review_page( + session: AsyncSession, + job_id: int, + *, + page: int = 1, + page_size: int = 25, +) -> ImportedStoryArcReviewPage: + """Return one deterministic, bounded page of staged story arcs.""" + if page < 1: + raise ValidationError("Story arc review page must be at least 1") + if page_size < 1 or page_size > 100: + raise ValidationError("Story arc review page_size must be between 1 and 100") + + await _require_review_job(session, job_id) + total = int( + await session.scalar( + select(func.count(ImportedStoryArc.id)).where(ImportedStoryArc.import_job_id == job_id) + ) + or 0 + ) + arcs = list( + ( + await session.execute( + select(ImportedStoryArc) + .where(ImportedStoryArc.import_job_id == job_id) + .order_by(ImportedStoryArc.source_ordinal.asc(), ImportedStoryArc.id.asc()) + .offset((page - 1) * page_size) + .limit(page_size) + ) + ) + .scalars() + .all() + ) + arc_ids = [int(arc.id) for arc in arcs] + counts_by_arc_id = await _load_entry_counts(session, arc_ids) + safety_blocked_arc_ids = await _load_arc_ids_with_current_safety(session, arc_ids) + candidates_by_arc_id = await _load_merge_candidates(session, arcs) + + rows: list[ImportedStoryArcReviewRow] = [] + for arc in arcs: + counts = counts_by_arc_id.get(int(arc.id), {}) + conflict_count = counts.get(StoryArcResolutionState.CONFLICT, 0) + safety_blocked = int(arc.id) in safety_blocked_arc_ids + block_reason: str | None = None + if safety_blocked: + block_reason = "Resolve safety findings before selecting this story arc." + elif conflict_count: + block_reason = "Resolve or skip conflict entries before selecting this story arc." + + candidates = candidates_by_arc_id.get(int(arc.id), ()) + proposed_name = next( + ( + candidate.name + for candidate in candidates + if candidate.id == arc.proposed_story_arc_id + ), + None, + ) + rows.append( + ImportedStoryArcReviewRow( + id=int(arc.id), + name=arc.name or "Unnamed story arc", + source_kind=arc.source_kind, + source_ordinal=int(arc.source_ordinal), + status=arc.status, + selected_for_import=bool(arc.selected_for_import), + proposed_story_arc_id=arc.proposed_story_arc_id, + proposed_story_arc_name=proposed_name, + merge_candidates=candidates, + entries_total=sum(counts.values()), + entries_resolved=counts.get(StoryArcResolutionState.RESOLVED, 0), + entries_missing=counts.get(StoryArcResolutionState.MISSING, 0), + entries_ambiguous=counts.get(StoryArcResolutionState.AMBIGUOUS, 0), + entries_conflict=conflict_count, + entries_pending=counts.get(StoryArcResolutionState.PENDING, 0), + entries_skipped=counts.get(StoryArcResolutionState.SKIPPED, 0), + selection_blocked=bool(block_reason), + selection_block_reason=block_reason, + policy_review=build_import_story_arc_policy_review( + arc.proposed_policy_snapshot or {}, + arc.source_settings_snapshot or {}, + arc.diagnostics or {}, + ), + ) + ) + + return ImportedStoryArcReviewPage( + items=tuple(rows), + total=total, + page=page, + page_size=page_size, + ) + + +async def update_import_story_arc_decision( + session: AsyncSession, + job_id: int, + imported_story_arc_id: int, + *, + action: StoryArcReviewAction, + proposed_story_arc_id: int | None, +) -> ImportedStoryArc: + """Persist one explicit select/skip decision without creating canonical rows.""" + await _require_review_job(session, job_id) + staged_arc = await session.get(ImportedStoryArc, imported_story_arc_id) + if staged_arc is None or staged_arc.import_job_id != job_id: + raise NotFoundError("ImportedStoryArc", imported_story_arc_id) + if action not in {"select", "skip"}: + raise ValidationError("Story arc review action must be select or skip") + + entries = list( + ( + await session.execute( + select(ImportedStoryArcEntry) + .where(ImportedStoryArcEntry.imported_story_arc_id == staged_arc.id) + .order_by( + ImportedStoryArcEntry.source_ordinal.asc(), + ImportedStoryArcEntry.id.asc(), + ) + ) + ) + .scalars() + .all() + ) + + if action == "skip": + if proposed_story_arc_id is not None: + raise ValidationError("A skipped story arc cannot have a proposed merge target") + staged_arc.status = ImportedStoryArcStatus.SKIPPED + staged_arc.selected_for_import = False + staged_arc.proposed_story_arc_id = None + for entry in entries: + entry.selected_for_import = False + await session.flush() + return staged_arc + + await _validate_merge_target(session, proposed_story_arc_id) + safety_blocked_arc_ids = await _load_arc_ids_with_current_safety( + session, + [int(staged_arc.id)], + ) + _assert_story_arc_selectable( + entries, + has_current_safety_block=int(staged_arc.id) in safety_blocked_arc_ids, + ) + staged_arc.status = ImportedStoryArcStatus.READY + staged_arc.selected_for_import = True + staged_arc.proposed_story_arc_id = proposed_story_arc_id + for entry in entries: + entry.selected_for_import = entry.resolution_state != StoryArcResolutionState.SKIPPED + await session.flush() + return staged_arc + + +async def confirm_import_story_arcs( + session: AsyncSession, + job_id: int, + *, + story_arc_ids: Sequence[int], + decisions: Sequence[StoryArcDecisionTuple], + batch_size: int = 250, +) -> int: + """Apply decisions and confirm selected arcs through bounded keyset pages.""" + if isinstance(batch_size, bool) or batch_size <= 0: + raise ValidationError("Story arc confirmation batch size must be positive") + await _require_review_job(session, job_id) + normalized_decisions = [ + (int(arc_id), action, proposed_story_arc_id) + for arc_id, action, proposed_story_arc_id in decisions + ] + decision_ids = {arc_id for arc_id, _, _ in normalized_decisions} + compatibility_ids = list(dict.fromkeys(int(value) for value in story_arc_ids)) + requested_ids = decision_ids.union(compatibility_ids) + arcs_by_id: dict[int, ImportedStoryArc] = {} + requested_id_list = sorted(requested_ids) + for start in range(0, len(requested_id_list), batch_size): + page_ids = requested_id_list[start : start + batch_size] + requested_result = await session.execute( + select(ImportedStoryArc) + .where( + ImportedStoryArc.import_job_id == job_id, + ImportedStoryArc.id.in_(page_ids), + ) + .options( + selectinload(ImportedStoryArc.entries).joinedload(ImportedStoryArcEntry.import_file) + ) + .order_by(ImportedStoryArc.id) + ) + arcs_by_id.update( + {int(staged_arc.id): staged_arc for staged_arc in requested_result.scalars().all()} + ) + for requested_id in requested_ids: + if requested_id not in arcs_by_id: + raise NotFoundError("ImportedStoryArc", requested_id) + + target_ids = { + int(target_id) + for _, action, target_id in normalized_decisions + if action == "select" and target_id is not None + } + target_ids.update( + int(staged_arc.proposed_story_arc_id) + for staged_arc in arcs_by_id.values() + if staged_arc.proposed_story_arc_id is not None + ) + merge_targets: dict[int, StoryArc] = {} + await _load_merge_targets_by_id(session, target_ids, merge_targets) + safety_blocked_arc_ids = { + arc_id + for arc_id, staged_arc in arcs_by_id.items() + if _loaded_arc_has_current_safety(staged_arc) + } + + for arc_id, action, proposed_story_arc_id in normalized_decisions: + _apply_loaded_story_arc_decision( + arcs_by_id[arc_id], + action=action, + proposed_story_arc_id=proposed_story_arc_id, + merge_targets=merge_targets, + has_current_safety_block=arc_id in safety_blocked_arc_ids, + ) + + for arc_id in compatibility_ids: + if arc_id in decision_ids: + continue + staged_arc = arcs_by_id[arc_id] + _apply_loaded_story_arc_decision( + staged_arc, + action="select", + proposed_story_arc_id=staged_arc.proposed_story_arc_id, + merge_targets=merge_targets, + has_current_safety_block=arc_id in safety_blocked_arc_ids, + ) + + confirmed_count = 0 + last_id = 0 + while True: + selected_result = await session.execute( + select(ImportedStoryArc) + .where( + ImportedStoryArc.import_job_id == job_id, + ImportedStoryArc.selected_for_import.is_(True), + ImportedStoryArc.id > last_id, + ) + .options( + selectinload(ImportedStoryArc.entries).joinedload(ImportedStoryArcEntry.import_file) + ) + .order_by(ImportedStoryArc.id) + .limit(batch_size) + ) + selected_arcs = list(selected_result.scalars().all()) + if not selected_arcs: + break + page_target_ids = { + int(staged_arc.proposed_story_arc_id) + for staged_arc in selected_arcs + if staged_arc.proposed_story_arc_id is not None + } + await _load_merge_targets_by_id(session, page_target_ids, merge_targets) + for staged_arc in selected_arcs: + _validate_loaded_merge_target( + staged_arc.proposed_story_arc_id, + merge_targets, + ) + _assert_story_arc_selectable( + staged_arc.entries, + has_current_safety_block=_loaded_arc_has_current_safety(staged_arc), + ) + if staged_arc.status != ImportedStoryArcStatus.READY: + raise ValidationError( + "Select each story arc from Step 3 before confirming the import" + ) + staged_arc.status = ImportedStoryArcStatus.CONFIRMED + staged_arc.selected_for_import = True + confirmed_count += 1 + await session.flush() + last_id = int(selected_arcs[-1].id) + + return confirmed_count + + +async def _load_merge_targets_by_id( + session: AsyncSession, + target_ids: set[int], + loaded: dict[int, StoryArc], +) -> None: + missing_ids = target_ids - loaded.keys() + if not missing_ids: + return + loaded.update( + { + int(target.id): target + for target in ( + await session.scalars(select(StoryArc).where(StoryArc.id.in_(missing_ids))) + ).all() + } + ) + + +def _apply_loaded_story_arc_decision( + staged_arc: ImportedStoryArc, + *, + action: StoryArcReviewAction, + proposed_story_arc_id: int | None, + merge_targets: Mapping[int, StoryArc], + has_current_safety_block: bool, +) -> None: + if action not in {"select", "skip"}: + raise ValidationError("Story arc review action must be select or skip") + if action == "skip": + if proposed_story_arc_id is not None: + raise ValidationError("A skipped story arc cannot have a proposed merge target") + staged_arc.status = ImportedStoryArcStatus.SKIPPED + staged_arc.selected_for_import = False + staged_arc.proposed_story_arc_id = None + for entry in staged_arc.entries: + entry.selected_for_import = False + return + + _validate_loaded_merge_target(proposed_story_arc_id, merge_targets) + _assert_story_arc_selectable( + staged_arc.entries, + has_current_safety_block=has_current_safety_block, + ) + staged_arc.status = ImportedStoryArcStatus.READY + staged_arc.selected_for_import = True + staged_arc.proposed_story_arc_id = proposed_story_arc_id + for entry in staged_arc.entries: + entry.selected_for_import = entry.resolution_state != StoryArcResolutionState.SKIPPED + + +def _validate_loaded_merge_target( + story_arc_id: int | None, + merge_targets: Mapping[int, StoryArc], +) -> None: + if story_arc_id is None: + return + target = merge_targets.get(story_arc_id) + if target is None: + raise NotFoundError("StoryArc", story_arc_id) + if target.lifecycle == StoryArcLifecycle.ARCHIVED: + raise ValidationError("An archived story arc cannot be selected as a merge target") + + +def _has_trusted_complete_arc_evidence(staged_arc: ImportedStoryArc) -> bool: + if not staged_arc.name or not staged_arc.name.strip() or not staged_arc.entries: + return False + active_entries = [ + entry + for entry in staged_arc.entries + if entry.resolution_state != StoryArcResolutionState.SKIPPED + ] + if not active_entries or any( + entry.resolution_state + in { + StoryArcResolutionState.PENDING, + StoryArcResolutionState.AMBIGUOUS, + StoryArcResolutionState.CONFLICT, + } + for entry in active_entries + ): + return False + + diagnostics = staged_arc.diagnostics or {} + if staged_arc.source_kind == StoryArcSourceKind.MYLAR3: + return diagnostics.get("source_name_present") is True and all( + _is_provider_id(entry.source_issue_id) for entry in active_entries + ) + if staged_arc.source_kind == StoryArcSourceKind.COMICINFO: + return all( + entry.resolution_state == StoryArcResolutionState.RESOLVED for entry in active_entries + ) + if staged_arc.source_kind == StoryArcSourceKind.FOLDER: + return diagnostics.get("reason") == "consistent_exact_arc_name" and all( + entry.resolution_state == StoryArcResolutionState.RESOLVED + and (entry.evidence or {}).get("has_comicinfo") is True + for entry in active_entries + ) + return False + + +def _is_provider_id(value: str | None) -> bool: + return value is not None and value.strip().isdecimal() and int(value.strip()) > 0 + + +def _logical_auto_policy(source_kind: StoryArcSourceKind) -> dict[str, object]: + return { + "schema_version": 1, + "source": source_kind.value, + "activation": "confirmed", + "monitored": False, + "search_missing": False, + "include_upcoming": False, + "sync_enabled": False, + "placement_policy": { + "schema_version": 1, + "mode": "logical", + "target_library_root_id": None, + "destination_root": None, + "folder_template": DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + "file_template": DEFAULT_STORY_ARC_FILE_TEMPLATE, + "symlink_style": None, + "synchronize": False, + }, + } + + +async def _require_review_job(session: AsyncSession, job_id: int) -> ImportJob: + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if job.status != ImportJobStatus.REVIEW: + raise ValidationError("Job must be in REVIEW state to update story arc decisions") + return job + + +async def _validate_merge_target(session: AsyncSession, story_arc_id: int | None) -> None: + if story_arc_id is None: + return + target = await session.get(StoryArc, story_arc_id) + if target is None: + raise NotFoundError("StoryArc", story_arc_id) + if target.lifecycle == StoryArcLifecycle.ARCHIVED: + raise ValidationError("An archived story arc cannot be selected as a merge target") + + +def _assert_story_arc_selectable( + entries: Sequence[ImportedStoryArcEntry], + *, + has_current_safety_block: bool, +) -> None: + if has_current_safety_block: + raise ValidationError("Resolve story arc safety findings before confirming this arc") + if any(entry.resolution_state == StoryArcResolutionState.CONFLICT for entry in entries): + raise ValidationError( + "Resolve or skip story arc conflict entries before confirming this arc" + ) + + +def _loaded_arc_has_current_safety(staged_arc: ImportedStoryArc) -> bool: + """Return whether an eagerly loaded arc still links to a safety-blocked file.""" + return any( + entry.import_file is not None + and entry.import_file.status == ImportedFileStatus.SAFETY_BLOCKED + for entry in staged_arc.entries + ) + + +async def _load_arc_ids_with_current_safety( + session: AsyncSession, + arc_ids: Sequence[int], +) -> set[int]: + """Return arcs whose currently linked import file is still safety blocked.""" + if not arc_ids: + return set() + return { + int(arc_id) + for arc_id in ( + await session.scalars( + select(ImportedStoryArcEntry.imported_story_arc_id) + .join(ImportedFile, ImportedFile.id == ImportedStoryArcEntry.import_file_id) + .where( + ImportedStoryArcEntry.imported_story_arc_id.in_(arc_ids), + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + ) + .distinct() + ) + ).all() + } + + +async def _load_entry_counts( + session: AsyncSession, + arc_ids: Sequence[int], +) -> dict[int, dict[StoryArcResolutionState, int]]: + if not arc_ids: + return {} + result = await session.execute( + select( + ImportedStoryArcEntry.imported_story_arc_id, + ImportedStoryArcEntry.resolution_state, + func.count(ImportedStoryArcEntry.id), + ) + .where(ImportedStoryArcEntry.imported_story_arc_id.in_(arc_ids)) + .group_by( + ImportedStoryArcEntry.imported_story_arc_id, + ImportedStoryArcEntry.resolution_state, + ) + ) + counts: dict[int, dict[StoryArcResolutionState, int]] = {} + for arc_id, state, count in result.all(): + counts.setdefault(int(arc_id), {})[state] = int(count) + return counts + + +async def _load_merge_candidates( + session: AsyncSession, + arcs: Sequence[ImportedStoryArc], +) -> dict[int, tuple[StoryArcMergeCandidate, ...]]: + if not arcs: + return {} + normalized_names = {arc.normalized_name for arc in arcs if arc.normalized_name} + proposed_ids = { + int(arc.proposed_story_arc_id) for arc in arcs if arc.proposed_story_arc_id is not None + } + if not normalized_names and not proposed_ids: + return {} + + filters = [] + if normalized_names: + filters.append(StoryArc.normalized_name.in_(normalized_names)) + if proposed_ids: + filters.append(StoryArc.id.in_(proposed_ids)) + existing_arcs = list( + ( + await session.execute( + select(StoryArc) + .where( + StoryArc.lifecycle == StoryArcLifecycle.ACTIVE, + or_(*filters), + ) + .order_by(StoryArc.name.asc(), StoryArc.id.asc()) + .limit(100) + ) + ) + .scalars() + .all() + ) + + candidates: dict[int, tuple[StoryArcMergeCandidate, ...]] = {} + for staged_arc in arcs: + matches = [ + StoryArcMergeCandidate(id=int(existing.id), name=existing.name) + for existing in existing_arcs + if existing.normalized_name == staged_arc.normalized_name + or existing.id == staged_arc.proposed_story_arc_id + ] + candidates[int(staged_arc.id)] = tuple(matches) + return candidates + + +def _mapping(value: object) -> Mapping[str, object]: + return value if isinstance(value, Mapping) else {} diff --git a/src/pullbox/services/import_story_arc_staging.py b/src/pullbox/services/import_story_arc_staging.py new file mode 100644 index 00000000..4fffa8d7 --- /dev/null +++ b/src/pullbox/services/import_story_arc_staging.py @@ -0,0 +1,929 @@ +"""Persist review-only story-arc evidence without providers or source I/O.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from typing import TYPE_CHECKING, cast + +from sqlalchemy import select + +from pullbox.core.mylar_story_arc_policy import build_mylar_story_arc_policy_draft +from pullbox.models.import_job import ImportedFile, ImportedFileStatus +from pullbox.models.story_arc import ( + ImportedStoryArcStatus, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.services.import_folder_story_arc_evidence import ( + _evidence_from_imported_file, + detect_imported_folder_story_arc, +) +from pullbox.services.import_story_arc_detection import FolderArcClassification + +if TYPE_CHECKING: + from collections.abc import Collection, Mapping, Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.core.mylar3_reader import ( + Mylar3ArcSettingsSnapshot, + Mylar3CollectionSnapshot, + Mylar3StoryArcEntrySnapshot, + Mylar3StoryArcSnapshot, + ) + from pullbox.services.import_story_arc_detection import FolderArcFileEvidence + + +_SAFE_CODE = re.compile(r"[^A-Za-z0-9_.:-]+") +_MISSING_MYLAR_STATUSES = frozenset({"missing", "wanted"}) +_SKIPPED_MYLAR_STATUSES = frozenset({"skipped", "skip"}) +_REVIEW_LOCKED_ARC_STATUSES = frozenset( + { + ImportedStoryArcStatus.CONFIRMED, + ImportedStoryArcStatus.SKIPPED, + ImportedStoryArcStatus.IMPORTED, + } +) + + +@dataclass(frozen=True, slots=True) +class StoryArcStagingResult: + """Bounded staging summary suitable for progress/review integration.""" + + arcs_staged: int = 0 + entries_staged: int = 0 + needs_review: int = 0 + cohorts_examined: int = 0 + cohorts_skipped: int = 0 + readlist_present: bool = False + readlist_count: int = 0 + + +CancellationCheck = Callable[[], Awaitable[None]] + + +async def stage_mylar_story_arcs( + session: AsyncSession, + *, + import_job_id: int, + snapshot: Mylar3CollectionSnapshot, + batch_size: int = 100, + source_ordinal_offset: int = 0, + cancellation_check: CancellationCheck | None = None, +) -> StoryArcStagingResult: + """Stage a pre-read Mylar collection snapshot without source or provider I/O. + + The caller owns the transaction. This function flushes at bounded checkpoints + but never commits, so a cancellation can still roll back the complete stage. + """ + _require_positive_batch_size(batch_size) + if source_ordinal_offset < 0: + raise ValueError("Mylar story-arc source ordinal offset cannot be negative.") + await _checkpoint(cancellation_check) + + settings_snapshot = _mylar_settings_snapshot(snapshot) + arcs_staged = 0 + entries_staged = 0 + needs_review = 0 + + arcs = tuple(snapshot.story_arcs) + seen_source_keys: set[str] = set() + for start in range(0, len(arcs), batch_size): + await _checkpoint(cancellation_check) + source_arcs = tuple( + ( + source_ordinal, + source_arc, + _mylar_source_key(source_arc), + ) + for source_ordinal, source_arc in enumerate( + arcs[start : start + batch_size], + start=source_ordinal_offset + start + 1, + ) + ) + page_source_keys = [source_key for _, _, source_key in source_arcs] + duplicate_source_keys = seen_source_keys.intersection(page_source_keys) + if len(set(page_source_keys)) != len(page_source_keys) or duplicate_source_keys: + msg = "Mylar story-arc staging requires unique source identities." + raise ValueError(msg) + seen_source_keys.update(page_source_keys) + + existing_arcs = list( + ( + await session.execute( + select(ImportedStoryArc).where( + ImportedStoryArc.import_job_id == import_job_id, + ImportedStoryArc.source_key.in_(page_source_keys), + ) + ) + ) + .scalars() + .all() + ) + existing_arc_by_source_key = {arc.source_key: arc for arc in existing_arcs} + existing_entries_by_arc_id: dict[int, dict[int, ImportedStoryArcEntry]] = {} + existing_arc_ids = [arc.id for arc in existing_arcs] + if existing_arc_ids: + existing_entries = list( + ( + await session.execute( + select(ImportedStoryArcEntry).where( + ImportedStoryArcEntry.imported_story_arc_id.in_(existing_arc_ids) + ) + ) + ) + .scalars() + .all() + ) + for entry in existing_entries: + existing_entries_by_arc_id.setdefault(entry.imported_story_arc_id, {})[ + entry.source_ordinal + ] = entry + + for source_ordinal, source_arc, source_key in source_arcs: + entry_payloads = tuple(_mylar_entry_payload(entry) for entry in source_arc.entries) + status = _mylar_arc_status(source_arc, entry_payloads, snapshot.arc_settings) + existing_arc = existing_arc_by_source_key.get(source_key) + staged_entry_count = await _upsert_staged_arc( + session, + import_job_id=import_job_id, + source_kind=StoryArcSourceKind.MYLAR3, + source_key=source_key, + source_arc_id=_bounded_identifier(source_arc.story_arc_id, 255), + source_ordinal=source_ordinal, + name=_bounded_text(source_arc.name, 500), + description=None, + status=status, + proposed_policy_snapshot=build_mylar_story_arc_policy_draft(snapshot.arc_settings), + source_settings_snapshot=settings_snapshot, + diagnostics=_mylar_arc_diagnostics( + snapshot=snapshot, + source_arc=source_arc, + entries=entry_payloads, + ), + entry_payloads=entry_payloads, + existing_staged_arc=existing_arc, + lookup_existing=False, + existing_entries=( + existing_entries_by_arc_id.get(existing_arc.id, {}) + if existing_arc is not None + else {} + ), + ) + arcs_staged += 1 + entries_staged += staged_entry_count + needs_review += status == ImportedStoryArcStatus.NEEDS_REVIEW + await session.flush() + + return StoryArcStagingResult( + arcs_staged=arcs_staged, + entries_staged=entries_staged, + needs_review=needs_review, + readlist_present=bool(snapshot.readlist_present), + readlist_count=max(int(snapshot.readlist_count), 0), + ) + + +async def stage_folder_story_arcs( + session: AsyncSession, + *, + import_job_id: int, + cohort_batch_size: int = 100, + confirmed_cohort_keys: Collection[str] = (), + cancellation_check: CancellationCheck | None = None, +) -> StoryArcStagingResult: + """Stage complete folder cohorts from cached ImportedFile evidence only. + + Cohort keys are keyset-paginated. Every selected key is then loaded as one + complete cohort, including rows split across several ImportedSeries records. + """ + _require_positive_batch_size(cohort_batch_size) + await _checkpoint(cancellation_check) + + confirmed = frozenset(confirmed_cohort_keys) + last_key: str | None = None + arcs_staged = 0 + entries_staged = 0 + needs_review = 0 + cohorts_examined = 0 + cohorts_skipped = 0 + + while True: + key_query = ( + select(ImportedFile.source_folder_cohort_key) + .where( + ImportedFile.import_job_id == import_job_id, + ImportedFile.source_folder_cohort_key.is_not(None), + ) + .distinct() + .order_by(ImportedFile.source_folder_cohort_key) + .limit(cohort_batch_size) + ) + if last_key is not None: + key_query = key_query.where(ImportedFile.source_folder_cohort_key > last_key) + keys = [ + key + for key in (await session.execute(key_query)).scalars().all() + if isinstance(key, str) and key + ] + if not keys: + break + + await _checkpoint(cancellation_check) + files_by_cohort: dict[str, list[ImportedFile]] = {key: [] for key in keys} + cohort_files = list( + ( + await session.execute( + select(ImportedFile) + .where( + ImportedFile.import_job_id == import_job_id, + ImportedFile.source_folder_cohort_key.in_(keys), + ) + .order_by( + ImportedFile.source_folder_cohort_key, + ImportedFile.source_ordinal.is_(None), + ImportedFile.source_ordinal, + ImportedFile.id, + ) + ) + ) + .scalars() + .all() + ) + for item in cohort_files: + cohort_key = item.source_folder_cohort_key + if cohort_key is not None: + files_by_cohort.setdefault(cohort_key, []).append(item) + + source_key_by_cohort = {key: _folder_source_key(key) for key in keys} + existing_arcs = list( + ( + await session.execute( + select(ImportedStoryArc).where( + ImportedStoryArc.import_job_id == import_job_id, + ImportedStoryArc.source_key.in_(source_key_by_cohort.values()), + ) + ) + ) + .scalars() + .all() + ) + existing_arc_by_source_key = {arc.source_key: arc for arc in existing_arcs} + existing_entries_by_arc_id: dict[int, dict[int, ImportedStoryArcEntry]] = {} + existing_arc_ids = [arc.id for arc in existing_arcs] + if existing_arc_ids: + existing_entries = list( + ( + await session.execute( + select(ImportedStoryArcEntry).where( + ImportedStoryArcEntry.imported_story_arc_id.in_(existing_arc_ids) + ) + ) + ) + .scalars() + .all() + ) + for entry in existing_entries: + existing_entries_by_arc_id.setdefault(entry.imported_story_arc_id, {})[ + entry.source_ordinal + ] = entry + + for cohort_key in keys: + files = files_by_cohort.get(cohort_key, []) + cohorts_examined += 1 + if not files: + cohorts_skipped += 1 + continue + + detection = detect_imported_folder_story_arc( + folder_label=_folder_label(cohort_key), + files=files, + confirmed_order_pattern=cohort_key in confirmed, + ) + source_key = source_key_by_cohort[cohort_key] + existing_arc = existing_arc_by_source_key.get(source_key) + if detection.classification not in { + FolderArcClassification.STORY_ARC, + FolderArcClassification.NEEDS_REVIEW, + }: + await _remove_obsolete_unconfirmed_arc( + session, + import_job_id=import_job_id, + source_key=source_key, + existing_staged_arc=existing_arc, + lookup_existing=False, + ) + cohorts_skipped += 1 + continue + + evidence = tuple(_evidence_from_imported_file(item) for item in files) + entry_payloads = tuple( + _folder_entry_payload(item, item_evidence, source_ordinal=source_ordinal) + for source_ordinal, (item, item_evidence) in enumerate( + zip(files, evidence, strict=True), + start=1, + ) + ) + status = ( + ImportedStoryArcStatus.NEEDS_REVIEW + if detection.classification == FolderArcClassification.NEEDS_REVIEW + else ImportedStoryArcStatus.DETECTED + ) + staged_entry_count = await _upsert_staged_arc( + session, + import_job_id=import_job_id, + source_kind=StoryArcSourceKind.FOLDER, + source_key=source_key, + source_arc_id=None, + source_ordinal=cohorts_examined, + name=_bounded_text(detection.proposed_name, 500), + description=None, + status=status, + proposed_policy_snapshot={ + "schema_version": 1, + "source": "folder", + "activation": "requires_confirmation", + }, + source_settings_snapshot={}, + diagnostics={ + "schema_version": 1, + "classification": detection.classification.value, + "reason": _safe_code(detection.reason), + "file_count": len(files), + "series_count": int(detection.series_count), + "ordered_file_count": int(detection.ordered_file_count), + "cohort_key_digest": _digest_text(cohort_key), + "safety_incomplete": any(not item.evidence_complete for item in evidence), + }, + entry_payloads=entry_payloads, + existing_staged_arc=existing_arc, + lookup_existing=False, + existing_entries=( + existing_entries_by_arc_id.get(existing_arc.id, {}) + if existing_arc is not None + else {} + ), + ) + arcs_staged += 1 + entries_staged += staged_entry_count + needs_review += status == ImportedStoryArcStatus.NEEDS_REVIEW + + await session.flush() + last_key = keys[-1] + + return StoryArcStagingResult( + arcs_staged=arcs_staged, + entries_staged=entries_staged, + needs_review=needs_review, + cohorts_examined=cohorts_examined, + cohorts_skipped=cohorts_skipped, + ) + + +async def _upsert_staged_arc( + session: AsyncSession, + *, + import_job_id: int, + source_kind: StoryArcSourceKind, + source_key: str, + source_arc_id: str | None, + source_ordinal: int, + name: str | None, + description: str | None, + status: ImportedStoryArcStatus, + proposed_policy_snapshot: dict[str, object], + source_settings_snapshot: dict[str, object], + diagnostics: dict[str, object], + entry_payloads: Sequence[dict[str, object]], + existing_staged_arc: ImportedStoryArc | None = None, + lookup_existing: bool = True, + existing_entries: Mapping[int, ImportedStoryArcEntry] | None = None, +) -> int: + staged_arc = existing_staged_arc + if lookup_existing: + staged_arc = ( + await session.execute( + select(ImportedStoryArc).where( + ImportedStoryArc.import_job_id == import_job_id, + ImportedStoryArc.source_key == source_key, + ) + ) + ).scalar_one_or_none() + if staged_arc is None: + review_locked = False + staged_arc = ImportedStoryArc( + import_job_id=import_job_id, + source_kind=source_kind, + source_key=source_key, + source_arc_id=source_arc_id, + source_ordinal=source_ordinal, + name=name, + description=description, + status=status, + selected_for_import=False, + proposed_policy_snapshot=proposed_policy_snapshot, + source_settings_snapshot=source_settings_snapshot, + diagnostics=diagnostics, + ) + session.add(staged_arc) + else: + review_locked = staged_arc.status in _REVIEW_LOCKED_ARC_STATUSES + staged_arc.source_kind = source_kind + staged_arc.source_arc_id = source_arc_id + staged_arc.source_ordinal = source_ordinal + staged_arc.name = name + staged_arc.description = description + if not review_locked: + staged_arc.status = status + staged_arc.proposed_policy_snapshot = proposed_policy_snapshot + staged_arc.source_settings_snapshot = source_settings_snapshot + staged_arc.diagnostics = diagnostics + + if existing_entries is None: + existing_entries = ( + { + entry.source_ordinal: entry + for entry in ( + ( + await session.execute( + select(ImportedStoryArcEntry).where( + ImportedStoryArcEntry.imported_story_arc_id == staged_arc.id + ) + ) + ) + .scalars() + .all() + ) + } + if staged_arc.id is not None + else {} + ) + seen_ordinals: set[int] = set() + for payload in entry_payloads: + source_entry_ordinal = cast("int", payload["source_ordinal"]) + if source_entry_ordinal in seen_ordinals: + msg = "Story-arc staging requires unique source ordinals." + raise ValueError(msg) + seen_ordinals.add(source_entry_ordinal) + entry = existing_entries.get(source_entry_ordinal) + if entry is None: + entry = ImportedStoryArcEntry( + imported_story_arc=staged_arc, + **payload, + ) + session.add(entry) + continue + for attribute, value in payload.items(): + if attribute in {"selected_for_import", "materialized_membership_id"}: + continue + if review_locked and attribute in { + "matched_issue_id", + "resolution_state", + "resolution_confidence", + "resolution_method", + }: + continue + setattr(entry, attribute, value) + + for source_entry_ordinal, entry in existing_entries.items(): + if source_entry_ordinal not in seen_ordinals and not review_locked: + await session.delete(entry) + + return len(entry_payloads) + + +async def _remove_obsolete_unconfirmed_arc( + session: AsyncSession, + *, + import_job_id: int, + source_key: str, + existing_staged_arc: ImportedStoryArc | None = None, + lookup_existing: bool = True, +) -> None: + staged_arc = existing_staged_arc + if lookup_existing: + staged_arc = ( + await session.execute( + select(ImportedStoryArc).where( + ImportedStoryArc.import_job_id == import_job_id, + ImportedStoryArc.source_key == source_key, + ) + ) + ).scalar_one_or_none() + if staged_arc is None: + return + if staged_arc.materialized_story_arc_id is not None: + return + if staged_arc.status in { + ImportedStoryArcStatus.CONFIRMED, + ImportedStoryArcStatus.SKIPPED, + ImportedStoryArcStatus.IMPORTED, + }: + return + await session.delete(staged_arc) + + +def _mylar_entry_payload(entry: Mylar3StoryArcEntrySnapshot) -> dict[str, object]: + exact_issue_number = _bounded_exact_text(entry.issue_number, 320) + exact_reading_order = _bounded_exact_text(entry.reading_order_raw, 50) + source_location = _bounded_source_location(entry.location) + resolution_state = _mylar_resolution_state(entry) + diagnostics: dict[str, object] = { + "schema_version": 1, + "review_reason": _mylar_review_reason(resolution_state), + "source_location_present": entry.location is not None, + "source_location_omitted": entry.location is not None and source_location is None, + "exact_issue_number_omitted": entry.issue_number is not None and exact_issue_number is None, + "exact_reading_order_omitted": entry.reading_order_raw is not None + and exact_reading_order is None, + } + source_publisher = entry.issue_publisher or entry.publisher + return { + "import_file_id": None, + "matched_issue_id": None, + "materialized_membership_id": None, + "source_ordinal": int(entry.ordinal), + "reading_order": entry.reading_order, + "reading_order_raw": exact_reading_order, + "resolution_state": resolution_state, + "source_kind": StoryArcSourceKind.MYLAR3, + "source_entry_id": _bounded_identifier(entry.issue_arc_id, 255), + "source_arc_id": _bounded_identifier(entry.story_arc_id, 255), + "source_issue_id": _bounded_identifier(entry.issue_id, 255), + "source_series_id": _bounded_identifier(entry.comic_id, 255), + "source_issue_number_text": exact_issue_number, + "source_series_name": _bounded_text(entry.comic_name, 500), + "source_issue_title": _bounded_text(entry.issue_name, 500), + "source_publisher": _bounded_text(source_publisher, 255), + "source_release_date_text": _bounded_text(entry.release_date, 50), + "source_issue_date_text": _bounded_text(entry.issue_date, 50), + "resolution_confidence": None, + "resolution_method": "trusted_mylar_snapshot", + "evidence": { + "schema_version": 1, + "series_year": _bounded_text(entry.series_year, 50), + "issue_year": _bounded_text(entry.issue_year, 50), + "status": _bounded_text(entry.status, 100), + "manual": _bounded_text(entry.manual, 100), + "date_added": _bounded_text(entry.date_added, 50), + "digital_date": _bounded_text(entry.digital_date, 50), + "issue_type": _bounded_text(entry.issue_type, 100), + "aliases": _bounded_text(entry.aliases, 1000), + "total_issues": _bounded_text(entry.total_issues, 50), + "in_cache_dir": _bounded_text(entry.in_cache_dir, 50), + "int_issue_number": _bounded_text(entry.int_issue_number, 320), + "dynamic_comic_name": _bounded_text(entry.dynamic_comic_name, 500), + "volume": _bounded_text(entry.volume, 100), + "cv_arc_id": _bounded_identifier(entry.cv_arc_id, 255), + "has_arc_image": entry.arc_image is not None, + "has_source_location": entry.location is not None, + }, + "source_location": source_location, + "selected_for_import": False, + "diagnostics": diagnostics, + } + + +def _folder_entry_payload( + item: ImportedFile, + evidence: FolderArcFileEvidence, + *, + source_ordinal: int, +) -> dict[str, object]: + comicinfo = _cached_comicinfo(item) + safety_code = _folder_safety_code(item) + resolution_state = _folder_resolution_state(item, evidence) + issue_identity = item.comicvine_issue_id or item.matched_issue_cv_id + series_identity = _mapping(item.diagnostics).get("comicvine_series_id") + source_location = _bounded_source_location(item.file_path) + return { + "import_file_id": int(item.id), + "matched_issue_id": item.matched_issue_id, + "materialized_membership_id": None, + "source_ordinal": source_ordinal, + "reading_order": _parse_integral_order(evidence.story_arc_number), + "reading_order_raw": _bounded_exact_text(evidence.story_arc_number, 50), + "resolution_state": resolution_state, + "source_kind": StoryArcSourceKind.FOLDER, + "source_entry_id": f"import-file:{int(item.id)}", + "source_arc_id": None, + "source_issue_id": _bounded_identifier(issue_identity, 255), + "source_series_id": _bounded_identifier(series_identity, 255), + "source_issue_number_text": _bounded_exact_text(evidence.issue_number, 320), + "source_series_name": _bounded_text(evidence.series, 500), + "source_issue_title": _bounded_text(comicinfo.get("title"), 500), + "source_publisher": _bounded_text(comicinfo.get("publisher"), 255), + "source_release_date_text": None, + "source_issue_date_text": None, + "resolution_confidence": None, + "resolution_method": _bounded_text(item.match_method, 50), + "evidence": { + "schema_version": 1, + "story_arc_number_source": _safe_code(evidence.story_arc_number_source), + "original_source_ordinal": item.source_ordinal, + "has_comicinfo": bool(item.has_comicinfo), + "matched_issue_from_import_job": item.matched_issue_id is not None, + "has_source_location": bool(item.file_path), + }, + "source_location": source_location, + "selected_for_import": False, + "diagnostics": { + "schema_version": 1, + "review_reason": ( + "safety_incomplete" + if not evidence.evidence_complete + else _folder_review_reason(resolution_state) + ), + "safety_code": safety_code, + "source_location_omitted": bool(item.file_path) and source_location is None, + }, + } + + +def _mylar_resolution_state(entry: Mylar3StoryArcEntrySnapshot) -> StoryArcResolutionState: + status = (entry.status or "").strip().casefold() + if status in _SKIPPED_MYLAR_STATUSES: + return StoryArcResolutionState.SKIPPED + if status in _MISSING_MYLAR_STATUSES or entry.location is None: + return StoryArcResolutionState.MISSING + return StoryArcResolutionState.PENDING + + +def _folder_resolution_state( + item: ImportedFile, + evidence: FolderArcFileEvidence, +) -> StoryArcResolutionState: + if item.status == ImportedFileStatus.CONFLICT: + return StoryArcResolutionState.CONFLICT + if not evidence.evidence_complete: + return StoryArcResolutionState.AMBIGUOUS + if item.matched_issue_id is not None: + return StoryArcResolutionState.RESOLVED + if evidence.series is None or evidence.issue_number is None: + return StoryArcResolutionState.AMBIGUOUS + return StoryArcResolutionState.PENDING + + +def _mylar_arc_status( + source_arc: Mylar3StoryArcSnapshot, + entries: Sequence[dict[str, object]], + settings: Mylar3ArcSettingsSnapshot, +) -> ImportedStoryArcStatus: + review_states = { + StoryArcResolutionState.MISSING, + StoryArcResolutionState.AMBIGUOUS, + StoryArcResolutionState.CONFLICT, + } + if source_arc.name is None or settings.parse_warnings: + return ImportedStoryArcStatus.NEEDS_REVIEW + if any(payload["resolution_state"] in review_states for payload in entries): + return ImportedStoryArcStatus.NEEDS_REVIEW + return ImportedStoryArcStatus.DETECTED + + +def _mylar_settings_snapshot(snapshot: Mylar3CollectionSnapshot) -> dict[str, object]: + settings = snapshot.arc_settings + values: dict[str, object] = {} + for setting in settings.values: + value: bool | str | None = setting.value + if isinstance(value, str): + value = _bounded_text(value, 1000) + values[setting.key] = { + "section": setting.section, + "value": value, + "raw_value": _bounded_text(setting.raw_value, 1000), + "used_default": bool(setting.used_default), + } + return { + "schema_version": 1, + "present": bool(settings.present), + "parse_warnings": [_safe_code(warning) for warning in settings.parse_warnings], + "values": values, + "readlist": { + "present": bool(snapshot.readlist_present), + "count": max(int(snapshot.readlist_count), 0), + "import_state": "deferred_v1.5.0", + }, + } + + +def _mylar_arc_diagnostics( + *, + snapshot: Mylar3CollectionSnapshot, + source_arc: Mylar3StoryArcSnapshot, + entries: Sequence[dict[str, object]], +) -> dict[str, object]: + reading_orders = [ + ( + "numeric", + payload["reading_order"], + ) + if payload["reading_order"] is not None + else ( + "raw", + payload["reading_order_raw"], + ) + for payload in entries + if payload["reading_order"] is not None or payload["reading_order_raw"] is not None + ] + missing_count = sum( + payload["resolution_state"] == StoryArcResolutionState.MISSING for payload in entries + ) + return { + "schema_version": 1, + "storyarcs_present": bool(snapshot.storyarcs_present), + "entry_count": len(entries), + "missing_entry_count": missing_count, + "duplicate_reading_order": len(reading_orders) != len(set(reading_orders)), + "settings_warning_codes": [ + _safe_code(warning) for warning in snapshot.arc_settings.parse_warnings + ], + "external_identities": _mylar_external_identities(source_arc), + "source_name_present": source_arc.name is not None, + "readlist_present": bool(snapshot.readlist_present), + "readlist_count": max(int(snapshot.readlist_count), 0), + "readlist_import_state": "deferred_v1.5.0", + } + + +def _mylar_source_key(source_arc: Mylar3StoryArcSnapshot) -> str: + if source_arc.story_arc_id is not None: + identity: object = { + "kind": "story_arc_id", + "value": source_arc.story_arc_id, + } + elif source_arc.cv_arc_id is not None: + identity = { + "kind": "cv_arc_id", + "value": source_arc.cv_arc_id, + } + elif source_arc.name is not None: + identity = { + "kind": "story_arc_name", + "value": source_arc.name, + } + else: + identity = { + "kind": "anonymous_entries", + "entries": [ + { + "ordinal": entry.ordinal, + "issue_arc_id": entry.issue_arc_id, + "issue_id": entry.issue_id, + "comic_id": entry.comic_id, + "issue_number": entry.issue_number, + } + for entry in source_arc.entries + ], + } + return f"mylar3:{_digest_json(identity)}" + + +def _mylar_external_identities( + source_arc: Mylar3StoryArcSnapshot, +) -> list[dict[str, str]]: + external_id = _bounded_identifier(source_arc.cv_arc_id, 255) + if external_id is None: + return [] + return [ + { + "source": "comicvine", + "namespace": "story_arc", + "external_id": external_id, + } + ] + + +def _folder_source_key(cohort_key: str) -> str: + return f"folder:{_digest_text(cohort_key)}" + + +def _folder_label(cohort_key: str) -> str: + normalized = cohort_key.replace("\\", "/").rstrip("/") + label = normalized.rsplit("/", 1)[-1] + return label or "Story Arc" + + +def _cached_comicinfo(item: ImportedFile) -> Mapping[str, object]: + diagnostics = _mapping(item.diagnostics) + source_metadata = _mapping(diagnostics.get("source_metadata")) or diagnostics + archive_evidence = _mapping(source_metadata.get("archive_member_evidence")) or _mapping( + diagnostics.get("archive_member_evidence") + ) + return _mapping(archive_evidence.get("comicinfo")) or _mapping(source_metadata.get("comicinfo")) + + +def _folder_safety_code(item: ImportedFile) -> str | None: + safety_block = _mapping(_mapping(item.diagnostics).get("safety_block")) + return _safe_code(safety_block.get("code")) + + +def _mylar_review_reason(state: StoryArcResolutionState) -> str | None: + if state == StoryArcResolutionState.MISSING: + return "source_issue_missing" + if state == StoryArcResolutionState.SKIPPED: + return "source_issue_skipped" + return None + + +def _folder_review_reason(state: StoryArcResolutionState) -> str | None: + if state == StoryArcResolutionState.AMBIGUOUS: + return "incomplete_identity" + if state == StoryArcResolutionState.CONFLICT: + return "identity_conflict" + return None + + +def _parse_integral_order(value: str | None) -> int | None: + if value is None: + return None + try: + parsed = Decimal(value.strip()) + except InvalidOperation: + return None + if not parsed.is_finite() or parsed != parsed.to_integral_value(): + return None + result = int(parsed) + return result if -(2**31) <= result <= 2**31 - 1 else None + + +def _bounded_source_location(value: object) -> str | None: + text = _text(value) + if text is None or len(text) > 1000: + return None + return text + + +def _bounded_exact_text(value: object, limit: int) -> str | None: + text = _text(value) + if text is None or len(text) > limit: + return None + return text + + +def _bounded_identifier(value: object, limit: int) -> str | None: + text = _text(value) + if text is None: + return None + if len(text) <= limit: + return text + suffix = f":sha256:{_digest_text(text)}" + return f"{text[: limit - len(suffix)]}{suffix}" + + +def _bounded_text(value: object, limit: int) -> str | None: + text = _text(value) + if text is None: + return None + return text if len(text) <= limit else text[:limit] + + +def _text(value: object) -> str | None: + if value is None: + return None + text = str(value).strip() + return text or None + + +def _safe_code(value: object) -> str | None: + text = _text(value) + if text is None: + return None + sanitized = _SAFE_CODE.sub("_", text)[:100] + return sanitized or None + + +def _mapping(value: object) -> Mapping[str, object]: + return value if isinstance(value, dict) else {} + + +def _digest_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8", errors="surrogatepass")).hexdigest() + + +def _digest_json(value: object) -> str: + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + return _digest_text(payload) + + +def _require_positive_batch_size(value: int) -> None: + if value <= 0: + msg = "Story-arc staging batch size must be positive." + raise ValueError(msg) + + +async def _checkpoint(callback: CancellationCheck | None) -> None: + if callback is not None: + await callback() diff --git a/src/pullbox/services/import_terminal_recovery.py b/src/pullbox/services/import_terminal_recovery.py new file mode 100644 index 00000000..fe77456b --- /dev/null +++ b/src/pullbox/services/import_terminal_recovery.py @@ -0,0 +1,19 @@ +"""Shared eligibility rules for safe actions after a durable import completion.""" + +from __future__ import annotations + +from pullbox.models.import_job import ImportControlRequest, ImportJob, ImportJobStatus + + +def allows_terminal_import_recovery(job: ImportJob) -> bool: + """Return whether post-import actions may safely operate on this job.""" + if ( + job.archived_at is not None + or job.control_request is not ImportControlRequest.NONE + or job.story_arc_rollback_waiting_work_id is not None + or dict(job.progress_snapshot or {}).get("mode") == "rollback" + ): + return False + if job.status is ImportJobStatus.COMPLETED: + return True + return job.status is ImportJobStatus.FAILED and job.import_completed_at is not None diff --git a/src/pullbox/services/import_workflow_state.py b/src/pullbox/services/import_workflow_state.py index 279d1667..cada4113 100644 --- a/src/pullbox/services/import_workflow_state.py +++ b/src/pullbox/services/import_workflow_state.py @@ -34,7 +34,24 @@ SCAN_PROGRESS_FILE_MATCH_START = 80 SCAN_PROGRESS_FILE_MATCH_END = 99 WORKFLOW_SNAPSHOT_VERSION = 2 +_PERSISTENT_IMPORT_CONTEXT_KEYS = ( + "deferred_recovery", + "clean_library_adoption", + "clean_library_adoption_prepared", + "clean_library_source_snapshot", + "source_import_job_id", +) ImportProgressMode = Literal["scan", "import", "rollback"] + + +def deferred_recovery_scope(job: ImportJob) -> tuple[int, ...] | None: + """A prepared follow-up authorizes only its own series groups, not the old review.""" + state = dict(dict(job.progress_snapshot or {}).get("deferred_recovery") or {}) + if state.get("state") != "prepared": + return None + return tuple(int(value) for value in state.get("series_ids", [])) + + _INVENTORY_PROGRESS_THRESHOLDS: tuple[tuple[int, int], ...] = ( (1, 1), (10, 2), @@ -278,6 +295,22 @@ def import_control_state_for_job(job: ImportJob) -> dict[str, object]: """Return durable UI control affordances for the current job state.""" requested_action = snapshot_requested_action_for_job(job) action_pending = requested_action != ImportControlRequest.NONE + snapshot = dict(job.progress_snapshot or {}) + placement_wait = ( + job.status in {ImportJobStatus.IMPORTING, ImportJobStatus.STALLED} + and snapshot.get("phase") == "story_arc_placements" + ) + failed_placements = snapshot.get("story_arc_placements_failed") + cancelled_placements = snapshot.get("story_arc_placements_cancelled") + has_retryable_terminal_placements = ( + isinstance(failed_placements, int) + and not isinstance(failed_placements, bool) + and failed_placements >= 0 + and isinstance(cancelled_placements, int) + and not isinstance(cancelled_placements, bool) + and cancelled_placements >= 0 + and failed_placements + cancelled_placements > 0 + ) can_pause = ( job.status in { @@ -288,10 +321,11 @@ def import_control_state_for_job(job: ImportJob) -> dict[str, object]: ImportJobStatus.IMPORTING, } and not action_pending + and not placement_wait ) - can_resume = job.status in {ImportJobStatus.PAUSED, ImportJobStatus.STALLED} or ( - job.status == ImportJobStatus.REVIEW and job.import_started_at is None - ) + can_resume = ( + job.status in {ImportJobStatus.PAUSED, ImportJobStatus.STALLED} and not placement_wait + ) or (job.status == ImportJobStatus.REVIEW and job.import_started_at is None) can_cancel = ( job.status in ACTIVE_IMPORT_JOB_STATUSES @@ -327,6 +361,13 @@ def import_control_state_for_job(job: ImportJob) -> dict[str, object]: ImportJobStatus.CANCELLED, ImportJobStatus.ROLLED_BACK, } + can_retry_story_arc_placements = bool( + job.status is ImportJobStatus.STALLED + and job.import_started_at is not None + and placement_wait + and not action_pending + and has_retryable_terminal_placements + ) can_rollback = bool(job.import_started_at) and job.status in { ImportJobStatus.COMPLETED, ImportJobStatus.FAILED, @@ -339,6 +380,7 @@ def import_control_state_for_job(job: ImportJob) -> dict[str, object]: "can_delete": can_delete, "can_view_results": can_view_results, "can_retry": can_retry, + "can_retry_story_arc_placements": can_retry_story_arc_placements, "can_rollback": can_rollback, "transfer_method": job.transfer_method, "convert_to_preferred_format": job.convert_to_preferred_format, @@ -463,6 +505,13 @@ def runtime_snapshot_payload( "total_files_no_match": job.total_files_no_match, "total_files_imported": job.total_files_imported, "total_files_failed": job.total_files_failed, + "story_arc_placements_total": snapshot.get("story_arc_placements_total"), + "story_arc_placements_queued": snapshot.get("story_arc_placements_queued"), + "story_arc_placements_running": snapshot.get("story_arc_placements_running"), + "story_arc_placements_retry_wait": snapshot.get("story_arc_placements_retry_wait"), + "story_arc_placements_failed": snapshot.get("story_arc_placements_failed"), + "story_arc_placements_completed": snapshot.get("story_arc_placements_completed"), + "story_arc_placements_cancelled": snapshot.get("story_arc_placements_cancelled"), "review_summary": ( review_summary if review_summary is not None else snapshot.get("review_summary") ), @@ -473,9 +522,23 @@ def runtime_snapshot_payload( ), "control_state": import_control_state_for_job(job), } + for key in _PERSISTENT_IMPORT_CONTEXT_KEYS: + if key in snapshot: + runtime[key] = snapshot[key] return runtime +def _scale_clean_library_progress(job: ImportJob, event: ImportProgressEvent) -> None: + """Reserve the first five percent for clean-library plan preparation.""" + snapshot = dict(job.progress_snapshot or {}) + if ( + snapshot.get("clean_library_adoption") is not True + or event.phase == "clean_library_preparing" + ): + return + event.progress = min(100, 5 + round(event.progress * 0.95)) + + def initialize_progress_snapshot( job: ImportJob, *, @@ -596,7 +659,11 @@ async def persist_progress_snapshot( event: ImportProgressEvent, ) -> None: """Persist the latest progress payload on the job for recovery/UI hydration.""" + existing_snapshot = dict(job.progress_snapshot or {}) payload = event.model_dump(mode="json") + for key in _PERSISTENT_IMPORT_CONTEXT_KEYS: + if key in existing_snapshot: + payload[key] = existing_snapshot[key] if int(payload.get("progress_revision") or 0) <= 0: payload["progress_revision"] = next_progress_revision(job) else: @@ -641,7 +708,9 @@ def apply_progress_event_contract( if event.current_item_stage_label is None: event.current_item_stage_label = stage_label(event.current_item_stage) - if event.current_item_progress_pct is None: + if event.current_item_kind == "scan" and event.current_item_stage == "inventory": + event.current_item_progress_pct = None + elif event.current_item_progress_pct is None: if event.current_file_progress_pct is not None: event.current_item_progress_pct = event.current_file_progress_pct elif event.current_item_kind is not None: @@ -662,6 +731,7 @@ async def emit_live_progress( """Publish an explicit live-only event without writing a durable snapshot.""" if job.status in _PROTECTED_RUNTIME_STATUSES and event.status != job.status: return + _scale_clean_library_progress(job, event) highest_revision = max( int(revision_state.get("value") or 0), @@ -699,6 +769,7 @@ async def emit_progress( """ if job.status in _PROTECTED_RUNTIME_STATUSES and event.status != job.status: return + _scale_clean_library_progress(job, event) event.mode = cast("ImportProgressMode", snapshot_mode_for_job(job, default=event.mode)) if event.progress_revision <= 0: event.progress_revision = next_progress_revision(job) diff --git a/src/pullbox/services/intervention_service.py b/src/pullbox/services/intervention_service.py index b63df2ca..054a6ccd 100644 --- a/src/pullbox/services/intervention_service.py +++ b/src/pullbox/services/intervention_service.py @@ -42,6 +42,7 @@ from pullbox.models.download import DownloadHistory from pullbox.providers.artifact_hosts.contract import HostResolutionRequest from pullbox.providers.base import ReleaseResult + from pullbox.services.airdcpp_search_types import DcValidatedCandidate from pullbox.services.direct_search_coordinator import DirectValidatedCandidate from pullbox.services.download_service import DownloadService from pullbox.services.release_validator import ValidationResult @@ -186,6 +187,30 @@ async def create_pending_match( ) return pm + async def create_dc_pending_match( + self, + session: AsyncSession, + issue_id: int, + result: DcValidatedCandidate, + search_log_id: int, + ) -> PendingMatch | None: + """Use the shared review UI while preserving the exact client/file route.""" + from pullbox.services.airdcpp_search_acquisition import dc_review_snapshot + + pending = await self.create_pending_match( + session, issue_id, result.release, result.validation + ) + if pending is not None: + pending.match_details = { + **pending.match_details, + "source_kind": "dc", + "dc_route_snapshot": dc_review_snapshot( + result, issue_id=issue_id, search_log_id=search_log_id + ), + } + await session.flush() + return pending + async def create_direct_pending_match( self, session: AsyncSession, @@ -334,6 +359,26 @@ async def approve_match( if pm.status != PendingMatchStatus.PENDING: raise ValueError(f"Pending match {pending_id} is not pending (status={pm.status})") + if pm.match_details.get("source_kind") == "dc": + from pullbox.services.airdcpp_search_acquisition import ( + acquire_dc_candidate, + dc_review_candidate, + ) + + candidate, search_log_id = dc_review_candidate(pm) + download, _created = await acquire_dc_candidate( + session, + candidate=candidate, + issue_id=pm.issue_id, + search_log_id=search_log_id, + request_key=f"dc-review:{pm.id}", + automatic=False, + ) + pm.status = PendingMatchStatus.APPROVED + pm.resolved_at = datetime.now(UTC) + pm.resolved_by = "user" + return download + direct_attempt_id = _direct_attempt_id(pm) if direct_attempt_id is not None: return await self._approve_direct_match(session, pm, direct_attempt_id) diff --git a/src/pullbox/services/issue_file_service.py b/src/pullbox/services/issue_file_service.py index c509045e..e13448df 100644 --- a/src/pullbox/services/issue_file_service.py +++ b/src/pullbox/services/issue_file_service.py @@ -14,7 +14,7 @@ from pullbox.core.config_resolver import load_system_config_values from pullbox.core.exceptions import NotFoundError, ValidationError from pullbox.models.issue import Issue, IssueStatus -from pullbox.models.library import LibraryFile +from pullbox.models.library import LibraryFile, LibraryFileStorageMode from pullbox.services.series_delete_targets import trash_relative_path from pullbox.utilities.settings import move_file_to_utility_trash, resolve_trash_directory @@ -72,7 +72,10 @@ async def delete_issue_library_file( file_deleted = False trashed = False - if await asyncio.to_thread(file_path.exists): + if ( + library_file.storage_mode is not LibraryFileStorageMode.REFERENCED + and await asyncio.to_thread(file_path.exists) + ): if trash_dir is not None: trash_path = await asyncio.to_thread( move_file_to_utility_trash, diff --git a/src/pullbox/services/issue_import_service.py b/src/pullbox/services/issue_import_service.py index b4057fd5..fd811219 100644 --- a/src/pullbox/services/issue_import_service.py +++ b/src/pullbox/services/issue_import_service.py @@ -13,7 +13,12 @@ from pullbox.core.exceptions import NotFoundError from pullbox.core.file_ops import register_library_file from pullbox.core.file_safety import get_allowed_extensions -from pullbox.core.library_policy import LibraryIngestPolicy, load_library_ingest_policy +from pullbox.core.library_policy import ( + LibraryIngestPolicy, + load_effective_library_ingest_policy, + load_library_ingest_policy, +) +from pullbox.core.library_root_resolution import preferred_managed_root_id from pullbox.models.issue import Issue from pullbox.models.library import LibraryFile, MatchConfidence from pullbox.models.series import Series @@ -121,11 +126,17 @@ async def prepare_manual_issue_import( detail=f"Unsupported format '{ext}'. Supported: {supported}", ) + root_id = getattr(getattr(issue, "series", None), "library_root_id", None) + ingest_policy = ( + await load_effective_library_ingest_policy(session, root_id) + if root_id is not None + else await load_library_ingest_policy(session) + ) return PreparedManualIssueImport( issue=issue, issue_id=issue.id, source_path=source_path, - ingest_policy=await load_library_ingest_policy(session), + ingest_policy=ingest_policy, ) @@ -183,7 +194,7 @@ async def materialize_cbz_with_progress( issue=prepared.issue, confidence=MatchConfidence.MANUAL, move_to_library=True, - library_root_id=prepared.issue.series.library_root_id, + library_root_id=preferred_managed_root_id(prepared.issue.series), loaded_issue=prepared.issue, ingest_policy=prepared.ingest_policy, allow_resource_safety_exception=allow_resource_safety_exception, diff --git a/src/pullbox/services/issue_service.py b/src/pullbox/services/issue_service.py index c03c4d8f..97fbe1b2 100644 --- a/src/pullbox/services/issue_service.py +++ b/src/pullbox/services/issue_service.py @@ -45,7 +45,7 @@ async def get_for_series( select(Issue) .options(joinedload(Issue.library_file)) .where(Issue.series_id == series_id) - .order_by(Issue.issue_number) + .order_by(Issue.issue_number, Issue.issue_number_text, Issue.id) .limit(limit) .offset(offset) ) @@ -57,7 +57,12 @@ async def get_wanted(session: AsyncSession) -> list[Issue]: result = await session.execute( select(Issue) .where(Issue.status == IssueStatus.WANTED) - .order_by(Issue.series_id, Issue.issue_number) + .order_by( + Issue.series_id, + Issue.issue_number, + Issue.issue_number_text, + Issue.id, + ) ) return list(result.scalars().all()) diff --git a/src/pullbox/services/library_convert_service.py b/src/pullbox/services/library_convert_service.py index a75de58c..77c569fa 100644 --- a/src/pullbox/services/library_convert_service.py +++ b/src/pullbox/services/library_convert_service.py @@ -11,6 +11,7 @@ from sqlalchemy import select from pullbox.core.exceptions import ValidationError +from pullbox.core.library_file_ownership import require_mutable_library_target from pullbox.models.library import FileFormat, LibraryFile from pullbox.utilities.executors.file_converter import convert_file from pullbox.utilities.settings import move_file_to_utility_trash, restore_file_from_utility_trash @@ -77,6 +78,12 @@ async def convert_library_file( trash_path: Path | None = None try: + await require_mutable_library_target( + session, + source, + include_descendants=False, + operation="converted", + ) converted_path = await convert_file(source, "cbz") trash_path = move_file_to_utility_trash( source, diff --git a/src/pullbox/services/library_delete_service.py b/src/pullbox/services/library_delete_service.py index 87b783a7..fe27ab34 100644 --- a/src/pullbox/services/library_delete_service.py +++ b/src/pullbox/services/library_delete_service.py @@ -13,13 +13,14 @@ from pullbox.core.exceptions import ValidationError from pullbox.models.issue import Issue, IssueStatus -from pullbox.models.library import LibraryFile, LibraryRoot +from pullbox.models.library import LibraryFile, LibraryFileStorageMode, LibraryRoot from pullbox.models.series import Series from pullbox.services.series_service import SeriesService from pullbox.utilities.settings import move_path_to_utility_trash if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement def _is_relative_to(path: Path, other: Path) -> bool: @@ -118,6 +119,8 @@ class LibraryDeleteContext: linked_file_count: int = 0 tracked_file_count: int = 0 tracked_series_count: int = 0 + managed_file_count: int = 0 + referenced_file_count: int = 0 has_linked_issue: bool = False issue_status_after_delete: str | None = None issue_status_reason: str | None = None @@ -132,6 +135,26 @@ class LibraryDeleteOutcome: source_path: str deleted_via_trash: bool result_path: str | None = None + managed_files_deleted: int = 0 + referenced_files_detached: int = 0 + + +async def _storage_mode_counts( + session: AsyncSession, + file_clause: ColumnElement[bool], +) -> tuple[int, int]: + rows = ( + await session.execute( + select(LibraryFile.storage_mode, func.count(LibraryFile.id)) + .where(file_clause) + .group_by(LibraryFile.storage_mode) + ) + ).all() + counts = {storage_mode: int(count) for storage_mode, count in rows} + return ( + counts.get(LibraryFileStorageMode.MANAGED, 0), + counts.get(LibraryFileStorageMode.REFERENCED, 0), + ) async def build_delete_context( @@ -162,12 +185,19 @@ async def build_delete_context( ) ).scalar_one() ) + issue_ids = select(Issue.id).where(Issue.series_id == series_id) + managed_file_count, referenced_file_count = await _storage_mode_counts( + session, + LibraryFile.issue_id.in_(issue_ids), + ) return LibraryDeleteContext( mode="series", trash_enabled=trash_enabled, series_id=series_id, series_title=series_title, linked_file_count=linked_file_count, + managed_file_count=managed_file_count, + referenced_file_count=referenced_file_count, ) file_clause = ( @@ -178,6 +208,7 @@ async def build_delete_context( tracked_file_count = int( (await session.execute(select(func.count(LibraryFile.id)).where(file_clause))).scalar_one() ) + managed_file_count, referenced_file_count = await _storage_mode_counts(session, file_clause) tracked_series_count = 0 if kind == "folder": @@ -217,6 +248,8 @@ async def build_delete_context( trash_enabled=trash_enabled, tracked_file_count=tracked_file_count, tracked_series_count=tracked_series_count, + managed_file_count=managed_file_count, + referenced_file_count=referenced_file_count, has_linked_issue=has_linked_issue, issue_status_after_delete=issue_status_after_delete, issue_status_reason=issue_status_reason, @@ -292,8 +325,39 @@ async def delete_library_entry( .all() ) + managed_files = [ + library_file + for library_file in tracked_files + if library_file.storage_mode == LibraryFileStorageMode.MANAGED + ] + referenced_files = [ + library_file + for library_file in tracked_files + if library_file.storage_mode == LibraryFileStorageMode.REFERENCED + ] + result_path: str | None = None - if trash_dir is not None: + deleted_via_trash = False + if referenced_files: + # A whole-file or whole-folder operation could mutate a referenced artifact. + # Preserve the target and remove only Pullbox-owned files individually. + for library_file in managed_files: + managed_path = Path(library_file.file_path) + if not (managed_path.exists() or managed_path.is_symlink()): + continue + if trash_dir is not None: + try: + move_path_to_utility_trash( + managed_path, + trash_dir, + relative_path=_trash_relative_path(managed_path, root), + ) + except FileExistsError as exc: + raise ValidationError(str(exc)) from exc + deleted_via_trash = True + else: + _delete_path_permanently(managed_path) + elif trash_dir is not None: try: result_path = str( move_path_to_utility_trash( @@ -302,6 +366,7 @@ async def delete_library_entry( relative_path=_trash_relative_path(target, root), ) ) + deleted_via_trash = True except FileExistsError as exc: raise ValidationError(str(exc)) from exc else: @@ -312,13 +377,16 @@ async def delete_library_entry( library_file.issue.status = _status_after_library_file_removed(library_file.issue) await session.delete(library_file) - for series in tracked_series: - series.path = None + if not referenced_files: + for series in tracked_series: + series.path = None return LibraryDeleteOutcome( kind=kind, mode=delete_context.mode, source_path=str(target), - deleted_via_trash=trash_dir is not None, + deleted_via_trash=deleted_via_trash, result_path=result_path, + managed_files_deleted=len(managed_files), + referenced_files_detached=len(referenced_files), ) diff --git a/src/pullbox/services/library_rename_service.py b/src/pullbox/services/library_rename_service.py index 217e04e0..06e8c765 100644 --- a/src/pullbox/services/library_rename_service.py +++ b/src/pullbox/services/library_rename_service.py @@ -12,6 +12,7 @@ from sqlalchemy import or_, select from pullbox.core.exceptions import ValidationError +from pullbox.core.library_file_ownership import require_mutable_library_target from pullbox.models.library import LibraryFile from pullbox.models.series import Series @@ -101,6 +102,12 @@ async def rename_library_entry( renamed = False try: + await require_mutable_library_target( + session, + source, + include_descendants=kind == "folder", + operation="renamed", + ) _rename_path(source, target) renamed = True diff --git a/src/pullbox/services/library_root_management.py b/src/pullbox/services/library_root_management.py new file mode 100644 index 00000000..86e9705a --- /dev/null +++ b/src/pullbox/services/library_root_management.py @@ -0,0 +1,1188 @@ +"""Explicit multi-library root management and live capability validation.""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import tempfile +from contextlib import suppress +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +from sqlalchemy import func, select, update + +from pullbox.core.config_resolver import get_application_secret +from pullbox.core.exceptions import ValidationError +from pullbox.core.filesystem_policy import is_sensitive_path +from pullbox.models.config import SystemConfig +from pullbox.models.import_job import ImportJob +from pullbox.models.library import LibraryFile, LibraryRoot +from pullbox.models.series import Series +from pullbox.models.story_arc import StoryArcPlacement +from pullbox.services.import_workflow_state import ACTIVE_IMPORT_JOB_STATUSES +from pullbox.services.library_root_policy_service import LibraryRootNotFoundError +from pullbox.services.library_root_removal import ( + preview_library_root_removal as preview_library_root_removal, +) +from pullbox.services.library_root_removal import ( + remove_library_root as remove_library_root, +) + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + +_LOW_CAPACITY_BYTES = 1024**3 +_MUTATION_BLOCKED_MESSAGE = "Library roots cannot be changed while an import is active." +_REBIND_TOKEN_SALT = "library-root-rebind-preview-v1" +_REBIND_TOKEN_MAX_AGE_SECONDS = 15 * 60 +_REBIND_TOKEN_VERSION = 1 +_REBIND_ACTION = "rebind_library_root" +_REBIND_PATH_PAGE_SIZE = 2_000 +_MUTABLE_ROOT_FIELDS = frozenset( + { + "name", + "enabled", + "allow_referenced_registrations", + "allow_managed_writes", + "is_default_managed_destination", + } +) + + +@dataclass(frozen=True, slots=True) +class _RootProbe: + display_path: Path + resolved_path: Path | None + identity: tuple[int, int] | None + available: bool + readable: bool + writable: bool + free_bytes: int | None + status: str + warnings: tuple[str, ...] + blocking_reasons: tuple[str, ...] + + +async def list_library_roots(session: AsyncSession) -> list[dict[str, Any]]: + """Return configured roots with non-persisted live capability snapshots.""" + roots = list((await session.scalars(select(LibraryRoot).order_by(LibraryRoot.id.asc()))).all()) + probes = await asyncio.gather( + *( + asyncio.to_thread( + _probe_configured_root, + root.path, + require_write_probe=False, + ) + for root in roots + ) + ) + return [ + _serialize_root( + root, + probe, + extra_warnings=_root_conflicts( + roots, + name=root.name, + probe=probe, + exclude_root_id=root.id, + ), + ) + for root, probe in zip(roots, probes, strict=True) + ] + + +async def validate_managed_library_root(root: LibraryRoot) -> dict[str, Any]: + """Fail closed unless a configured root is a live managed destination. + + Callers use this immediately before freezing or executing managed placement + so an enabled database row cannot mask an offline or newly read-only mount. + Low capacity remains visible as a warning; workload-specific capacity checks + belong to the import preflight that knows the required byte count. + """ + if not root.enabled: + raise ValidationError("The selected managed library root is disabled.") + if not root.allow_managed_writes: + raise ValidationError("The selected library root does not allow managed writes.") + probe = await asyncio.to_thread( + _probe_configured_root, + root.path, + require_write_probe=True, + ) + blockers = list(probe.blocking_reasons) + blockers.extend( + _capability_blockers( + probe, + allow_referenced_registrations=False, + allow_managed_writes=True, + ) + ) + _raise_blockers(blockers) + return _capability_dict(probe) + + +async def validate_reference_library_root(root: LibraryRoot) -> dict[str, Any]: + """Fail closed unless a configured root can register referenced artifacts.""" + if not root.enabled: + raise ValidationError("The selected reference library root is disabled.") + if not root.allow_referenced_registrations: + raise ValidationError("The selected library root does not allow referenced registrations.") + probe = await asyncio.to_thread( + _probe_configured_root, + root.path, + require_write_probe=False, + ) + blockers = list(probe.blocking_reasons) + blockers.extend( + _capability_blockers( + probe, + allow_referenced_registrations=True, + allow_managed_writes=False, + ) + ) + _raise_blockers(blockers) + return _capability_dict(probe) + + +async def preview_library_root( + session: AsyncSession, + *, + name: str, + path: str, + allow_referenced_registrations: bool, + allow_managed_writes: bool, + is_default_managed_destination: bool, +) -> dict[str, Any]: + """Validate a proposed root without writing database or filesystem state.""" + normalized_name = _normalize_name(name) + _validate_roles( + allow_referenced_registrations=allow_referenced_registrations, + allow_managed_writes=allow_managed_writes, + is_default_managed_destination=is_default_managed_destination, + ) + probe = await asyncio.to_thread( + _probe_candidate_root, + path, + require_write_probe=allow_managed_writes, + ) + roots = list((await session.scalars(select(LibraryRoot).order_by(LibraryRoot.id))).all()) + conflicts = _root_conflicts( + roots, + name=normalized_name, + probe=probe, + exclude_root_id=None, + ) + blockers = list(probe.blocking_reasons) + blockers.extend(conflicts) + blockers.extend( + _capability_blockers( + probe, + allow_referenced_registrations=allow_referenced_registrations, + allow_managed_writes=allow_managed_writes, + ) + ) + blockers = _deduplicate(blockers) + becomes_default = is_default_managed_destination or ( + allow_managed_writes and not any(root.is_default_managed_destination for root in roots) + ) + capabilities = _capability_dict(probe) + warnings = list(capabilities["warnings"]) + if not blockers and becomes_default and not is_default_managed_destination: + warnings.append("The first managed root will become the default destination.") + return { + "name": normalized_name, + "path": str(probe.display_path), + "allow_referenced_registrations": allow_referenced_registrations, + "allow_managed_writes": allow_managed_writes, + "is_default_managed_destination": becomes_default, + **capabilities, + "warnings": _deduplicate(warnings), + "blocking_reasons": blockers, + "can_create": not blockers, + } + + +async def create_library_root( + session: AsyncSession, + *, + name: str, + path: str, + allow_referenced_registrations: bool, + allow_managed_writes: bool, + is_default_managed_destination: bool, +) -> dict[str, Any]: + """Create a validated root and atomically maintain the one-default invariant.""" + normalized_name = _normalize_name(name) + _validate_roles( + allow_referenced_registrations=allow_referenced_registrations, + allow_managed_writes=allow_managed_writes, + is_default_managed_destination=is_default_managed_destination, + ) + probe = await asyncio.to_thread( + _probe_candidate_root, + path, + require_write_probe=allow_managed_writes, + ) + await _assert_no_active_import(session) + roots = list( + ( + await session.scalars( + select(LibraryRoot).order_by(LibraryRoot.id.asc()).with_for_update() + ) + ).all() + ) + blockers = list(probe.blocking_reasons) + blockers.extend( + _root_conflicts( + roots, + name=normalized_name, + probe=probe, + exclude_root_id=None, + ) + ) + blockers.extend( + _capability_blockers( + probe, + allow_referenced_registrations=allow_referenced_registrations, + allow_managed_writes=allow_managed_writes, + ) + ) + _raise_blockers(blockers) + await _assert_no_active_import(session) + + becomes_default = is_default_managed_destination or ( + allow_managed_writes and not any(root.is_default_managed_destination for root in roots) + ) + if becomes_default: + await _clear_default(session) + root = LibraryRoot( + name=normalized_name, + path=str(probe.display_path), + enabled=True, + allow_referenced_registrations=allow_referenced_registrations, + allow_managed_writes=allow_managed_writes, + is_default_managed_destination=becomes_default, + ) + session.add(root) + await session.flush() + if becomes_default: + await _sync_legacy_default(session, root.path) + return _serialize_root(root, probe) + + +async def update_library_root( + session: AsyncSession, + library_root_id: int, + changes: Mapping[str, object], +) -> dict[str, Any]: + """Update mutable flags while keeping the root path immutable.""" + if not changes: + raise ValidationError("At least one library root field must be changed.") + if "path" in changes: + raise ValidationError("Library root paths cannot be changed by this endpoint.") + unknown_fields = set(changes) - _MUTABLE_ROOT_FIELDS + if unknown_fields: + raise ValidationError("Unknown library root update field.") + if any(value is None for value in changes.values()): + raise ValidationError("Library root update fields cannot be null.") + + await _assert_no_active_import(session) + current = await session.get(LibraryRoot, library_root_id) + if current is None: + raise LibraryRootNotFoundError() + + proposed_name = _normalize_name(str(changes.get("name", current.name))) + proposed_enabled = bool(changes.get("enabled", current.enabled)) + proposed_referenced = bool( + changes.get( + "allow_referenced_registrations", + current.allow_referenced_registrations, + ) + ) + proposed_managed = bool(changes.get("allow_managed_writes", current.allow_managed_writes)) + proposed_default = bool( + changes.get( + "is_default_managed_destination", + current.is_default_managed_destination, + ) + ) + if current.is_default_managed_destination and ( + not proposed_enabled or not proposed_managed or not proposed_default + ): + raise ValidationError( + "Select another root as the default managed destination before disabling " + "or demoting this root." + ) + _validate_roles( + allow_referenced_registrations=proposed_referenced, + allow_managed_writes=proposed_managed, + is_default_managed_destination=proposed_default, + ) + if proposed_default and not proposed_enabled: + raise ValidationError("The default managed destination must be enabled.") + + role_activation = proposed_enabled and ( + (proposed_referenced and not current.allow_referenced_registrations) + or (proposed_managed and not current.allow_managed_writes) + or (proposed_enabled and not current.enabled) + or (proposed_default and not current.is_default_managed_destination) + ) + probe = await asyncio.to_thread( + _probe_configured_root, + current.path, + require_write_probe=role_activation and proposed_managed, + ) + if role_activation: + blockers = list(probe.blocking_reasons) + blockers.extend( + _capability_blockers( + probe, + allow_referenced_registrations=proposed_referenced, + allow_managed_writes=proposed_managed, + ) + ) + _raise_blockers(blockers) + + roots = list( + ( + await session.scalars( + select(LibraryRoot).order_by(LibraryRoot.id.asc()).with_for_update() + ) + ).all() + ) + root = next((item for item in roots if item.id == library_root_id), None) + if root is None: + raise LibraryRootNotFoundError() + name_conflicts = [ + item + for item in roots + if item.id != root.id and item.name.strip().casefold() == proposed_name.casefold() + ] + if name_conflicts: + raise ValidationError("Library root names must be unique, ignoring case.") + await _assert_no_active_import(session) + + was_default = root.is_default_managed_destination + root.name = proposed_name + root.enabled = proposed_enabled + root.allow_referenced_registrations = proposed_referenced + root.allow_managed_writes = proposed_managed + + should_become_default = proposed_default and not was_default + if not any(item.is_default_managed_destination for item in roots) and ( + root.enabled and root.allow_managed_writes + ): + should_become_default = True + if should_become_default: + await _clear_default(session) + root.is_default_managed_destination = True + else: + root.is_default_managed_destination = proposed_default + await session.flush() + if root.is_default_managed_destination: + await _sync_legacy_default(session, root.path) + return _serialize_root(root, probe) + + +async def preview_library_root_rebind( + session: AsyncSession, + library_root_id: int, + *, + replacement_path: str, + actor_id: int, +) -> dict[str, Any]: + """Preview an explicit path-identity rebind without persisting changes.""" + roots = await _load_library_roots_for_rebind(session, lock=False) + preview, _snapshot = await _build_library_root_rebind_preview( + session, + roots, + library_root_id=library_root_id, + replacement_path=replacement_path, + actor_id=actor_id, + issue_token=True, + ) + return preview + + +async def rebind_library_root( + session: AsyncSession, + library_root_id: int, + *, + replacement_path: str, + preview_token: str, + actor_id: int, +) -> dict[str, Any]: + """Apply one signed, drift-checked root path rebind without rewriting file paths.""" + normalized_replacement = str(_normalize_candidate_path(replacement_path)) + token_payload = _load_rebind_preview_token(preview_token) + token_snapshot = _validate_rebind_token_scope( + token_payload, + library_root_id=library_root_id, + replacement_path=normalized_replacement, + actor_id=actor_id, + ) + + roots = await _load_library_roots_for_rebind(session, lock=True) + preview, current_snapshot = await _build_library_root_rebind_preview( + session, + roots, + library_root_id=library_root_id, + replacement_path=normalized_replacement, + actor_id=actor_id, + issue_token=False, + ) + if preview["blocking_reasons"]: + raise ValidationError( + "The library root rebind is no longer safe. Preview it again.", + details={"blocking_reasons": preview["blocking_reasons"]}, + ) + if token_snapshot != current_snapshot: + raise ValidationError("The library root changed after preview. Preview the rebind again.") + + root = next((item for item in roots if item.id == library_root_id), None) + if root is None: + raise LibraryRootNotFoundError() + await _assert_no_active_import(session) + + # Re-probe the exact confirmed replacement immediately before persistence. + post_probe = await asyncio.to_thread( + _probe_candidate_root, + normalized_replacement, + require_write_probe=root.allow_managed_writes, + ) + post_blockers = list(post_probe.blocking_reasons) + post_blockers.extend( + _capability_blockers( + post_probe, + allow_referenced_registrations=root.allow_referenced_registrations, + allow_managed_writes=root.allow_managed_writes, + ) + ) + _raise_blockers(post_blockers) + + root.path = str(post_probe.display_path) + await session.flush() + if root.is_default_managed_destination: + await _sync_legacy_default(session, root.path) + return _serialize_root(root, post_probe) + + +async def _load_library_roots_for_rebind( + session: AsyncSession, + *, + lock: bool, +) -> list[LibraryRoot]: + statement = select(LibraryRoot).order_by(LibraryRoot.id.asc()) + if lock: + statement = statement.with_for_update() + return list((await session.scalars(statement)).all()) + + +async def _build_library_root_rebind_preview( + session: AsyncSession, + roots: Sequence[LibraryRoot], + *, + library_root_id: int, + replacement_path: str, + actor_id: int, + issue_token: bool, +) -> tuple[dict[str, Any], dict[str, Any]]: + root = next((item for item in roots if item.id == library_root_id), None) + if root is None: + raise LibraryRootNotFoundError() + + replacement_probe = await asyncio.to_thread( + _probe_candidate_root, + replacement_path, + require_write_probe=root.allow_managed_writes, + ) + current_probe = await asyncio.to_thread( + _probe_configured_root, + root.path, + require_write_probe=False, + ) + current_display = Path(os.path.normpath(root.path)) + same_configured_path = current_display == replacement_probe.display_path + same_physical_directory = _probes_share_identity(current_probe, replacement_probe) + overlaps_current_path = not same_physical_directory and ( + _paths_overlap(current_display, replacement_probe.display_path) + or ( + current_probe.resolved_path is not None + and replacement_probe.resolved_path is not None + and _paths_overlap(current_probe.resolved_path, replacement_probe.resolved_path) + ) + ) + + blockers = list(replacement_probe.blocking_reasons) + blockers.extend( + _root_conflicts( + roots, + name=root.name, + probe=replacement_probe, + exclude_root_id=root.id, + ) + ) + blockers.extend( + _capability_blockers( + replacement_probe, + allow_referenced_registrations=root.allow_referenced_registrations, + allow_managed_writes=root.allow_managed_writes, + ) + ) + if same_configured_path: + blockers.append("Replacement path matches the current library root path.") + if await _has_active_import(session): + blockers.append(_MUTATION_BLOCKED_MESSAGE) + blockers = _deduplicate(blockers) + + warnings = list(replacement_probe.warnings) + if same_physical_directory and not same_configured_path: + warnings.append("Replacement path is an alias of the current root's physical directory.") + if overlaps_current_path: + warnings.append("Replacement path overlaps the current root path.") + warnings = _deduplicate(warnings) + + impact, association_digest, association_blockers = await _load_library_root_rebind_impact( + session, + root, + replacement_probe=replacement_probe, + ) + blockers.extend(association_blockers) + blockers = _deduplicate(blockers) + snapshot: dict[str, Any] = { + "root_id": root.id, + "root_updated_at": _timestamp_value(root.updated_at), + "root_name": root.name, + "current_path": root.path, + "replacement_path": str(replacement_probe.display_path), + "enabled": root.enabled, + "allow_referenced_registrations": root.allow_referenced_registrations, + "allow_managed_writes": root.allow_managed_writes, + "is_default_managed_destination": root.is_default_managed_destination, + "current_identity": _probe_identity_digest(current_probe), + "replacement_identity": _probe_identity_digest(replacement_probe), + "replacement_available": replacement_probe.available, + "replacement_readable": replacement_probe.readable, + "replacement_writable": replacement_probe.writable, + "replacement_status": replacement_probe.status, + "same_physical_directory": same_physical_directory, + "overlaps_current_path": overlaps_current_path, + "configured_roots_digest": _configured_roots_digest(roots), + "path_associations_digest": association_digest, + "impact": impact, + } + can_rebind = not blockers + preview_token = ( + _build_rebind_preview_token(snapshot=snapshot, actor_id=actor_id) + if can_rebind and issue_token + else None + ) + preview = { + "library_root_id": root.id, + "root_name": root.name, + "current_path": root.path, + "replacement_path": str(replacement_probe.display_path), + **_capability_dict(replacement_probe), + "warnings": warnings, + "blocking_reasons": blockers, + "same_physical_directory": same_physical_directory, + "overlaps_current_path": overlaps_current_path, + "impact": impact, + "can_rebind": can_rebind, + "preview_token": preview_token, + } + return preview, snapshot + + +async def _load_library_root_rebind_impact( + session: AsyncSession, + root: LibraryRoot, + *, + replacement_probe: _RootProbe, +) -> tuple[dict[str, int | bool], str, list[str]]: + ( + library_file_count, + library_file_blocking_count, + library_file_digest, + ) = await _inspect_rebind_path_scope( + session, + category="library_file", + id_column=LibraryFile.id, + path_column=LibraryFile.file_path, + root_column=LibraryFile.library_root_id, + root_id=root.id, + replacement_probe=replacement_probe, + ) + series_count = int( + await session.scalar(select(func.count(Series.id)).where(Series.library_root_id == root.id)) + or 0 + ) + _series_path_count, series_blocking_count, series_digest = await _inspect_rebind_path_scope( + session, + category="series", + id_column=Series.id, + path_column=Series.path, + root_column=Series.library_root_id, + root_id=root.id, + replacement_probe=replacement_probe, + exclude_null_paths=True, + ) + preferred_series_count = int( + await session.scalar( + select(func.count(Series.id)).where(Series.preferred_library_root_id == root.id) + ) + or 0 + ) + ( + story_arc_placement_count, + story_arc_placement_blocking_count, + story_arc_placement_digest, + ) = await _inspect_rebind_path_scope( + session, + category="story_arc_placement", + id_column=StoryArcPlacement.id, + path_column=StoryArcPlacement.placement_path, + root_column=StoryArcPlacement.library_root_id, + root_id=root.id, + replacement_probe=replacement_probe, + ) + impact: dict[str, int | bool] = { + "library_file_count": library_file_count, + "series_count": series_count, + "preferred_series_count": preferred_series_count, + "story_arc_placement_count": story_arc_placement_count, + "library_file_blocking_count": library_file_blocking_count, + "series_blocking_count": series_blocking_count, + "story_arc_placement_blocking_count": story_arc_placement_blocking_count, + "affects_default_destination": root.is_default_managed_destination, + "affects_preferred_series": preferred_series_count > 0, + } + association_digest = sha256( + ( + f"library_file:{library_file_digest}\n" + f"series:{series_digest}\n" + f"story_arc_placement:{story_arc_placement_digest}\n" + ).encode() + ).hexdigest() + blockers: list[str] = [] + if library_file_blocking_count: + blockers.append( + _path_migration_required_message( + library_file_blocking_count, + singular="registered library file path", + plural="registered library file paths", + ) + ) + if series_blocking_count: + blockers.append( + _path_migration_required_message( + series_blocking_count, + singular="current series path", + plural="current series paths", + ) + ) + if story_arc_placement_blocking_count: + blockers.append( + _path_migration_required_message( + story_arc_placement_blocking_count, + singular="Story Arc placement path", + plural="Story Arc placement paths", + ) + ) + return impact, association_digest, blockers + + +async def _inspect_rebind_path_scope( + session: AsyncSession, + *, + category: str, + id_column: Any, + path_column: Any, + root_column: Any, + root_id: int, + replacement_probe: _RootProbe, + exclude_null_paths: bool = False, +) -> tuple[int, int, str]: + """Inspect one path-bearing root scope in bounded keyset pages.""" + digest = sha256() + cursor = 0 + total_count = 0 + blocking_count = 0 + while True: + statement = select(id_column, path_column).where( + root_column == root_id, + id_column > cursor, + ) + if exclude_null_paths: + statement = statement.where(path_column.is_not(None)) + rows = ( + await session.execute(statement.order_by(id_column.asc()).limit(_REBIND_PATH_PAGE_SIZE)) + ).all() + if not rows: + break + for association_id, raw_path in rows: + path_value = str(raw_path) if raw_path is not None else "" + cursor = int(association_id) + total_count += 1 + digest.update(f"{category}|{cursor}|{path_value}\n".encode()) + if not _path_is_live_inside_replacement(path_value, replacement_probe): + blocking_count += 1 + if len(rows) < _REBIND_PATH_PAGE_SIZE: + break + return total_count, blocking_count, digest.hexdigest() + + +def _path_is_live_inside_replacement(raw_path: str, replacement_probe: _RootProbe) -> bool: + if ( + not raw_path + or not raw_path.isprintable() + or "\x00" in raw_path + or replacement_probe.resolved_path is None + ): + return False + untrusted = Path(raw_path) + if not untrusted.is_absolute() or ".." in untrusted.parts: + return False + candidate = Path(os.path.normpath(raw_path)) + if not ( + candidate == replacement_probe.display_path + or candidate.is_relative_to(replacement_probe.display_path) + ): + return False + try: + # Persisted paths are not rewritten by rebind, so both their lexical + # location and live resolved target must remain inside the replacement. + # codeql[py/path-injection] + resolved_candidate = candidate.resolve(strict=True) + except (OSError, RuntimeError): + return False + return ( + resolved_candidate == replacement_probe.resolved_path + or resolved_candidate.is_relative_to(replacement_probe.resolved_path) + ) + + +def _path_migration_required_message(count: int, *, singular: str, plural: str) -> str: + subject = singular if count == 1 else plural + verb = "falls" if count == 1 else "fall" + return ( + f"{count} {subject} {verb} outside or cannot be validated within the replacement root. " + "An explicit path migration or repair is required before rebinding." + ) + + +def _rebind_serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(get_application_secret(), salt=_REBIND_TOKEN_SALT) + + +def _build_rebind_preview_token(*, snapshot: dict[str, Any], actor_id: int) -> str: + return str( + _rebind_serializer().dumps( + { + "version": _REBIND_TOKEN_VERSION, + "action": _REBIND_ACTION, + "actor_id": actor_id, + "library_root_id": snapshot["root_id"], + "replacement_path": snapshot["replacement_path"], + "snapshot": snapshot, + } + ) + ) + + +def _load_rebind_preview_token(token: str) -> dict[str, object]: + try: + payload = _rebind_serializer().loads(token, max_age=_REBIND_TOKEN_MAX_AGE_SECONDS) + except SignatureExpired as exc: + raise ValidationError("The library root rebind preview expired. Preview it again.") from exc + except BadSignature as exc: + raise ValidationError( + "The library root rebind preview is invalid. Preview it again." + ) from exc + if not isinstance(payload, dict): + raise ValidationError("The library root rebind preview is invalid. Preview it again.") + return payload + + +def _validate_rebind_token_scope( + payload: dict[str, object], + *, + library_root_id: int, + replacement_path: str, + actor_id: int, +) -> dict[str, Any]: + snapshot = payload.get("snapshot") + if ( + payload.get("version") != _REBIND_TOKEN_VERSION + or payload.get("action") != _REBIND_ACTION + or payload.get("actor_id") != actor_id + or payload.get("library_root_id") != library_root_id + or payload.get("replacement_path") != replacement_path + or not isinstance(snapshot, dict) + ): + raise ValidationError("The library root rebind preview does not match this request.") + return snapshot + + +def _timestamp_value(value: object) -> str: + isoformat = getattr(value, "isoformat", None) + return str(isoformat(timespec="microseconds")) if callable(isoformat) else "" + + +def _probe_identity_digest(probe: _RootProbe) -> str | None: + if probe.resolved_path is None: + return None + identity = probe.identity or (0, 0) + payload = f"{probe.resolved_path}\0{identity[0]}\0{identity[1]}".encode() + return sha256(payload).hexdigest() + + +def _probes_share_identity(first: _RootProbe, second: _RootProbe) -> bool: + if first.identity is not None and second.identity is not None: + return first.identity == second.identity + return ( + first.resolved_path is not None + and second.resolved_path is not None + and first.resolved_path == second.resolved_path + ) + + +def _configured_roots_digest(roots: Sequence[LibraryRoot]) -> str: + payload = [ + { + "id": root.id, + "name": root.name, + "path": root.path, + "updated_at": _timestamp_value(root.updated_at), + } + for root in roots + ] + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + return sha256(encoded).hexdigest() + + +async def _has_active_import(session: AsyncSession) -> bool: + active_job_id = await session.scalar( + select(ImportJob.id).where(ImportJob.status.in_(tuple(ACTIVE_IMPORT_JOB_STATUSES))).limit(1) + ) + return active_job_id is not None + + +async def _assert_no_active_import(session: AsyncSession) -> None: + if await _has_active_import(session): + raise ValidationError(_MUTATION_BLOCKED_MESSAGE) + + +async def _clear_default(session: AsyncSession) -> None: + await session.execute( + update(LibraryRoot) + .where(LibraryRoot.is_default_managed_destination.is_(True)) + .values(is_default_managed_destination=False) + ) + await session.flush() + + +async def _sync_legacy_default(session: AsyncSession, path: str) -> None: + config = await session.get(SystemConfig, "comics_directory") + if config is None: + config = SystemConfig( + key="comics_directory", + value=path, + value_type="string", + description="Default managed library destination.", + ) + session.add(config) + else: + config.value = path + config.value_type = "string" + await session.flush() + + +def _normalize_name(name: str) -> str: + normalized = name.strip() + if not normalized or len(normalized) > 255 or not normalized.isprintable(): + raise ValidationError("Library root name must be printable and 1-255 characters long.") + return normalized + + +def _validate_roles( + *, + allow_referenced_registrations: bool, + allow_managed_writes: bool, + is_default_managed_destination: bool, +) -> None: + if not allow_referenced_registrations and not allow_managed_writes: + raise ValidationError("A library root must allow at least one role.") + if is_default_managed_destination and not allow_managed_writes: + raise ValidationError("The default managed destination must allow managed writes.") + + +def _normalize_candidate_path(raw_path: str) -> Path: + if not raw_path or len(raw_path) > 1000 or not raw_path.isprintable() or "\x00" in raw_path: + raise ValidationError("Library root path contains unsafe characters.") + untrusted = Path(raw_path) + if not untrusted.is_absolute(): + raise ValidationError("Library root path must be an absolute container-visible path.") + if ".." in untrusted.parts: + raise ValidationError("Library root path cannot contain traversal components.") + display_path = Path(os.path.normpath(raw_path)) + if display_path == Path("/"): + raise ValidationError("The filesystem root cannot be configured as a library root.") + return display_path + + +def _probe_candidate_root(raw_path: str, *, require_write_probe: bool) -> _RootProbe: + display_path = _normalize_candidate_path(raw_path) + return _probe_path( + display_path, + require_write_probe=require_write_probe, + reject_sensitive=True, + ) + + +def _probe_configured_root(raw_path: str, *, require_write_probe: bool) -> _RootProbe: + try: + display_path = _normalize_candidate_path(raw_path) + except ValidationError as exc: + return _unavailable_probe(Path(raw_path), exc.message) + return _probe_path( + display_path, + require_write_probe=require_write_probe, + reject_sensitive=False, + ) + + +def _probe_path( + display_path: Path, + *, + require_write_probe: bool, + reject_sensitive: bool, +) -> _RootProbe: + try: + # The authenticated root workflow already requires a bounded absolute + # path without traversal; this strict resolution is the safety probe. + # codeql[py/path-injection] + resolved = display_path.resolve(strict=True) + except (OSError, RuntimeError): + return _unavailable_probe(display_path, "Library root path must be an existing directory.") + if not resolved.is_dir(): + return _unavailable_probe(display_path, "Library root path must be an existing directory.") + if is_sensitive_path(resolved): + message = "Sensitive system directories cannot be configured as library roots." + if reject_sensitive: + raise ValidationError(message) + return _unavailable_probe(display_path, message) + + readable = _can_read_and_traverse(resolved) + writable = _can_write(resolved, create_probe=require_write_probe) + try: + usage = shutil.disk_usage(resolved) + free_bytes: int | None = usage.free + except OSError: + free_bytes = None + try: + stat_result = resolved.stat() + identity = _filesystem_identity(stat_result.st_dev, stat_result.st_ino) + except OSError: + identity = None + + warnings: list[str] = [] + if not readable: + warnings.append("Directory is not readable and traversable by Pullbox.") + elif not writable: + warnings.append("Directory is read-only to Pullbox.") + if free_bytes is not None and free_bytes < _LOW_CAPACITY_BYTES: + warnings.append("Directory has less than 1 GiB of free space.") + if not readable: + status = "unavailable" + elif not writable: + status = "read_only" + elif free_bytes is not None and free_bytes < _LOW_CAPACITY_BYTES: + status = "low_capacity" + else: + status = "ready" + return _RootProbe( + display_path=display_path, + resolved_path=resolved, + identity=identity, + available=True, + readable=readable, + writable=writable, + free_bytes=free_bytes, + status=status, + warnings=tuple(warnings), + blocking_reasons=(), + ) + + +def _unavailable_probe(display_path: Path, reason: str) -> _RootProbe: + return _RootProbe( + display_path=display_path, + resolved_path=None, + identity=None, + available=False, + readable=False, + writable=False, + free_bytes=None, + status="unavailable", + warnings=(reason,), + blocking_reasons=(reason,), + ) + + +def _can_read_and_traverse(path: Path) -> bool: + if not os.access(path, os.R_OK | os.X_OK): + return False + try: + # Root validation intentionally opens only the selected directory and + # returns no entry names or contents. + # codeql[py/path-injection] + with os.scandir(path) as entries: + next(entries, None) + except OSError: + return False + return True + + +def _can_write(path: Path, *, create_probe: bool) -> bool: + if not os.access(path, os.W_OK | os.X_OK): + return False + if not create_probe: + return True + descriptor: int | None = None + probe_path: str | None = None + try: + # The directory passed the root-path policy; mkstemp owns the random + # leaf name and the probe is removed before this call returns. + # codeql[py/path-injection] + descriptor, probe_path = tempfile.mkstemp(prefix=".pullbox-root-probe-", dir=path) + os.close(descriptor) + descriptor = None + os.unlink(probe_path) + probe_path = None + except OSError: + return False + finally: + if descriptor is not None: + with suppress(OSError): + os.close(descriptor) + if probe_path is not None: + with suppress(OSError): + os.unlink(probe_path) + return True + + +def _capability_blockers( + probe: _RootProbe, + *, + allow_referenced_registrations: bool, + allow_managed_writes: bool, +) -> list[str]: + blockers: list[str] = [] + if allow_referenced_registrations and not probe.readable: + blockers.append("Referenced registrations require a readable, traversable directory.") + if allow_managed_writes and (not probe.readable or not probe.writable): + blockers.append("Managed writes require a readable, traversable, writable directory.") + return blockers + + +def _root_conflicts( + roots: Sequence[LibraryRoot], + *, + name: str, + probe: _RootProbe, + exclude_root_id: int | None, +) -> list[str]: + conflicts: list[str] = [] + for root in roots: + if root.id == exclude_root_id: + continue + if root.name.strip().casefold() == name.casefold(): + conflicts.append("Library root names must be unique, ignoring case.") + + existing_display = Path(os.path.normpath(root.path)) + if existing_display == probe.display_path: + conflicts.append("This library root path is already configured.") + continue + existing_resolved: Path | None = None + existing_identity: tuple[int, int] | None = None + try: + existing_resolved = existing_display.resolve(strict=True) + stat_result = existing_resolved.stat() + existing_identity = _filesystem_identity(stat_result.st_dev, stat_result.st_ino) + except (OSError, RuntimeError): + pass + if ( + probe.identity is not None + and existing_identity is not None + and probe.identity == existing_identity + ): + conflicts.append("This path resolves to the same physical directory as another root.") + continue + if _paths_overlap(probe.display_path, existing_display) or ( + probe.resolved_path is not None + and existing_resolved is not None + and _paths_overlap(probe.resolved_path, existing_resolved) + ): + conflicts.append("This library root overlaps or is nested inside another root.") + return _deduplicate(conflicts) + + +def _paths_overlap(first: Path, second: Path) -> bool: + return first == second or first in second.parents or second in first.parents + + +def _filesystem_identity(device: int, inode: int) -> tuple[int, int] | None: + """Return useful identity evidence without conflating zero-inode network mounts.""" + return (device, inode) if inode else None + + +def _deduplicate(messages: Sequence[str]) -> list[str]: + return list(dict.fromkeys(messages)) + + +def _raise_blockers(blockers: Sequence[str]) -> None: + unique = _deduplicate(blockers) + if unique: + raise ValidationError(unique[0], details={"blocking_reasons": unique}) + + +def _capability_dict(probe: _RootProbe) -> dict[str, Any]: + return { + "available": probe.available, + "readable": probe.readable, + "writable": probe.writable, + "free_bytes": probe.free_bytes, + "status": probe.status, + "warnings": list(probe.warnings), + } + + +def _serialize_root( + root: LibraryRoot, + probe: _RootProbe, + *, + extra_warnings: Sequence[str] = (), +) -> dict[str, Any]: + warnings = list(probe.warnings) + warnings.extend(extra_warnings) + if not root.enabled: + warnings.append("Library root is disabled.") + return { + "id": root.id, + "name": root.name, + "path": root.path, + "enabled": root.enabled, + "allow_referenced_registrations": root.allow_referenced_registrations, + "allow_managed_writes": root.allow_managed_writes, + "is_default_managed_destination": root.is_default_managed_destination, + **_capability_dict(probe), + "warnings": _deduplicate(warnings), + "can_disable": not root.is_default_managed_destination, + } diff --git a/src/pullbox/services/library_root_policy_service.py b/src/pullbox/services/library_root_policy_service.py new file mode 100644 index 00000000..f4e73c71 --- /dev/null +++ b/src/pullbox/services/library_root_policy_service.py @@ -0,0 +1,303 @@ +"""Read, preview, and mutate explicit naming policies for library roots.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from sqlalchemy import select + +from pullbox.core.exceptions import PullboxError +from pullbox.core.library_naming import build_series_relative_path, compute_target_filename +from pullbox.core.library_policy import load_effective_library_ingest_policy +from pullbox.models.issue import Issue, IssueType +from pullbox.models.library import LibraryRoot, LibraryRootPolicy, LibraryRootPolicySource +from pullbox.models.publisher import Publisher +from pullbox.models.series import Series +from pullbox.services.import_root_policy_activation import ( + RootPolicyActivationConflictError, + apply_future_root_policy_to_ingest_policy, + normalize_root_policy_definition, +) + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.core.library_policy import LibraryIngestPolicy + + +class LibraryRootNotFoundError(PullboxError): + """The selected library root does not exist.""" + + def __init__(self) -> None: + super().__init__( + message="Library root not found.", + code="LIBRARY_ROOT_NOT_FOUND", + status_code=404, + ) + + +async def get_library_root_policy_state( + session: AsyncSession, + library_root_id: int, +) -> dict[str, object]: + """Return the root's explicit scope and complete effective naming policy.""" + root = await _load_root(session, library_root_id) + return await _serialize_state(session, root) + + +async def update_library_root_policy( + session: AsyncSession, + library_root_id: int, + *, + expected_revision: int, + definition: Mapping[str, object], +) -> dict[str, object]: + """Create or replace a root override after an optimistic revision check.""" + root = await _load_root(session, library_root_id, for_update=True) + current = await _load_explicit_policy(session, root.id, for_update=True) + _assert_revision(current, expected_revision) + proposal = normalize_root_policy_definition(definition) + next_revision = (current.revision if current is not None else 0) + 1 + + if current is None: + current = LibraryRootPolicy( + library_root_id=root.id, + schema_version=1, + series_path_template=str(proposal["series_path_template"]), + comic_file_template=str(proposal["comic_file_template"]), + annual_file_template=str(proposal["annual_file_template"]), + non_standard_file_template=str(proposal["non_standard_file_template"]), + single_non_standard_file_template=str(proposal["single_non_standard_file_template"]), + replace_illegal_characters=bool(proposal["replace_illegal_characters"]), + colon_replacement=str(proposal["colon_replacement"]), + source=LibraryRootPolicySource.MANUAL, + source_import_job_id=None, + revision=next_revision, + ) + session.add(current) + else: + _assign_definition(current, proposal) + current.source = LibraryRootPolicySource.MANUAL + current.source_import_job_id = None + current.revision = next_revision + + await session.flush() + return await _serialize_state(session, root) + + +async def clear_library_root_policy( + session: AsyncSession, + library_root_id: int, + *, + expected_revision: int, +) -> dict[str, object]: + """Remove a root override and resume inheritance from global defaults.""" + root = await _load_root(session, library_root_id, for_update=True) + current = await _load_explicit_policy(session, root.id, for_update=True) + _assert_revision(current, expected_revision) + if current is not None: + await session.delete(current) + await session.flush() + return await _serialize_state(session, root) + + +async def preview_library_root_policy( + session: AsyncSession, + library_root_id: int, + *, + definition: Mapping[str, object], + examples: Sequence[Mapping[str, object]] = (), +) -> dict[str, object]: + """Render current and proposed outputs without persisting the proposal.""" + root = await _load_root(session, library_root_id) + current = await load_effective_library_ingest_policy(session, root) + proposal = normalize_root_policy_definition(definition) + proposed = apply_future_root_policy_to_ingest_policy(current, proposal) + subjects = _preview_subjects(examples) + + return { + "current_scope": _scope(current), + "current_series_paths": _series_path_examples(current, subjects), + "proposed_series_paths": _series_path_examples(proposed, subjects), + "current_file_names": _file_name_examples(current, subjects), + "proposed_file_names": _file_name_examples(proposed, subjects), + } + + +async def _load_root( + session: AsyncSession, + library_root_id: int, + *, + for_update: bool = False, +) -> LibraryRoot: + statement = select(LibraryRoot).where(LibraryRoot.id == library_root_id) + if for_update: + statement = statement.with_for_update() + root = await session.scalar(statement) + if root is None: + raise LibraryRootNotFoundError() + return root + + +async def _load_explicit_policy( + session: AsyncSession, + library_root_id: int, + *, + for_update: bool = False, +) -> LibraryRootPolicy | None: + statement = select(LibraryRootPolicy).where( + LibraryRootPolicy.library_root_id == library_root_id + ) + if for_update: + statement = statement.with_for_update() + return cast("LibraryRootPolicy | None", await session.scalar(statement)) + + +def _assert_revision(policy: LibraryRootPolicy | None, expected_revision: int) -> None: + current_revision = policy.revision if policy is not None else 0 + if expected_revision != current_revision: + raise RootPolicyActivationConflictError( + "Library root policy changed after it was loaded; refresh and try again." + ) + + +def _assign_definition( + policy: LibraryRootPolicy, + proposal: Mapping[str, object], +) -> None: + policy.schema_version = 1 + policy.series_path_template = str(proposal["series_path_template"]) + policy.comic_file_template = str(proposal["comic_file_template"]) + policy.annual_file_template = str(proposal["annual_file_template"]) + policy.non_standard_file_template = str(proposal["non_standard_file_template"]) + policy.single_non_standard_file_template = str(proposal["single_non_standard_file_template"]) + policy.replace_illegal_characters = bool(proposal["replace_illegal_characters"]) + policy.colon_replacement = str(proposal["colon_replacement"]) + + +async def _serialize_state( + session: AsyncSession, + root: LibraryRoot, +) -> dict[str, object]: + policy = await load_effective_library_ingest_policy(session, root) + return { + "library_root_id": root.id, + "library_root_name": root.name, + "scope": _scope(policy), + "policy_id": policy.root_policy_id, + "revision": policy.policy_revision, + "effective_policy": { + "schema_version": 1, + "series_path_template": policy.series_path_template or policy.series_folder_template, + "series_folder_template": policy.series_folder_template, + "comic_file_template": policy.comic_file_template, + "annual_file_template": policy.annual_file_template, + "non_standard_file_template": policy.non_standard_file_template, + "single_non_standard_file_template": policy.single_non_standard_file_template, + "replace_illegal_characters": policy.replace_illegal_characters, + "colon_replacement": policy.colon_replacement, + "source": str(policy.policy_source), + "source_import_job_id": policy.source_import_job_id, + }, + } + + +def _scope(policy: LibraryIngestPolicy) -> str: + return "root_override" if policy.root_policy_id is not None else "global_default" + + +def _sample_series() -> Series: + publisher = Publisher(name="DC Comics") + return Series( + title="Batman", + sort_title="batman", + year_start=2024, + publisher=publisher, + ) + + +def _preview_subjects( + examples: Sequence[Mapping[str, object]], +) -> list[tuple[Series, Issue]]: + if examples: + subjects: list[tuple[Series, Issue]] = [] + for example in examples: + publisher_value = example.get("publisher") + publisher = ( + Publisher(name=publisher_value) + if isinstance(publisher_value, str) and publisher_value + else None + ) + series_value = example.get("series") + year_value = example.get("year") + issue_number_value = example.get("issue_number") + issue_title_value = example.get("issue_title") + if not isinstance(series_value, str) or not series_value: + continue + if isinstance(issue_number_value, bool) or not isinstance( + issue_number_value, int | float + ): + continue + series = Series( + title=series_value, + sort_title=series_value.casefold(), + year_start=year_value if isinstance(year_value, int) else None, + publisher=publisher, + ) + issue = Issue( + series=series, + issue_number=float(issue_number_value), + title=issue_title_value if isinstance(issue_title_value, str) else None, + issue_type=IssueType.ISSUE, + ) + subjects.append((series, issue)) + if subjects: + return subjects + + series = _sample_series() + return [ + ( + series, + Issue( + series=series, + issue_number=number, + title=title, + issue_type=issue_type, + ), + ) + for number, title, issue_type in ( + (17.0, "The Brave and the Bold", IssueType.ISSUE), + (1.0, "Annual Adventure", IssueType.ANNUAL), + (1.0, "The Long Halloween", IssueType.ONE_SHOT), + ) + ] + + +def _series_path_examples( + policy: LibraryIngestPolicy, + subjects: Sequence[tuple[Series, Issue]], +) -> list[str]: + paths: list[str] = [] + for series, _issue in subjects: + path = build_series_relative_path(series, policy).as_posix() + if path not in paths: + paths.append(path) + return paths + + +def _file_name_examples( + policy: LibraryIngestPolicy, + subjects: Sequence[tuple[Series, Issue]], +) -> list[str]: + return [ + compute_target_filename( + issue, + series, + Path("sample.cbz"), + policy, + ) + for series, issue in subjects + ] diff --git a/src/pullbox/services/library_root_removal.py b/src/pullbox/services/library_root_removal.py new file mode 100644 index 00000000..68007ae3 --- /dev/null +++ b/src/pullbox/services/library_root_removal.py @@ -0,0 +1,222 @@ +"""Preview and remove unused roots without touching physical library files.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from itsdangerous import BadSignature, URLSafeTimedSerializer +from sqlalchemy import delete, func, select, update +from sqlalchemy.exc import IntegrityError + +from pullbox.core.config_resolver import get_application_secret +from pullbox.core.exceptions import ValidationError +from pullbox.models.config import SystemConfig +from pullbox.models.import_job import ImportJob +from pullbox.models.library import LibraryFile, LibraryRoot, LibraryRootPolicy +from pullbox.models.series import Series +from pullbox.models.story_arc import StoryArc, StoryArcPlacement +from pullbox.services.import_workflow_state import ACTIVE_IMPORT_JOB_STATUSES +from pullbox.services.library_root_policy_service import LibraryRootNotFoundError +from pullbox.utilities.models import JobState, UtilityJob + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +_ACTIVE_UTILITIES = tuple( + state + for state in JobState + if state + not in { + JobState.COMPLETED, + JobState.FAILED, + JobState.CANCELLED, + JobState.ROLLED_BACK, + } +) + + +def _serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(get_application_secret(), salt="library-root-remove-v1") + + +def _dependencies(root_id: int, path: str) -> list[tuple[str, Any, str]]: + return [ + ( + "library_files", + LibraryFile.library_root_id == root_id, + "registered files still use this root. Relocate or remove those library entries " + "first; disabling does not detach them.", + ), + ( + "series", + Series.library_root_id == root_id, + "series still use this root. Relocate or remove those series first.", + ), + ( + "preferred_series", + Series.preferred_library_root_id == root_id, + "series use this as their preferred destination. " + "Choose another destination for those series first.", + ), + ( + "story_arcs", + StoryArc.target_library_root_id == root_id, + "Story Arcs target this root. Change their placement destinations first.", + ), + ( + "story_arc_placements", + StoryArcPlacement.library_root_id == root_id, + "Story Arc placements still use this root. Resolve those placements first.", + ), + ( + "destination_settings", + ( + ( + (SystemConfig.key == "story_arc_files_library_root_id") + & (SystemConfig.value == str(root_id)) + ) + | ((SystemConfig.key == "comics_directory") & (SystemConfig.value == path)) + ), + "destination settings still use this root. " + "Choose another default library or Story Arc destination first.", + ), + ( + "active_imports", + ImportJob.status.in_(tuple(ACTIVE_IMPORT_JOB_STATUSES)), + "imports are active or paused. Finish or cancel them before changing library roots.", + ), + ( + "active_utilities", + UtilityJob.state.in_(_ACTIVE_UTILITIES), + "file utilities are active or paused. Finish or cancel them before removing a root.", + ), + ( + "pending_import_followups", + (ImportJob.target_library_root_id == root_id) + & ( + ImportJob.story_arc_placement_followup_pending.is_(True) + | ImportJob.story_arc_rollback_waiting_work_id.is_not(None) + ), + "import follow-ups still depend on this root. " + "Complete their placement or rollback first.", + ), + ] + + +async def preview_library_root_removal( + session: AsyncSession, + library_root_id: int, + *, + actor_id: int, +) -> dict[str, Any]: + """Return exact blockers without probing or changing the filesystem.""" + root = await session.get(LibraryRoot, library_root_id, populate_existing=True) + if root is None: + raise LibraryRootNotFoundError() + blockers = [] + if root.enabled: + blockers.append("Disable this library root before removing it.") + if root.is_default_managed_destination: + blockers.append("Select another default managed destination before removing this root.") + counts = {} + for key, condition, explanation in _dependencies(root.id, root.path): + count = int(await session.scalar(select(func.count()).where(condition)) or 0) + counts[key] = count + if count: + blockers.append(f"{count} {explanation}") + history_count = int( + await session.scalar( + select(func.count(ImportJob.id)).where( + ImportJob.target_library_root_id == root.id, + ) + ) + or 0 + ) + policy = await session.scalar( + select(LibraryRootPolicy).where(LibraryRootPolicy.library_root_id == root.id) + ) + snapshot = {"id": root.id, "name": root.name, "path": root.path} + signed_state = { + "root": snapshot, + "actor_id": actor_id, + "updated_at": str(root.updated_at), + "history_count": history_count, + "policy_updated_at": str(policy.updated_at) if policy else None, + } + return { + **snapshot, + "can_remove": not blockers, + "blocking_reasons": blockers, + "dependencies": counts, + "history_count": history_count, + "has_naming_policy": policy is not None, + "preview_token": _serializer().dumps(signed_state) if not blockers else None, + } + + +async def remove_library_root( + session: AsyncSession, + library_root_id: int, + *, + actor_id: int, + preview_token: str, +) -> None: + """Recheck under a short write lock; retain detached historical identity.""" + try: + confirmed = _serializer().loads(preview_token, max_age=900) + except BadSignature as exc: + raise ValidationError( + "Removal preview expired or is invalid. Review the root again." + ) from exc + # SQLite needs a write reservation, not SELECT FOR UPDATE, before the recheck. + await session.execute( + update(LibraryRoot) + .where(LibraryRoot.id == library_root_id) + .values( + updated_at=LibraryRoot.updated_at, + ) + ) + preview = await preview_library_root_removal(session, library_root_id, actor_id=actor_id) + if preview["blocking_reasons"]: + raise ValidationError(" ".join(preview["blocking_reasons"])) + current = _serializer().loads(preview["preview_token"], max_age=900) + if confirmed != current: + raise ValidationError("Removal preview changed. Review the root again before confirming.") + snapshot = {key: preview[key] for key in ("id", "name", "path")} + await session.execute( + update(ImportJob) + .where( + ImportJob.target_library_root_id == library_root_id, + ImportJob.status.not_in(tuple(ACTIVE_IMPORT_JOB_STATUSES)), + ImportJob.story_arc_placement_followup_pending.is_(False), + ImportJob.story_arc_rollback_waiting_work_id.is_(None), + ) + .values( + removed_library_root_snapshot=snapshot, + target_library_root_id=None, + ) + ) + guards = [ + ~select(1).where(condition).exists() + for _, condition, _ in _dependencies(library_root_id, preview["path"]) + ] + await session.execute( + delete(LibraryRootPolicy).where(LibraryRootPolicy.library_root_id == library_root_id) + ) + try: + deleted = await session.scalar( + delete(LibraryRoot) + .where( + LibraryRoot.id == library_root_id, + LibraryRoot.enabled.is_(False), + LibraryRoot.is_default_managed_destination.is_(False), + *guards, + ) + .returning(LibraryRoot.id) + ) + except IntegrityError as exc: + raise ValidationError( + "This root gained a dependency. Review it again before removing it." + ) from exc + if deleted is None: + raise ValidationError("This root gained a dependency. Review it again before removing it.") diff --git a/src/pullbox/services/library_service.py b/src/pullbox/services/library_service.py index 3b223904..d20fc77e 100644 --- a/src/pullbox/services/library_service.py +++ b/src/pullbox/services/library_service.py @@ -2,16 +2,14 @@ from __future__ import annotations -import os from pathlib import Path -from typing import TYPE_CHECKING, overload +from typing import TYPE_CHECKING import structlog from sqlalchemy import func, select from pullbox.models.config import SystemConfig from pullbox.models.library import FileFormat, LibraryFile, LibraryRoot, MatchConfidence -from pullbox.models.series import Series if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -24,27 +22,6 @@ def _normalize_library_path(path: str | Path) -> str: return str(Path(path).expanduser().resolve(strict=False)) -@overload -def _rewrite_library_prefix(path_value: str, old_prefix: str, new_prefix: str) -> str: ... - - -@overload -def _rewrite_library_prefix(path_value: None, old_prefix: str, new_prefix: str) -> None: ... - - -def _rewrite_library_prefix(path_value: str | None, old_prefix: str, new_prefix: str) -> str | None: - """Rewrite one absolute library path from an old root prefix to a new one.""" - if not path_value: - return path_value - - normalized = _normalize_library_path(path_value) - if normalized == old_prefix: - return new_prefix - if normalized.startswith(old_prefix + os.sep): - return new_prefix + normalized[len(old_prefix) :] - return path_value - - class LibraryService: """Read-only library statistics and unmatched file queries.""" @@ -114,10 +91,11 @@ async def get_comics_directory(session: AsyncSession) -> Path | None: async def set_comics_directory(session: AsyncSession, path: Path) -> LibraryRoot: - """Set the primary comics directory. + """Set the legacy primary comics directory through explicit root management. - Validates the path, stores it in SystemConfig, and ensures a - LibraryRoot record exists for the path. + This compatibility entry point preserves path identity: an existing root is + promoted, while a new path creates a separate validated root. It never + rewrites paths owned by an established root. Raises: ValueError: If the path does not exist or is not a directory. @@ -132,44 +110,83 @@ async def set_comics_directory(session: AsyncSession, path: Path) -> LibraryRoot raise ValueError(f"Path '{path}' is not a directory") path = path.resolve(strict=True) - # Upsert the SystemConfig row - row = await session.get(SystemConfig, "comics_directory") - if row: - row.value = str(path) - else: - session.add(SystemConfig(key="comics_directory", value=str(path), value_type="string")) + from pullbox.core.exceptions import ValidationError + from pullbox.services.library_root_management import ( + create_library_root, + update_library_root, + ) - # Find or create a LibraryRoot for this path result = await session.execute(select(LibraryRoot).where(LibraryRoot.path == str(path))) root = result.scalar_one_or_none() + try: + if root is not None: + configured_names = { + name.casefold() + for name in ( + await session.scalars(select(LibraryRoot.name).where(LibraryRoot.id != root.id)) + ).all() + } + promoted_name = ( + root.name if "comics directory" in configured_names else "Comics Directory" + ) + await update_library_root( + session, + root.id, + { + "name": promoted_name, + "enabled": True, + "allow_referenced_registrations": True, + "allow_managed_writes": True, + "is_default_managed_destination": True, + }, + ) + return root - if root: - root.name = "Comics Directory" - root.enabled = True - else: - root = LibraryRoot(name="Comics Directory", path=str(path), enabled=True) - session.add(root) + configured_names = { + name.casefold() for name in (await session.scalars(select(LibraryRoot.name))).all() + } + name = "Comics Directory" + suffix = 2 + while name.casefold() in configured_names: + name = f"Comics Directory {suffix}" + suffix += 1 + created = await create_library_root( + session, + name=name, + path=str(path), + allow_referenced_registrations=True, + allow_managed_writes=True, + is_default_managed_destination=True, + ) + except ValidationError as exc: + raise ValueError(exc.message) from exc - await session.flush() - return root + created_root = await session.get(LibraryRoot, int(created["id"])) + if created_root is None: # pragma: no cover - guarded by the flush above + raise RuntimeError("Created library root could not be reloaded.") + return created_root async def reconcile_runtime_library_paths( session: AsyncSession, runtime_root: Path, -) -> dict[str, int | str] | None: - """Reconcile persisted library paths to the active runtime library root. - - Pullbox stores library roots, series folders, and tracked file paths as - absolute paths. When the app moves between host-local development and a - containerized runtime, those absolute prefixes can drift even though both - runtimes point at the same mounted library data. This helper rewrites the - persisted primary library prefix so health checks and filesystem operations - continue to target the active runtime path. +) -> dict[str, bool | int | str] | None: + """Bootstrap a fresh runtime root without rebinding established paths. + + A changed container path is not proof that it represents the same physical + library. Established roots and tracked paths therefore remain untouched + until an operator completes an explicit rebind workflow. """ runtime_root_str = _normalize_library_path(runtime_root) config_row = await session.get(SystemConfig, "comics_directory") - if config_row is None or not config_row.value.strip(): + roots = list((await session.execute(select(LibraryRoot).order_by(LibraryRoot.id))).scalars()) + stored_root_str = ( + _normalize_library_path(config_row.value) + if config_row is not None and config_row.value.strip() + else "" + ) + + if not stored_root_str and not roots: if config_row is None: config_row = SystemConfig( key="comics_directory", @@ -180,137 +197,69 @@ async def reconcile_runtime_library_paths( else: config_row.value = runtime_root_str - root_result = await session.execute( - select(LibraryRoot).where(LibraryRoot.path == runtime_root_str) + root = LibraryRoot( + name="Comics Directory", + path=runtime_root_str, + enabled=True, + allow_referenced_registrations=True, + allow_managed_writes=True, + is_default_managed_destination=True, ) - root = root_result.scalar_one_or_none() - if root is None: - root = LibraryRoot( - name="Comics Directory", - path=runtime_root_str, - enabled=True, - ) - session.add(root) - else: - root.name = "Comics Directory" - root.enabled = True + session.add(root) await session.flush() return { + "status": "bootstrapped", "old_root": "", "new_root": runtime_root_str, "series_updated": 0, "library_files_updated": 0, + "rebind_required": False, } - stored_root_str = _normalize_library_path(config_row.value) - roots = list((await session.execute(select(LibraryRoot))).scalars().all()) - target_root = next( + runtime_record = next( (root for root in roots if _normalize_library_path(root.path) == runtime_root_str), None, ) if stored_root_str == runtime_root_str: - if target_root is None: - target_root = LibraryRoot( + if runtime_record is None: + runtime_record = LibraryRoot( name="Comics Directory", path=runtime_root_str, enabled=True, + allow_referenced_registrations=True, + allow_managed_writes=True, + is_default_managed_destination=not any( + root.is_default_managed_destination for root in roots + ), ) - session.add(target_root) - elif not target_root.enabled: - target_root.name = "Comics Directory" - target_root.enabled = True - else: - return None - - await session.flush() - return { - "old_root": stored_root_str, - "new_root": runtime_root_str, - "series_updated": 0, - "library_files_updated": 0, - } - - config_row.value = runtime_root_str - - old_root = next( - (root for root in roots if _normalize_library_path(root.path) == stored_root_str), - None, - ) - - if target_root is None: - if old_root is not None: - old_root.path = runtime_root_str - old_root.enabled = True - target_root = old_root - else: - target_root = LibraryRoot( - name="Comics Directory", - path=runtime_root_str, - enabled=True, - ) - session.add(target_root) + session.add(runtime_record) await session.flush() + return { + "status": "bootstrapped", + "old_root": stored_root_str, + "new_root": runtime_root_str, + "series_updated": 0, + "library_files_updated": 0, + "rebind_required": False, + } + if not runtime_record.enabled: + return { + "status": "root_unavailable", + "old_root": stored_root_str, + "new_root": runtime_root_str, + "series_updated": 0, + "library_files_updated": 0, + "rebind_required": False, + } + return None - series_rows = list((await session.execute(select(Series))).scalars().all()) - library_files = list((await session.execute(select(LibraryFile))).scalars().all()) - - series_updated = 0 - library_files_updated = 0 - for series in series_rows: - next_path = _rewrite_library_prefix(series.path, stored_root_str, runtime_root_str) - root_id_changed = False - if ( - old_root is not None - and target_root is not None - and series.library_root_id == old_root.id - and target_root.id is not None - and series.library_root_id != target_root.id - ): - series.library_root_id = target_root.id - root_id_changed = True - if next_path != series.path: - series.path = next_path - series_updated += 1 - elif root_id_changed: - series_updated += 1 - - for library_file in library_files: - next_path = _rewrite_library_prefix( - library_file.file_path, - stored_root_str, - runtime_root_str, - ) - root_id_changed = False - if ( - old_root is not None - and target_root is not None - and library_file.library_root_id == old_root.id - and target_root.id is not None - and library_file.library_root_id != target_root.id - ): - library_file.library_root_id = target_root.id - root_id_changed = True - if next_path != library_file.file_path: - library_file.file_path = next_path - library_file.file_name = Path(next_path).name - library_files_updated += 1 - elif root_id_changed: - library_files_updated += 1 - - if ( - old_root is not None - and target_root is not None - and old_root.id != target_root.id - and old_root.path != runtime_root_str - ): - old_root.enabled = False - - await session.flush() return { + "status": "rebind_required", "old_root": stored_root_str, "new_root": runtime_root_str, - "series_updated": series_updated, - "library_files_updated": library_files_updated, + "series_updated": 0, + "library_files_updated": 0, + "rebind_required": True, } diff --git a/src/pullbox/services/matching_service.py b/src/pullbox/services/matching_service.py index f8c762b2..aeb74a84 100644 --- a/src/pullbox/services/matching_service.py +++ b/src/pullbox/services/matching_service.py @@ -17,6 +17,10 @@ from pullbox.core.archive import ArchiveError from pullbox.core.events import EventBus, FileMatched +from pullbox.core.issue_numbers import ( + issue_number_text_matches_numeric, + normalize_issue_number_text, +) from pullbox.core.name_matcher import NameMatcher from pullbox.core.naming import ( detect_series_type, @@ -371,7 +375,12 @@ async def _match_metadata_to_candidates( matched_series_only = True continue - issue = await _find_issue(session, series.id, metadata.issue_number) + issue = await _find_issue( + session, + series.id, + metadata.issue_number, + issue_number_text=metadata.issue_number_text, + ) if issue is None: logger.debug( "matching_semantic_series_hit_no_issue", @@ -493,12 +502,32 @@ async def _find_issue( session: AsyncSession, series_id: int, issue_number: float, + *, + issue_number_text: str | None = None, ) -> Issue | None: - """Find an issue by series ID and issue number.""" + """Find one exact issue, failing closed on ambiguous legacy numeric identity.""" + if issue_number_text is not None: + try: + normalized_text = normalize_issue_number_text(issue_number_text) + except ValueError: + return None + if not issue_number_text_matches_numeric(issue_number, normalized_text): + return None + result = await session.execute( + select(Issue).where( + Issue.series_id == series_id, + Issue.issue_number_text == normalized_text, + ) + ) + return result.scalar_one_or_none() + result = await session.execute( - select(Issue).where( + select(Issue) + .where( Issue.series_id == series_id, Issue.issue_number == issue_number, ) + .limit(2) ) - return result.scalar_one_or_none() + candidates = list(result.scalars().all()) + return candidates[0] if len(candidates) == 1 else None diff --git a/src/pullbox/services/metadata_service.py b/src/pullbox/services/metadata_service.py index ac072494..6bc8aa82 100644 --- a/src/pullbox/services/metadata_service.py +++ b/src/pullbox/services/metadata_service.py @@ -6,6 +6,7 @@ from __future__ import annotations +import asyncio import contextlib from datetime import UTC, date, datetime, timedelta from pathlib import Path # noqa: TC003 — used at runtime via parameter values @@ -15,6 +16,7 @@ from sqlalchemy import delete, select from pullbox.core.exceptions import NotFoundError, ProviderError +from pullbox.core.issue_numbers import format_issue_number, normalize_issue_number_text from pullbox.core.name_matcher import NameMatcher from pullbox.core.naming import ( classify_series_type, @@ -34,13 +36,15 @@ ) from pullbox.providers.base import SeriesMetadata from pullbox.providers.metadata.comicvine import ComicVineError +from pullbox.services.catalog.reader import CatalogIssueSummary, CatalogSeriesMetadata from pullbox.services.cover_cache_service import purge_series_cover_cache if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession - from pullbox.providers.base import IssueSummary + from pullbox.providers.base import IssueMetadata, IssueSummary from pullbox.providers.metadata.comicvine import ComicVineProvider + from pullbox.services.catalog.reader import CatalogReader logger = structlog.get_logger(__name__) @@ -54,6 +58,16 @@ def _provider_error_from_comicvine(exc: ComicVineError) -> ProviderError: ) +def _exact_issue_number_text( + issue_number: float, + issue_number_text: str | None, +) -> str: + """Normalize provider exact text or derive it for legacy provider DTOs.""" + if issue_number_text is None: + return format_issue_number(issue_number) + return normalize_issue_number_text(issue_number_text) + + # Mapping from IssueType → SeriesType for propagating detected issue types # to the parent series when all issues share a single non-standard type. _ISSUE_TO_SERIES_TYPE: dict[IssueType, SeriesType] = { @@ -117,10 +131,13 @@ def __init__( provider: ComicVineProvider, covers_dir: Path, refresh_days: int = 30, + *, + catalog: CatalogReader | None = None, ) -> None: self._provider = provider self._covers_dir = covers_dir self._refresh_days = refresh_days + self._catalog = catalog async def fetch_series( self, @@ -138,7 +155,7 @@ async def fetch_series( log = logger.bind(comicvine_id=comicvine_id) log.debug("metadata_fetch_series") - meta = await self.get_series_metadata(comicvine_id) + meta = await self.get_series_metadata(comicvine_id, use_catalog=False) series = await self.upsert_series_metadata( session, comicvine_id, @@ -164,6 +181,11 @@ async def upsert_series_metadata( """ log = logger.bind(comicvine_id=comicvine_id) + source = "pullbox_catalog" if isinstance(meta, CatalogSeriesMetadata) else "comicvine" + refreshed_at = ( + meta.source_cutoff_at if isinstance(meta, CatalogSeriesMetadata) else datetime.now(UTC) + ) + publisher_id = None if meta.publisher: publisher_id = await self._ensure_publisher(session, meta.publisher) @@ -172,6 +194,13 @@ async def upsert_series_metadata( await session.execute(select(Series).where(Series.comicvine_id == comicvine_id)) ).scalar_one_or_none() + if ( + existing + and isinstance(meta, CatalogSeriesMetadata) + and existing.metadata_source == "comicvine" + ): + # Basic catalog hydration must never replace a completed full refresh. + return existing if existing: existing.title = meta.title existing.sort_title = meta.sort_title or meta.title @@ -191,8 +220,8 @@ async def upsert_series_metadata( existing.comicvine_url = meta.comicvine_url existing.cover_url = meta.cover_url existing.publisher_id = publisher_id - existing.metadata_last_refreshed = datetime.now(UTC) - existing.metadata_source = "comicvine" + existing.metadata_last_refreshed = refreshed_at + existing.metadata_source = source series = existing log.debug("metadata_series_updated", series_id=series.id) else: @@ -208,8 +237,8 @@ async def upsert_series_metadata( comicvine_url=meta.comicvine_url, cover_url=meta.cover_url, publisher_id=publisher_id, - metadata_last_refreshed=datetime.now(UTC), - metadata_source="comicvine", + metadata_last_refreshed=refreshed_at, + metadata_source=source, ) session.add(series) await session.flush() @@ -227,21 +256,57 @@ async def upsert_series_metadata( async def get_series_metadata( self, comicvine_id: int, + *, + use_catalog: bool = True, ) -> SeriesMetadata: """Fetch provider series metadata without creating or updating local rows.""" log = logger.bind(comicvine_id=comicvine_id) log.debug("metadata_get_series_metadata") + if use_catalog and self._catalog is not None and self._catalog.available: + from pullbox.services.catalog.lookup import CatalogLookupService + + return await CatalogLookupService(self._catalog).get_series(str(comicvine_id)) + try: return await self._provider.get_series(str(comicvine_id)) except ComicVineError as exc: raise _provider_error_from_comicvine(exc) from exc + async def get_series_metadata_batch( + self, + comicvine_ids: list[int], + ) -> dict[int, SeriesMetadata]: + """Fetch multiple provider series profiles through the optional bulk contract.""" + if self._catalog is not None and self._catalog.available: + profiles: dict[int, SeriesMetadata] = {} + for key in dict.fromkeys(comicvine_ids): + profile = await self._catalog.series(key) + if profile is not None: + profiles[key] = profile + return profiles + batch_fetch = getattr(type(self._provider), "get_series_batch", None) + try: + if callable(batch_fetch): + result = await batch_fetch( + self._provider, + [str(comicvine_id) for comicvine_id in comicvine_ids], + ) + return {int(provider_id): metadata for provider_id, metadata in result.items()} + metadata = await asyncio.gather( + *(self.get_series_metadata(comicvine_id) for comicvine_id in comicvine_ids) + ) + return dict(zip(comicvine_ids, metadata, strict=True)) + except ComicVineError as exc: + raise _provider_error_from_comicvine(exc) from exc + async def get_cached_series_metadata( self, comicvine_id: int, ) -> SeriesMetadata | None: """Return fresh cached series metadata without starting a provider request.""" + if self._catalog is not None and self._catalog.available: + return await self._catalog.series(comicvine_id) cached_lookup = getattr(type(self._provider), "get_series_cached", None) if cached_lookup is None: return None @@ -251,16 +316,53 @@ async def get_cached_series_metadata( async def get_issue_summaries_for_series( self, comicvine_id: int, + *, + use_catalog: bool = True, ) -> list[IssueSummary]: """Fetch provider issue summaries for a series without touching local issues.""" log = logger.bind(comicvine_id=comicvine_id) log.debug("metadata_get_issue_summaries_for_series") + if use_catalog and self._catalog is not None and self._catalog.available: + from pullbox.services.catalog.lookup import CatalogLookupService + + return await CatalogLookupService(self._catalog).get_issues_for_series( + str(comicvine_id) + ) + try: return await self._provider.get_issues_for_series(str(comicvine_id)) except ComicVineError as exc: raise _provider_error_from_comicvine(exc) from exc + async def get_issue_catalog_batch( + self, + comicvine_ids: list[int], + ) -> dict[int, list[IssueSummary]]: + """Fetch multiple full issue catalogs through the optional bulk contract.""" + if self._catalog is not None and self._catalog.available: + return { + key: await self.get_issue_summaries_for_series(key) + for key in dict.fromkeys(comicvine_ids) + } + batch_fetch = getattr(type(self._provider), "get_issue_catalog_batch", None) + try: + if callable(batch_fetch): + result = await batch_fetch( + self._provider, + [str(comicvine_id) for comicvine_id in comicvine_ids], + ) + return {int(provider_id): summaries for provider_id, summaries in result.items()} + catalogs = await asyncio.gather( + *( + self.get_issue_summaries_for_series(comicvine_id) + for comicvine_id in comicvine_ids + ) + ) + return dict(zip(comicvine_ids, catalogs, strict=True)) + except ComicVineError as exc: + raise _provider_error_from_comicvine(exc) from exc + async def get_recent_issue_summaries_for_series( self, comicvine_id: int, @@ -469,6 +571,11 @@ async def fetch_issue( ).scalar_one_or_none() if existing: + existing.issue_number = meta.issue_number + existing.issue_number_text = _exact_issue_number_text( + meta.issue_number, + meta.issue_number_text, + ) existing.title = meta.title existing.description = meta.description existing.comicvine_url = meta.comicvine_url @@ -487,6 +594,23 @@ async def fetch_issue( return issue + async def prefetch_issue_metadata_batch( + self, + comicvine_ids: list[int], + ) -> dict[int, IssueMetadata]: + """Warm full issue metadata cache in bulk without holding a DB session.""" + batch_fetch = getattr(type(self._provider), "get_issue_batch", None) + if not callable(batch_fetch) or not comicvine_ids: + return {} + try: + result = await batch_fetch( + self._provider, + [str(comicvine_id) for comicvine_id in comicvine_ids], + ) + except ComicVineError as exc: + raise _provider_error_from_comicvine(exc) from exc + return {int(provider_id): metadata for provider_id, metadata in result.items()} + async def _sync_issue_creators( self, session: AsyncSession, @@ -560,7 +684,9 @@ async def fetch_issues_for_series( if not series or not series.comicvine_id: raise NotFoundError("Series", series_id) - summaries = await self.get_issue_summaries_for_series(series.comicvine_id) + summaries = await self.get_issue_summaries_for_series( + series.comicvine_id, use_catalog=False + ) return await self.upsert_issue_summaries( session, series, @@ -616,7 +742,12 @@ async def upsert_issue_summaries( select(Issue).where(Issue.comicvine_id.in_(provider_issue_ids)) ) existing_provider_issues = list(provider_result.scalars().all()) - existing_by_number = {issue.issue_number: issue for issue in existing_issues} + existing_by_text = {issue.effective_issue_number_text: issue for issue in existing_issues} + legacy_by_number = { + issue.issue_number: issue + for issue in existing_issues + if issue.issue_number_text is None + } existing_by_provider_id = { int(issue.comicvine_id): issue for issue in existing_provider_issues @@ -624,9 +755,17 @@ async def upsert_issue_summaries( } summary_evidence_types: list[IssueType] = [] for summary in summaries: + source = "pullbox_catalog" if isinstance(summary, CatalogIssueSummary) else "comicvine" provider_issue_id = int(summary.provider_id) + exact_issue_number_text = _exact_issue_number_text( + summary.issue_number, + summary.issue_number_text, + ) assign_provider_issue_id = True - existing = existing_by_number.get(summary.issue_number) + sync_issue_identity = True + existing = existing_by_text.get(exact_issue_number_text) + if existing is None: + existing = legacy_by_number.get(summary.issue_number) existing_by_provider = existing_by_provider_id.get(provider_issue_id) if existing_by_provider is not None and existing_by_provider.series_id != series_id: log.warning( @@ -638,13 +777,26 @@ async def upsert_issue_summaries( ) existing_by_provider = None assign_provider_issue_id = False + if ( + source == "pullbox_catalog" + and existing_by_provider is not None + and existing_by_provider.metadata_source == "comicvine" + ): + # A basic snapshot cannot establish that live fields are stale. + # Keep its identity and fields, and do not infer parent type here. + summary_evidence_types.append(IssueType.ISSUE) + continue if existing_by_provider is not None and existing_by_provider is not existing: if existing is None: old_issue_number = existing_by_provider.issue_number + old_issue_number_text = existing_by_provider.effective_issue_number_text existing = existing_by_provider existing.issue_number = summary.issue_number - existing_by_number.pop(old_issue_number, None) - existing_by_number[summary.issue_number] = existing + existing.issue_number_text = exact_issue_number_text + if legacy_by_number.get(old_issue_number) is existing: + legacy_by_number.pop(old_issue_number, None) + existing_by_text.pop(old_issue_number_text, None) + existing_by_text[exact_issue_number_text] = existing else: log.warning( "issue_summary_provider_id_collision", @@ -655,6 +807,7 @@ async def upsert_issue_summaries( target_issue_number=summary.issue_number, ) existing = existing_by_provider + sync_issue_identity = False # Compact provider type and provider title are explicit evidence. # Series inheritance is a fallback and cannot establish consensus. @@ -673,6 +826,16 @@ async def upsert_issue_summaries( ) if existing: + if sync_issue_identity: + old_issue_number = existing.issue_number + old_issue_number_text = existing.effective_issue_number_text + existing.issue_number = summary.issue_number + existing.issue_number_text = exact_issue_number_text + if legacy_by_number.get(old_issue_number) is existing: + legacy_by_number.pop(old_issue_number, None) + if existing_by_text.get(old_issue_number_text) is existing: + existing_by_text.pop(old_issue_number_text, None) + existing_by_text[exact_issue_number_text] = existing if assign_provider_issue_id: existing.comicvine_id = provider_issue_id if summary.title: @@ -688,21 +851,23 @@ async def upsert_issue_summaries( ) if not preserve_explicit_import_type: existing.issue_type = detected_type - existing.metadata_source = "comicvine" + if source != "pullbox_catalog" or existing.metadata_source != "comicvine": + existing.metadata_source = source else: issue = Issue( series_id=series_id, comicvine_id=provider_issue_id if assign_provider_issue_id else None, issue_number=summary.issue_number, + issue_number_text=exact_issue_number_text, title=summary.title, release_date=_parse_date(summary.release_date), cover_url=summary.cover_url, issue_type=detected_type, - metadata_source="comicvine", + metadata_source=source, ) session.add(issue) created.append(issue) - existing_by_number[summary.issue_number] = issue + existing_by_text[exact_issue_number_text] = issue if assign_provider_issue_id: existing_by_provider_id[provider_issue_id] = issue @@ -834,6 +999,17 @@ async def refresh_series( if not series.comicvine_id: raise ProviderError("comicvine", "Series has no ComicVine ID") + if force: + refresh_series = getattr(type(self._provider), "refresh_series", None) + refresh_catalog = getattr(type(self._provider), "refresh_issue_catalog", None) + try: + if callable(refresh_series): + await refresh_series(self._provider, str(series.comicvine_id)) + if callable(refresh_catalog): + await refresh_catalog(self._provider, str(series.comicvine_id)) + except ComicVineError as exc: + raise _provider_error_from_comicvine(exc) from exc + # noinspection PyTypeChecker series = await self.fetch_series(session, series.comicvine_id) await self.fetch_issues_for_series(session, series.id) diff --git a/src/pullbox/services/naming_settings.py b/src/pullbox/services/naming_settings.py new file mode 100644 index 00000000..889f6584 --- /dev/null +++ b/src/pullbox/services/naming_settings.py @@ -0,0 +1,151 @@ +"""Unified naming editor over the existing global and library policy stores.""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING + +from sqlalchemy import select + +from pullbox.core.exceptions import PullboxError, ValidationError +from pullbox.core.library_policy import load_library_naming_policy +from pullbox.core.naming import get_naming_preview +from pullbox.models.config import DEFAULT_SYSTEM_CONFIG, SystemConfig +from pullbox.models.library import LibraryRoot +from pullbox.schemas.config import ( + LibraryRootPolicyState, + NamingSettingsPreview, + NamingSettingsState, +) +from pullbox.schemas.import_job import FutureRootPolicyPayload +from pullbox.services.import_root_policy_activation import normalize_root_policy_definition +from pullbox.services.library_root_policy_service import ( + clear_library_root_policy, + get_library_root_policy_state, + update_library_root_policy, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.schemas.config import NamingSettingsUpdate + +_PREVIEW_TYPES = { + "series_path_template": "folder", + "comic_file_template": "standard", + "annual_file_template": "annual", + "non_standard_file_template": "non_standard_collection", + "single_non_standard_file_template": "non_standard_single", +} +_CONFIG_KEYS = { + **{field: field for field in _PREVIEW_TYPES}, + "series_path_template": "series_folder_template", + "replace_illegal_characters": "replace_illegal_characters", + "colon_replacement": "colon_replacement", +} + + +async def get_naming_settings( + session: AsyncSession, library_root_id: int | None = None +) -> NamingSettingsState: + """Read current effective values, including provenance in the stale-edit token.""" + revision = 0 + source = "global_default" + if library_root_id is None: + current = await load_library_naming_policy(session) + definition = {field: getattr(current, key) for field, key in _CONFIG_KEYS.items()} + definition["schema_version"] = 1 + identity: object = None + use_global = False + else: + state = LibraryRootPolicyState.model_validate( + await get_library_root_policy_state(session, library_root_id) + ) + definition = state.effective_policy.model_dump() + revision = state.revision + source = str(definition["source"]) + identity = (state.policy_id, revision, source, definition["source_import_job_id"]) + use_global = state.scope == "global_default" + policy = FutureRootPolicyPayload.model_validate(definition) + fingerprint = hashlib.sha256( + json.dumps([library_root_id, identity, policy.model_dump()], sort_keys=True).encode("utf-8") + ).hexdigest() + return NamingSettingsState( + library_root_id=library_root_id, + fingerprint=fingerprint, + policy=policy, + use_global=use_global, + revision=revision, + source=source, + ) + + +async def save_naming_settings( + session: AsyncSession, update: NamingSettingsUpdate +) -> NamingSettingsState: + """Save just naming; preserve import snapshots and existing revision guards.""" + if update.library_root_id is None and update.use_global: + raise ValidationError("Select a library before restoring global naming defaults.") + # Lock existing rows before comparing. Root helpers retain their revision check. + await session.scalars( + select(SystemConfig) + .where(SystemConfig.key.in_(_CONFIG_KEYS.values())) + .order_by(SystemConfig.key) + .with_for_update() + ) + if update.library_root_id is not None: + await session.scalar( + select(LibraryRoot).where(LibraryRoot.id == update.library_root_id).with_for_update() + ) + current = await get_naming_settings(session, update.library_root_id) + if current.fingerprint != update.expected_fingerprint: + raise PullboxError( + message=( + "Naming settings changed after you loaded them. Reload this scope and try again." + ), + code="NAMING_SETTINGS_CONFLICT", + status_code=409, + ) + if update.library_root_id is not None: + if update.use_global: + await clear_library_root_policy( + session, update.library_root_id, expected_revision=current.revision + ) + else: + await update_library_root_policy( + session, + update.library_root_id, + expected_revision=current.revision, + definition=update.policy.model_dump(), + ) + else: + definition = normalize_root_policy_definition(update.policy.model_dump()) + for field, key in _CONFIG_KEYS.items(): + value = definition[field] + text = str(value).lower() if isinstance(value, bool) else str(value) + row = await session.get(SystemConfig, key) + if row is None: + session.add( + SystemConfig(key=key, value=text, value_type=DEFAULT_SYSTEM_CONFIG[key][1]) + ) + else: + row.value = text + await session.flush() + return await get_naming_settings(session, update.library_root_id) + + +def preview_naming_settings(policy: FutureRootPolicyPayload) -> NamingSettingsPreview: + """Use identical validation and cleanup for both scopes, without any writes.""" + definition = normalize_root_policy_definition(policy.model_dump()) + return NamingSettingsPreview( + examples={ + field: get_naming_preview( + str(definition[field]), + template_type, + replace_illegal=policy.replace_illegal_characters, + colon_replacement=policy.colon_replacement, + ) + for field, template_type in _PREVIEW_TYPES.items() + } + ) diff --git a/src/pullbox/services/reader_content_service.py b/src/pullbox/services/reader_content_service.py index 13b1839a..e5a9ccd9 100644 --- a/src/pullbox/services/reader_content_service.py +++ b/src/pullbox/services/reader_content_service.py @@ -150,7 +150,7 @@ async def load_reader_source_record(session: AsyncSession, issue_id: int) -> Rea return ReaderSourceRecord( issue_id=issue.id, issue_title=issue.title, - issue_number=f"{issue.issue_number:g}", + issue_number=issue.effective_issue_number_text, issue_number_value=issue.issue_number, series_id=issue.series_id, series_title=issue.series.title, diff --git a/src/pullbox/services/reader_state_service.py b/src/pullbox/services/reader_state_service.py index 4d92a77e..0262c97b 100644 --- a/src/pullbox/services/reader_state_service.py +++ b/src/pullbox/services/reader_state_service.py @@ -17,6 +17,17 @@ from sqlalchemy.ext.asyncio import AsyncSession +def calculate_reader_position_percent( + last_page_index: int | None, + page_count: int | None, +) -> int: + """Return visible whole-number progress for a started reader session.""" + if last_page_index is None or page_count is None or last_page_index < 0 or page_count <= 0: + return 0 + percentage = ((last_page_index + 1) * 100) // page_count + return max(1, min(100, percentage)) + + class ReaderStateValidationError(Exception): """Raised when a progress write does not match the current content contract.""" @@ -90,9 +101,9 @@ def is_explicitly_unread(self) -> bool: @property def position_percent(self) -> int: - if not self.has_progress or self.last_page_index is None or self.page_count is None: + if not self.has_progress: return 0 - return max(0, min(100, ((self.last_page_index + 1) * 100) // self.page_count)) + return calculate_reader_position_percent(self.last_page_index, self.page_count) @property def is_continue_candidate(self) -> bool: diff --git a/src/pullbox/services/reading_query_service.py b/src/pullbox/services/reading_query_service.py index 7e958b99..1b4f01f0 100644 --- a/src/pullbox/services/reading_query_service.py +++ b/src/pullbox/services/reading_query_service.py @@ -12,7 +12,12 @@ from pullbox.models.library import FileFormat, LibraryFile from pullbox.models.reader import IssueReaderState from pullbox.models.series import Series -from pullbox.services.reader_state_service import ReaderStateSnapshot, snapshot_reader_state +from pullbox.models.story_arc import IssueStoryArc, StoryArcResolutionState +from pullbox.services.reader_state_service import ( + ReaderStateSnapshot, + calculate_reader_position_percent, + snapshot_reader_state, +) if TYPE_CHECKING: from datetime import datetime @@ -52,9 +57,7 @@ def is_explicitly_unread(self) -> bool: @property def position_percent(self) -> int: - if self.last_page_index is None or self.page_count is None: - return 0 - return max(0, min(100, ((self.last_page_index + 1) * 100) // self.page_count)) + return calculate_reader_position_percent(self.last_page_index, self.page_count) @property def is_continue_candidate(self) -> bool: @@ -118,6 +121,16 @@ def completion_percent(self) -> int: return (self.completed_count * 100) // self.readable_count +@dataclass(frozen=True, slots=True) +class StoryArcReadingAggregate: + """Readable and completed issue totals for one visible Story Arc.""" + + story_arc_id: int + readable_count: int + completed_count: int + in_progress_count: int + + @dataclass(frozen=True, slots=True) class AdjacentIssueReference: """Path-free identity for a readable issue beside the active issue.""" @@ -307,6 +320,72 @@ async def load_series_reading_aggregates( } +async def load_story_arc_reading_aggregates( + session: AsyncSession, + *, + user_id: int, + story_arc_ids: tuple[int, ...], +) -> dict[int, StoryArcReadingAggregate]: + """Load private reader totals for resolved members of bounded visible arcs.""" + if not story_arc_ids: + return {} + if len(story_arc_ids) > 100: + raise ValueError("Visible Story Arc aggregate queries are limited to 100 arcs.") + completed_issue = case((IssueReaderState.completed_at.is_not(None), Issue.id)) + in_progress_issue = case( + ( + and_( + IssueReaderState.last_page_index.is_not(None), + IssueReaderState.content_revision.is_not(None), + IssueReaderState.page_count.is_not(None), + IssueReaderState.last_page_index >= 0, + IssueReaderState.page_count > 0, + IssueReaderState.completed_at.is_(None), + IssueReaderState.last_page_index < IssueReaderState.page_count - 1, + ), + Issue.id, + ) + ) + result = await session.execute( + select( + IssueStoryArc.story_arc_id, + func.count(func.distinct(Issue.id)), + func.count(func.distinct(completed_issue)), + func.count(func.distinct(in_progress_issue)), + ) + .join(Issue, Issue.id == IssueStoryArc.issue_id) + .join( + LibraryFile, + and_( + LibraryFile.issue_id == Issue.id, + LibraryFile.file_format.in_(SUPPORTED_READER_FORMATS), + ), + ) + .outerjoin( + IssueReaderState, + and_( + IssueReaderState.issue_id == Issue.id, + IssueReaderState.user_id == user_id, + ), + ) + .where( + IssueStoryArc.story_arc_id.in_(story_arc_ids), + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + Issue.status == IssueStatus.OWNED, + ) + .group_by(IssueStoryArc.story_arc_id) + ) + return { + int(story_arc_id): StoryArcReadingAggregate( + story_arc_id=int(story_arc_id), + readable_count=int(readable_count), + completed_count=int(completed_count), + in_progress_count=int(in_progress_count), + ) + for story_arc_id, readable_count, completed_count, in_progress_count in result.all() + } + + async def load_adjacent_readable_issues( session: AsyncSession, *, diff --git a/src/pullbox/services/release_validator.py b/src/pullbox/services/release_validator.py index 3316cc88..1567d789 100644 --- a/src/pullbox/services/release_validator.py +++ b/src/pullbox/services/release_validator.py @@ -24,6 +24,7 @@ ) if TYPE_CHECKING: + from pullbox.core.release_year_matching import ReleaseYearContext from pullbox.providers.base import ReleaseResult from pullbox.services.search_types import SearchEvalKwargs, ValidatorKwargs @@ -74,6 +75,7 @@ class ValidationResult: year_match: bool | None = None issue_type_match: bool = False size_warning: str | None = None + year_match_basis: str | None = None # Confidence sort order for descending sort @@ -238,6 +240,7 @@ def validate_results( alternate_names: list[str] | None = None, wanted_issue_title: str | None = None, wanted_series_issue_count: int | None = None, + year_context: ReleaseYearContext | None = None, ) -> list[ValidationResult]: """Validate a list of search results against a wanted issue. @@ -265,6 +268,7 @@ def validate_results( alternate_names=alternate_names, wanted_issue_title=wanted_issue_title, wanted_series_issue_count=wanted_series_issue_count, + year_context=year_context, ) if vr.is_match: validated.append(vr) @@ -319,6 +323,7 @@ def validate_all_results( alternate_names: list[str] | None = None, wanted_issue_title: str | None = None, wanted_series_issue_count: int | None = None, + year_context: ReleaseYearContext | None = None, ) -> tuple[list[ValidationResult], list[ValidationResult]]: """Validate results, returning (matched, rejected) tuples. @@ -351,6 +356,7 @@ def validate_all_results( alternate_names=alternate_names, wanted_issue_title=wanted_issue_title, wanted_series_issue_count=wanted_series_issue_count, + year_context=year_context, ) if vr.is_match: matched.append(vr) @@ -379,9 +385,12 @@ def _validate_one( alternate_names: list[str] | None, wanted_issue_title: str | None = None, wanted_series_issue_count: int | None = None, + year_context: ReleaseYearContext | None = None, ) -> ValidationResult: """Run the validation pipeline on a single result.""" - metadata = self._extractor.from_release_title(result.title) + metadata = self._extractor.from_release_title( + result.title, expected_series=(wanted_series, *(alternate_names or [])) + ) parsed = metadata.parsed_release if parsed is None: return self._reject(result, "Failed to parse release title") @@ -471,6 +480,7 @@ def _validate_one( alternate_names=alternate_names, wanted_issue_title=wanted_issue_title, wanted_series_issue_count=wanted_series_issue_count, + year_context=year_context, ) if not decision.is_match: return self._reject( @@ -482,9 +492,7 @@ def _validate_one( issue_type_match=decision.match_method != "type_mismatch", ) - year_matched: bool | None = None - if parsed.year is not None and wanted_year is not None: - year_matched = abs(parsed.year - wanted_year) <= self._year_tolerance + year_matched = decision.match_diagnostics.get("year_match") # Step 8: File size heuristic (modifies confidence, never rejects) confidence, size_warning = _apply_size_heuristic( @@ -507,6 +515,7 @@ def _validate_one( year_match=year_matched, issue_type_match=True, size_warning=size_warning, + year_match_basis=decision.match_diagnostics.get("year_match_basis"), ) @staticmethod diff --git a/src/pullbox/services/search_acquisition_router.py b/src/pullbox/services/search_acquisition_router.py index 41fb6c34..55188280 100644 --- a/src/pullbox/services/search_acquisition_router.py +++ b/src/pullbox/services/search_acquisition_router.py @@ -6,11 +6,15 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Literal, Protocol +from sqlalchemy import select +from sqlalchemy.orm import selectinload + from pullbox.core.exceptions import ProviderError from pullbox.models.direct_acquisition import ( DirectAcquisitionAttempt, DirectAcquisitionState, DirectArtifactFailureClass, + DirectArtifactState, DirectProviderConfig, DirectProviderState, ) @@ -23,6 +27,7 @@ from pullbox.services.direct_acquisition_state import ( advance_acquisition_progress, transition_acquisition, + transition_artifact, ) from pullbox.services.direct_provider_quota import ( automatic_quota_available, @@ -39,6 +44,7 @@ from pullbox.providers.artifact_hosts.contract import HostResolutionRequest from pullbox.providers.base import ReleaseResult + from pullbox.services.airdcpp_search_types import DcValidatedCandidate from pullbox.services.direct_search_coordinator import ( DirectSearchDiscovery, DirectValidatedCandidate, @@ -76,6 +82,14 @@ async def create_direct_pending_match( result: DirectValidatedCandidate, ) -> object: ... + async def create_dc_pending_match( + self, + session: AsyncSession, + issue_id: int, + result: DcValidatedCandidate, + search_log_id: int, + ) -> object: ... + class DirectRunnerLike(Protocol): async def dispatch( @@ -88,6 +102,7 @@ async def dispatch( DirectPlanner = Callable[..., Awaitable[DirectAcquisitionPlanningResult]] +AcquisitionEligibilityCheck = Callable[[], Awaitable[bool]] @dataclass(frozen=True, slots=True) @@ -117,8 +132,11 @@ async def route_search_acquisition( runner: DirectRunnerLike | None, source_priority: list[str] | None = None, planner: DirectPlanner = plan_direct_acquisition, + eligibility_check: AcquisitionEligibilityCheck | None = None, ) -> SearchAcquisitionRoutingResult: - """Persist all discoveries and route the best result through its own adapter.""" + """Route a winner, optionally rechecking read-only scope eligibility at each handoff.""" + if stopped := await _stop_if_ineligible(session, eligibility_check): + return stopped target = outcome.target discoveries: tuple[DirectSearchDiscovery, ...] = () if outcome.direct_outcome is not None: @@ -139,6 +157,8 @@ async def route_search_acquisition( notices: list[str] = [] first_confidence = ranked[0].validation.confidence.value for selected in ranked: + if stopped := await _stop_if_ineligible(session, eligibility_check, discoveries): + return stopped confidence = selected.validation.confidence.value auto_grab = should_auto_grab( selected.validation.confidence, @@ -170,6 +190,8 @@ async def route_search_acquisition( release_title=selected.release.title, ) if not await intervention_service.has_pending_for_issue(session, target.issue_id): + if stopped := await _stop_if_ineligible(session, eligibility_check, discoveries): + return stopped await intervention_service.create_pending_match( session, target.issue_id, @@ -198,15 +220,65 @@ async def route_search_acquisition( if selected.source_kind == "dc": if selected.dc_result is None: raise RuntimeError("Selected DC result is unavailable.") - # R5 evaluates automatic DC participation and records the winner, - # while R6 owns durable provenance and queue mutation. + from pullbox.services.airdcpp_search_acquisition import ( + DcIssueAlreadyOwnedError, + acquire_dc_candidate, + ready_dc_client, + ) + + try: + await ready_dc_client(session, selected.dc_result, automatic=True) + if not auto_grab: + pending = None + if not await intervention_service.has_pending_for_issue( + session, target.issue_id + ): + pending = await intervention_service.create_dc_pending_match( + session, + target.issue_id, + selected.dc_result, + search_log_id, + ) + return SearchAcquisitionRoutingResult( + 0, + int(pending is not None), + "queued" if pending is not None else "pending_exists", + confidence, + "dc", + tuple(notices), + release_title=selected.release.title, + ) + route = selected.dc_result.route + download, created = await acquire_dc_candidate( + session, + candidate=selected.dc_result, + issue_id=target.issue_id, + search_log_id=search_log_id, + request_key=f"dc-auto:{search_log_id}:{target.issue_id}:{route.client_config_id}:{route.tth}", + automatic=True, + ) + except DcIssueAlreadyOwnedError: + return SearchAcquisitionRoutingResult(0, 0, "already_owned", confidence, "dc") + except ProviderError: + await session.commit() + notices.append(_indexer_failure_notice(selected.release.indexer_name)) + continue + # An ambiguous mutation is already owned by reconciliation: do not + # fall through and start a duplicate on another source. return SearchAcquisitionRoutingResult( + int(created), 0, - 0, - "dc_evaluation_only", + ( + "already_downloading" + if not created + else "retry_pending" + if download.state == DownloadState.RETRY_PENDING + else "downloading" + ), confidence, "dc", tuple(notices), + download_id=download.id, release_title=selected.release.title, ) @@ -238,6 +310,8 @@ async def route_search_acquisition( ) if provider is None: raise RuntimeError("Direct provider configuration was not found.") + if stopped := await _stop_if_ineligible(session, eligibility_check, discoveries): + return stopped reserve_notice = _automatic_reserve_notice(provider) if reserve_notice is not None: notices.append(reserve_notice) @@ -254,6 +328,8 @@ async def route_search_acquisition( try: planned = await planner(session, acquisition_id=discovery.attempt_id) except DirectAcquisitionPlanningError as exc: + if stopped := await _stop_if_ineligible(session, eligibility_check, discoveries): + return stopped if exc.intervention: await intervention_service.create_direct_pending_match( session, @@ -275,6 +351,8 @@ async def route_search_acquisition( await session.commit() continue await session.commit() + if stopped := await _stop_if_ineligible(session, eligibility_check, discoveries): + return stopped if runner is None: raise RuntimeError("Direct acquisition runner is not initialized.") await runner.dispatch( @@ -303,6 +381,62 @@ async def route_search_acquisition( ) +async def _stop_if_ineligible( + session: AsyncSession, + eligibility_check: AcquisitionEligibilityCheck | None, + discoveries: tuple[DirectSearchDiscovery, ...] = (), +) -> SearchAcquisitionRoutingResult | None: + """Re-query current scope without carrying its read snapshot across remote work.""" + if eligibility_check is None: + return None + # Resolution may have retained a read snapshot or produced durable planning + # state. Preserve that state before the caller checks current eligibility. + await session.commit() + eligible = await eligibility_check() + await session.commit() + if eligible is True: + return None + if discoveries: + attempts = await session.scalars( + select(DirectAcquisitionAttempt) + .where(DirectAcquisitionAttempt.id.in_(item.attempt_id for item in discoveries)) + .options(selectinload(DirectAcquisitionAttempt.artifact_attempts)) + .execution_options(populate_existing=True) + ) + for attempt in attempts: + _cancel_unsubmitted_attempt(attempt) + await session.commit() + return SearchAcquisitionRoutingResult(0, 0, "no_longer_eligible", None, None) + + +def _cancel_unsubmitted_attempt(attempt: DirectAcquisitionAttempt) -> None: + """Cancel only this router's unsubmitted plans, never rewrite terminal history.""" + if attempt.state not in { + DirectAcquisitionState.DISCOVERED, + DirectAcquisitionState.PLANNED, + DirectAcquisitionState.INTERVENTION, + }: + return + transition_acquisition(attempt, DirectAcquisitionState.CANCELLED) + attempt.failure_class = DirectArtifactFailureClass.USER_ACTION + attempt.failure_code = "no_longer_eligible" + attempt.error_message = "Search scope changed before acquisition could be submitted." + attempt.next_retry_at = None + for artifact in attempt.artifact_attempts: + if artifact.state not in {DirectArtifactState.PLANNED, DirectArtifactState.INTERVENTION}: + continue + transition_artifact(artifact, DirectArtifactState.CANCELLED) + artifact.failure_class = DirectArtifactFailureClass.USER_ACTION + artifact.failure_code = attempt.failure_code + artifact.error_message = attempt.error_message + artifact.next_retry_at = None + advance_acquisition_progress( + attempt, + revision=attempt.progress_revision + 1, + snapshot={"schema_version": 1, "stage": "cancelled", "failure_code": attempt.failure_code}, + ) + + def _indexer_failure_notice(indexer_name: str) -> str: """Describe an indexer queue failure without exposing provider details.""" return f"{indexer_name} could not be queued; continuing with other sources." diff --git a/src/pullbox/services/search_evaluation.py b/src/pullbox/services/search_evaluation.py index 1247bacb..a6adbb00 100644 --- a/src/pullbox/services/search_evaluation.py +++ b/src/pullbox/services/search_evaluation.py @@ -172,6 +172,9 @@ def _result_diagnostic( ), "rejection_reason": validation.rejection_reason, } + if validation.year_match_basis is not None: + diagnostic["year_match"] = validation.year_match + diagnostic["year_match_basis"] = validation.year_match_basis if query_provenance is not None: query = query_provenance.get(release_provenance_key(validation.release)) if query: diff --git a/src/pullbox/services/search_issue_runner.py b/src/pullbox/services/search_issue_runner.py index d8cecb89..00163b2a 100644 --- a/src/pullbox/services/search_issue_runner.py +++ b/src/pullbox/services/search_issue_runner.py @@ -150,6 +150,7 @@ async def search_issue_target( wanted_series=target.series_title, wanted_issue=target.issue_number, wanted_year=target.search_year, + year_context=target.year_context, wanted_issue_type=target.issue_type, alternate_names=target.alternate_names, wanted_issue_title=target.issue_title, diff --git a/src/pullbox/services/search_query_helpers.py b/src/pullbox/services/search_query_helpers.py index 685ee638..f0545cf0 100644 --- a/src/pullbox/services/search_query_helpers.py +++ b/src/pullbox/services/search_query_helpers.py @@ -5,6 +5,7 @@ import re from typing import TYPE_CHECKING +from pullbox.core.issue_numbers import format_issue_number from pullbox.core.issue_title import collection_title_fragment, collection_title_subtitle from pullbox.core.type_semantics import TypeFamily, issue_type_family from pullbox.models.issue import IssueType @@ -96,6 +97,7 @@ def _collection_query_strings(target: IssueSearchTarget) -> list[str]: target.series_title, target.issue_number, target.issue_type, + issue_number_text=target.effective_issue_number_text, ): _append_unique(queries, query) return queries @@ -136,12 +138,20 @@ def _sanitize_query(query: str) -> str: return re.sub(r"\s{2,}", " ", cleaned).strip() -def _standard_issue_query_variants(series_title: str, issue_number: float | None) -> list[str]: +def _standard_issue_query_variants( + series_title: str, + issue_number: float | None, + *, + issue_number_text: str | None = None, +) -> list[str]: """Build issue query variants for indexers with strict issue-number token matching.""" if issue_number is None: return [_sanitize_query(series_title)] + numeric_text = format_issue_number(issue_number) + if issue_number_text is not None and issue_number_text != numeric_text: + return [_sanitize_query(f"{series_title} {issue_number_text}")] if issue_number != int(issue_number): - return [_sanitize_query(f"{series_title} {issue_number}")] + return [_sanitize_query(f"{series_title} {issue_number_text or numeric_text}")] issue_int = int(issue_number) variants = [ @@ -163,21 +173,27 @@ def _build_type_queries( series_title: str, issue_number: float | None, issue_type: IssueType, + *, + issue_number_text: str | None = None, ) -> list[str]: """Build search query strings, appending type keywords for non-standard types.""" keywords = _TYPE_QUERY_KEYWORDS.get(issue_type.value, []) def _fmt_issue(num: float) -> str: - return str(int(num)) if num == int(num) else str(num) + return format_issue_number(num) if not keywords: - return _standard_issue_query_variants(series_title, issue_number) + return _standard_issue_query_variants( + series_title, + issue_number, + issue_number_text=issue_number_text, + ) queries: list[str] = [] for kw in keywords: query = f"{series_title} {kw}" if issue_number is not None: - query += f" {_fmt_issue(issue_number)}" + query += f" {issue_number_text or _fmt_issue(issue_number)}" queries.append(_sanitize_query(query)) return queries @@ -191,7 +207,10 @@ def build_issue_queries( """Build search queries for one issue target and mode.""" issue_type = target.issue_type if mode == "fast": - if issue_type == IssueType.ISSUE and not force_generic: + exact_differs_from_numeric = target.effective_issue_number_text != format_issue_number( + target.issue_number + ) + if issue_type == IssueType.ISSUE and (not force_generic or exact_differs_from_numeric): return [ SearchQuery( series_title=query_string, @@ -202,6 +221,7 @@ def build_issue_queries( for query_string in _standard_issue_query_variants( target.series_title, target.issue_number, + issue_number_text=target.effective_issue_number_text, ) ] if issue_type_family(issue_type) is TypeFamily.COLLECTION and not force_generic: @@ -228,6 +248,7 @@ def build_issue_queries( target.series_title, target.issue_number, IssueType.ISSUE, + issue_number_text=target.effective_issue_number_text, ) else: query_strings = _collection_query_strings(target) @@ -236,6 +257,7 @@ def build_issue_queries( target.series_title, target.issue_number, issue_type, + issue_number_text=target.effective_issue_number_text, ) if not force_generic and issue_type in _COLLECTION_TYPES: @@ -276,6 +298,9 @@ def build_auto_fallback_queries(target: IssueSearchTarget) -> list[SearchQuery]: target.series_title, fallback_issue, IssueType.ISSUE, + issue_number_text=( + target.effective_issue_number_text if fallback_issue is not None else None + ), ) return [ SearchQuery( @@ -287,6 +312,18 @@ def build_auto_fallback_queries(target: IssueSearchTarget) -> list[SearchQuery]: for query_string in generic_queries ] + if target.effective_issue_number_text != format_issue_number(target.issue_number): + return [ + SearchQuery( + series_title=_sanitize_query( + f"{target.series_title} {target.effective_issue_number_text}" + ), + issue_number=None, + year=target.search_year, + issue_type=target.issue_type.value, + ) + ] + return [ SearchQuery( series_title=_sanitize_query(target.series_title), diff --git a/src/pullbox/services/search_service.py b/src/pullbox/services/search_service.py index 63af7f1c..30f66a05 100644 --- a/src/pullbox/services/search_service.py +++ b/src/pullbox/services/search_service.py @@ -10,6 +10,7 @@ import structlog +from pullbox.core.issue_numbers import format_issue_number from pullbox.models.issue import IssueType from pullbox.providers import base as _provider_base from pullbox.services import search_evaluation as _search_evaluation @@ -196,7 +197,7 @@ def _format_query_label(query: SearchQuery) -> str: if query.issue_number is None: return query.series_title - issue = f"{query.issue_number:g}" + issue = format_issue_number(query.issue_number) return f"{query.series_title} {issue}" diff --git a/src/pullbox/services/search_targets.py b/src/pullbox/services/search_targets.py index b8d8b532..58c5f97d 100644 --- a/src/pullbox/services/search_targets.py +++ b/src/pullbox/services/search_targets.py @@ -5,14 +5,25 @@ import asyncio from collections.abc import Awaitable, Callable from dataclasses import dataclass +from datetime import date from typing import TYPE_CHECKING, Any, Protocol -from sqlalchemy import and_, exists, or_, select +from sqlalchemy import and_, exists, func, or_, select +from pullbox.core.issue_numbers import format_issue_number +from pullbox.core.release_year_matching import ReleaseYearContext from pullbox.core.type_semantics import TypeFamily, issue_type_family +from pullbox.models.download import DownloadHistory, DownloadState from pullbox.models.issue import Issue, IssueStatus, IssueType +from pullbox.models.library import LibraryFile from pullbox.models.pending_match import PendingMatch, PendingMatchStatus from pullbox.models.series import Series +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcResolutionState, +) if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -34,11 +45,25 @@ class IssueSearchTarget: series_title: str issue_number: float issue_type: IssueType + issue_number_text: str | None = None issue_title: str | None = None series_year: int | None = None release_year: int | None = None alternate_names: list[str] | None = None series_issue_count: int | None = None + store_year: int | None = None + series_continuing: bool = False + + @property + def year_context(self) -> ReleaseYearContext: + """Use existing catalog dates for confidence, not extra provider lookups.""" + return ReleaseYearContext( + series_year=self.series_year, + publication_years=tuple( + sorted({year for year in (self.release_year, self.store_year) if year is not None}) + ), + series_continuing=self.series_continuing, + ) @property def search_year(self) -> int | None: @@ -47,6 +72,11 @@ def search_year(self) -> int | None: return self.release_year or self.series_year return self.series_year + @property + def effective_issue_number_text(self) -> str: + """Return the canonical search identity with legacy numeric fallback.""" + return self.issue_number_text or format_issue_number(self.issue_number) + @dataclass(frozen=True) class IssueSearchOutcome: @@ -67,10 +97,105 @@ class IssueSearchOutcome: direct_outcome: DirectSearchOutcome | None = None dc_outcome: DcSearchOutcome | None = None + @property + def results_found_count(self) -> int: + """Count discovered candidates, independently of acquisition success.""" + return len(self.raw_results) + sum( + len(outcome.matched) + len(outcome.rejected) + for outcome in (self.direct_outcome, self.dc_outcome) + if outcome is not None + ) + + @property + def results_rejected_count(self) -> int: + """Only validation rejections count, not unused or unavailable matches.""" + return len(self.rejected) + sum( + len(outcome.rejected) + for outcome in (self.direct_outcome, self.dc_outcome) + if outcome is not None + ) + SearchOutcomeCallback = Callable[[IssueSearchOutcome], Awaitable[None]] +def arc_issue_release_filter(*, today: date | None = None) -> Any: + """Track upcoming members now, but search on/after their publication date. + + Store date takes precedence over Comic Vine's later cover date. Undated + issues remain eligible; a legacy upcoming flag never bypasses this rule. + """ + publication_date = func.coalesce(Issue.store_date, Issue.release_date) + return or_(publication_date.is_(None), publication_date <= (today or date.today())) + + +def wanted_issue_eligibility_filter(*, today: date | None = None) -> Any: + """Return the shared series-or-story-arc wanted-search eligibility filter. + + Story arcs only widen the existing target gate. A resolved arc member may + retain the canonical series-owned SKIPPED state, but it still enters the + normal wanted-search runner, so provider cooldowns, result selection, + pending intervention suppression, and duplicate acquisition remain shared. + """ + monitored_story_arc = exists().where( + and_( + IssueStoryArc.issue_id == Issue.id, + IssueStoryArc.story_arc_id == StoryArc.id, + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + StoryArc.lifecycle == StoryArcLifecycle.ACTIVE, + StoryArc.monitored.is_(True), + ) + ) + return and_( + Issue.manual_skip.is_(False), + or_( + and_( + Issue.status == IssueStatus.WANTED, + Series.monitored.is_(True), + ), + and_( + arc_issue_acquisition_filter(today=today), + monitored_story_arc, + ), + ), + ) + + +def arc_issue_acquisition_filter(*, today: date | None = None) -> Any: + """One missing-issue contract for manual and scheduled arc searches.""" + active_download = exists().where( + DownloadHistory.issue_id == Issue.id, + or_( + DownloadHistory.state.in_( + ( + DownloadState.QUEUED, + DownloadState.SENT, + DownloadState.DOWNLOADING, + DownloadState.FINALIZING, + DownloadState.PAUSED, + DownloadState.RETRY_PENDING, + DownloadState.POST_PROCESSING, + ) + ), + and_( + DownloadHistory.state == DownloadState.COMPLETED, + DownloadHistory.imported_at.is_(None), + ), + ), + ) + return and_( + arc_issue_release_filter(today=today), + Issue.status.in_((IssueStatus.WANTED, IssueStatus.SKIPPED)), + Issue.manual_skip.is_(False), + ~exists().where(LibraryFile.issue_id == Issue.id), + ~active_download, + ~exists().where( + PendingMatch.issue_id == Issue.id, + PendingMatch.status == PendingMatchStatus.PENDING, + ), + ) + + class SearchIssueTargetFunc(Protocol): """Callable shape for searching a single issue target.""" @@ -94,15 +219,22 @@ def _target_from_row(row: Any) -> IssueSearchTarget: """Build a search target from a SQLAlchemy row with the expected labels.""" release_date = getattr(row, "release_date", None) or getattr(row, "store_date", None) series_issue_count = getattr(row, "series_issue_count", None) + store_date = getattr(row, "store_date", None) + lifecycle = getattr(row, "status_override", None) or getattr(row, "series_status", None) return IssueSearchTarget( issue_id=int(row.issue_id), series_id=int(row.series_id), series_title=str(row.series_title), issue_number=float(row.issue_number), issue_type=IssueType(str(row.issue_type)) if row.issue_type else IssueType.ISSUE, + issue_number_text=( + str(row.issue_number_text) if getattr(row, "issue_number_text", None) else None + ), issue_title=str(row.issue_title) if row.issue_title else None, series_year=int(row.series_year) if row.series_year else None, release_year=release_date.year if release_date is not None else None, + store_year=store_date.year if store_date is not None else None, + series_continuing=str(lifecycle).casefold() == "continuing", alternate_names=list(row.alternate_names) if row.alternate_names else None, series_issue_count=(int(series_issue_count) if series_issue_count is not None else None), ) @@ -118,6 +250,7 @@ async def load_issue_search_target( Issue.id.label("issue_id"), Issue.series_id.label("series_id"), Issue.issue_number.label("issue_number"), + Issue.issue_number_text.label("issue_number_text"), Issue.issue_type.label("issue_type"), Issue.title.label("issue_title"), Issue.release_date.label("release_date"), @@ -126,6 +259,8 @@ async def load_issue_search_target( Series.year_start.label("series_year"), Series.alternate_names.label("alternate_names"), Series.issue_count.label("series_issue_count"), + Series.status.label("series_status"), + Series.status_override.label("status_override"), ) .join(Series, Series.id == Issue.series_id) .where(Issue.id == issue_id) @@ -147,6 +282,7 @@ async def load_series_wanted_search_targets( Issue.id.label("issue_id"), Issue.series_id.label("series_id"), Issue.issue_number.label("issue_number"), + Issue.issue_number_text.label("issue_number_text"), Issue.issue_type.label("issue_type"), Issue.title.label("issue_title"), Issue.release_date.label("release_date"), @@ -155,11 +291,13 @@ async def load_series_wanted_search_targets( Series.year_start.label("series_year"), Series.alternate_names.label("alternate_names"), Series.issue_count.label("series_issue_count"), + Series.status.label("series_status"), + Series.status_override.label("status_override"), ) .join(Series, Series.id == Issue.series_id) .where(Issue.series_id == series_id) .where(Issue.status == IssueStatus.WANTED) - .order_by(Issue.issue_number) + .order_by(Issue.issue_number, Issue.issue_number_text, Issue.id) ) return [_target_from_row(row) for row in result.all()] @@ -217,8 +355,7 @@ async def load_wanted_issue_search_targets( ) -> list[IssueSearchTarget]: """Load wanted issue targets for the global sweep.""" filters = [ - Issue.status == IssueStatus.WANTED, - Series.monitored.is_(True), + wanted_issue_eligibility_filter(), ~exists().where( and_( PendingMatch.issue_id == Issue.id, @@ -245,6 +382,7 @@ async def load_wanted_issue_search_targets( Issue.id.label("issue_id"), Issue.series_id.label("series_id"), Issue.issue_number.label("issue_number"), + Issue.issue_number_text.label("issue_number_text"), Issue.issue_type.label("issue_type"), Issue.title.label("issue_title"), Issue.release_date.label("release_date"), @@ -253,6 +391,8 @@ async def load_wanted_issue_search_targets( Series.year_start.label("series_year"), Series.alternate_names.label("alternate_names"), Series.issue_count.label("series_issue_count"), + Series.status.label("series_status"), + Series.status_override.label("status_override"), ) .join(Series, Series.id == Issue.series_id) .where(*filters) @@ -274,6 +414,7 @@ async def load_wanted_issue_search_targets_by_ids( Issue.id.label("issue_id"), Issue.series_id.label("series_id"), Issue.issue_number.label("issue_number"), + Issue.issue_number_text.label("issue_number_text"), Issue.issue_type.label("issue_type"), Issue.title.label("issue_title"), Issue.release_date.label("release_date"), @@ -282,12 +423,13 @@ async def load_wanted_issue_search_targets_by_ids( Series.year_start.label("series_year"), Series.alternate_names.label("alternate_names"), Series.issue_count.label("series_issue_count"), + Series.status.label("series_status"), + Series.status_override.label("status_override"), ) .join(Series, Series.id == Issue.series_id) .where( Issue.id.in_(issue_ids), - Issue.status == IssueStatus.WANTED, - Series.monitored.is_(True), + wanted_issue_eligibility_filter(), ~exists().where( and_( PendingMatch.issue_id == Issue.id, diff --git a/src/pullbox/services/semantic_matching.py b/src/pullbox/services/semantic_matching.py index 87e851fd..1b994476 100644 --- a/src/pullbox/services/semantic_matching.py +++ b/src/pullbox/services/semantic_matching.py @@ -10,6 +10,7 @@ from pullbox.core.name_matcher import NameMatcher, NameMatchResult from pullbox.core.naming import extract_base_series_title from pullbox.core.release_parser import issues_match, normalize_issue_number +from pullbox.core.release_year_matching import ReleaseYearContext, match_release_year from pullbox.core.source_metadata import MetadataSignal from pullbox.core.type_semantics import ( TypeFamily, @@ -156,6 +157,7 @@ def match_against_issue( wanted_issue_cv_id: int | None = None, wanted_issue_title: str | None = None, wanted_series_issue_count: int | None = None, + year_context: ReleaseYearContext | None = None, ) -> IssueMatchDecision: """Return a workflow-aware semantic match decision for one issue target.""" series_name = metadata.series_name or "" @@ -409,15 +411,22 @@ def match_against_issue( match_diagnostics={"type_mode": compatibility.mode}, ) - year_match: bool | None = None - if metadata.year is not None and wanted_year is not None: - year_match = abs(metadata.year - wanted_year) <= self._config.year_tolerance + year_evidence = match_release_year( + metadata.year, + wanted_year=wanted_year, + volume_year=metadata.parsed_release.volume_year if metadata.parsed_release else None, + context=year_context, + issue_type=wanted_issue_type, + tolerance=self._config.year_tolerance, + ) confidence = self._compute_confidence( match_type=match_result.match_type, similarity=match_result.similarity, - year_match=year_match, + year_match=year_evidence.matches, ) + if year_evidence.weak and confidence is MatchConfidence.HIGH: + confidence = MatchConfidence.MEDIUM if compatibility.lowers_confidence: confidence = _lower_confidence(confidence) if issue_check_skipped and wanted_issue_type not in {IssueType.ISSUE, IssueType.ANNUAL}: @@ -434,6 +443,8 @@ def match_against_issue( "series_similarity": round(match_result.similarity, 4), "match_type": match_result.match_type, "single_word_collection_prefix": single_word_collection_prefix, + "year_match": year_evidence.matches, + "year_match_basis": year_evidence.basis, }, ) diff --git a/src/pullbox/services/series_delete_targets.py b/src/pullbox/services/series_delete_targets.py index 1aedc722..1a9f4d8a 100644 --- a/src/pullbox/services/series_delete_targets.py +++ b/src/pullbox/services/series_delete_targets.py @@ -12,7 +12,7 @@ from sqlalchemy import select from pullbox.models.issue import Issue -from pullbox.models.library import LibraryFile, LibraryRoot +from pullbox.models.library import LibraryFile, LibraryFileStorageMode, LibraryRoot from pullbox.models.series import Series if TYPE_CHECKING: @@ -32,6 +32,8 @@ class SeriesDeleteTarget: folder_paths: tuple[Path, ...] linked_file_count: int + managed_file_count: int + referenced_file_count: int @dataclass(frozen=True, slots=True) @@ -40,6 +42,8 @@ class SeriesDeleteContext: series_count: int linked_file_count: int + managed_file_count: int + referenced_file_count: int def is_relative_to(path: Path, other: Path) -> bool: @@ -224,6 +228,34 @@ async def count_linked_existing_files( return count +async def count_linked_existing_files_by_storage( + session: AsyncSession, + *, + series_id: int, + folder_paths: tuple[Path, ...], +) -> tuple[int, int]: + """Count existing managed and referenced files included in a delete target.""" + issue_ids_subq = select(Issue.id).where(Issue.series_id == series_id) + result = await session.execute( + select(LibraryFile.file_path, LibraryFile.storage_mode).where( + LibraryFile.issue_id.in_(issue_ids_subq) + ) + ) + managed_count = 0 + referenced_count = 0 + for file_path_value, storage_mode in result.all(): + file_path = Path(file_path_value).expanduser() + if not file_path.is_file(): + continue + if folder_paths and not any(is_relative_to(file_path, folder) for folder in folder_paths): + continue + if storage_mode == LibraryFileStorageMode.REFERENCED: + referenced_count += 1 + else: + managed_count += 1 + return managed_count, referenced_count + + async def build_series_delete_target( session: AsyncSession, series: Series, @@ -252,9 +284,16 @@ async def build_series_delete_target( series_id=series.id, folder_paths=folder_paths, ) + managed_file_count, referenced_file_count = await count_linked_existing_files_by_storage( + session, + series_id=series.id, + folder_paths=folder_paths, + ) return SeriesDeleteTarget( folder_paths=folder_paths, linked_file_count=linked_file_count, + managed_file_count=managed_file_count, + referenced_file_count=referenced_file_count, ) @@ -266,6 +305,8 @@ async def build_series_delete_context( ) -> SeriesDeleteContext: """Build delete-modal UI state for one or more series.""" linked_file_count = 0 + managed_file_count = 0 + referenced_file_count = 0 loaded_count = 0 for series_id in dict.fromkeys(series_ids): @@ -274,9 +315,13 @@ async def build_series_delete_context( continue target = await target_builder(session, series) linked_file_count += target.linked_file_count + managed_file_count += target.managed_file_count + referenced_file_count += target.referenced_file_count loaded_count += 1 return SeriesDeleteContext( series_count=loaded_count, linked_file_count=linked_file_count, + managed_file_count=managed_file_count, + referenced_file_count=referenced_file_count, ) diff --git a/src/pullbox/services/series_service.py b/src/pullbox/services/series_service.py index b2eea336..2b279538 100644 --- a/src/pullbox/services/series_service.py +++ b/src/pullbox/services/series_service.py @@ -4,6 +4,7 @@ import asyncio import shutil +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path from typing import TYPE_CHECKING @@ -13,12 +14,20 @@ from pullbox.core.events import EventBus, IssueWanted, SeriesAdded from pullbox.core.exceptions import NotFoundError, ValidationError -from pullbox.core.library_policy import load_library_naming_policy -from pullbox.core.naming import format_series_folder +from pullbox.core.library_file_ownership import ( + referenced_library_files_for_target, + require_mutable_library_target, +) +from pullbox.core.library_naming import build_series_relative_path +from pullbox.core.library_policy import ( + load_effective_library_ingest_policy, + load_library_naming_policy, +) +from pullbox.core.library_root_resolution import preferred_managed_root_id from pullbox.models.download import DownloadClientType, DownloadHistory, DownloadState from pullbox.models.import_job import ImportJob, ImportSourceType from pullbox.models.issue import Issue, IssueStatus -from pullbox.models.library import LibraryFile, LibraryRoot +from pullbox.models.library import LibraryFile, LibraryFileStorageMode, LibraryRoot from pullbox.models.series import ( IssueCatalogState, Series, @@ -32,6 +41,7 @@ find_imported_series_cover, purge_series_cover_cache, ) +from pullbox.services.library_root_management import validate_managed_library_root from pullbox.services.series_delete_targets import ( SeriesDeleteContext as SeriesDeleteContext, ) @@ -58,6 +68,64 @@ logger = structlog.get_logger(__name__) +@dataclass(frozen=True, slots=True) +class SeriesFolderCreationResult: + """Exact directory ownership captured while assigning a Series path.""" + + folder_path: Path + ownership_boundary_path: Path + created_directory_paths: tuple[Path, ...] + + def ownership_payload(self) -> dict[str, object]: + return { + "schema_version": 1, + "folder_path": str(self.folder_path), + "ownership_boundary_path": str(self.ownership_boundary_path), + "created_directory_paths": [str(path) for path in self.created_directory_paths], + } + + +def _create_series_folder_directories( + folder_path: Path, + ownership_boundary_path: Path, +) -> tuple[Path, ...]: + """Create and return only strict descendant directory segments created here.""" + relative = folder_path.relative_to(ownership_boundary_path) + if not relative.parts: + raise ValueError("Series folder cannot be the library root") + boundary_resolved = ownership_boundary_path.resolve(strict=True) + if not boundary_resolved.is_dir(): + raise NotADirectoryError(ownership_boundary_path) + + created: list[Path] = [] + current = ownership_boundary_path + try: + for segment in relative.parts: + current /= segment + current_created = False + try: + current.mkdir() + except FileExistsError: + if not current.is_dir(): + raise + else: + current_created = True + resolved_current = current.resolve(strict=True) + resolved_relative = resolved_current.relative_to(boundary_resolved) + if not resolved_relative.parts: + raise ValueError("Series folder segment cannot resolve to the library root") + if current_created: + created.append(resolved_current) + except (OSError, ValueError): + for directory in reversed(created): + try: + directory.rmdir() + except OSError: + continue + raise + return tuple(created) + + def _targeted_import_folder_type_hint( series: Series, issue_summaries: list[IssueSummary], @@ -207,16 +275,21 @@ async def add_from_import_review_targeted( series.issue_catalog_error = None series.issue_catalog_last_synced_at = None series.issue_catalog_last_checked_at = None - series.metadata_source = "comicvine_partial" + if series.metadata_source != "pullbox_catalog": + series.metadata_source = "comicvine_partial" + diagnostics = dict(import_series.diagnostics or {}) + diagnostics.pop("series_folder_ownership", None) if library_root_id is not None and not series.path: - await self._create_series_folder( + folder_creation = await self._create_series_folder( session, series, library_root_id, cv_id, folder_series_type=_targeted_import_folder_type_hint(series, issue_summaries), ) + diagnostics["series_folder_ownership"] = folder_creation.ownership_payload() + import_series.diagnostics = diagnostics local_cover = ( find_imported_series_cover(Path(import_series.source_folder)) @@ -228,7 +301,25 @@ async def add_from_import_review_targeted( select(ImportJob.source_type).where(ImportJob.id == import_series.import_job_id) ) if source_type == ImportSourceType.MYLAR3: - await cache_imported_series_cover(session, series, local_cover) + diagnostics = dict(import_series.diagnostics or {}) + diagnostics.pop("cover_cache_ownership", None) + cached_cover = await cache_imported_series_cover(session, series, local_cover) + if ( + cached_cover is not None + and cached_cover.artifact_created + and cached_cover.artifact_signature is not None + ): + diagnostics["cover_cache_ownership"] = { + "schema_version": 1, + "base_path": str(cached_cover.covers_base), + "ownership_boundary_path": str(cached_cover.ownership_boundary_path), + "created_directory_paths": [ + str(path) for path in cached_cover.created_directory_paths + ], + "artifact_path": str(cached_cover.path), + "artifact_signature": dict(cached_cover.artifact_signature), + } + import_series.diagnostics = diagnostics if issue_summaries: await self._metadata.upsert_issue_summaries(session, series, issue_summaries) @@ -259,7 +350,7 @@ async def hydrate_series_catalog( hydrated = await self.add_from_comicvine_prefetched( session, comicvine_id=int(series.comicvine_id), - library_root_id=series.library_root_id, + library_root_id=preferred_managed_root_id(series), search_on_add=series.monitored if search_on_add is None else search_on_add, series_meta=series_meta, issue_summaries=issue_summaries, @@ -286,6 +377,31 @@ async def prefetch_comicvine_bundle( ) return series_meta, issue_summaries + async def prefetch_comicvine_profiles( + self, + comicvine_ids: list[int], + ) -> dict[int, SeriesMetadata]: + """Fetch provider profiles in bulk before the slower issue-catalog pass.""" + return await self._metadata.get_series_metadata_batch(comicvine_ids) + + async def prefetch_comicvine_issue_catalogs( + self, + comicvine_ids: list[int], + ) -> dict[int, list[IssueSummary]]: + """Fetch and group issue catalogs for multiple imported series.""" + return await self._metadata.get_issue_catalog_batch(comicvine_ids) + + async def upsert_comicvine_profile( + self, + session: AsyncSession, + comicvine_id: int, + series_meta: SeriesMetadata, + ) -> Series: + """Persist visible profile fields without claiming catalog completion.""" + series = await self._metadata.upsert_series_metadata(session, comicvine_id, series_meta) + await session.flush() + return series + async def add_from_comicvine_prefetched( self, session: AsyncSession, @@ -356,11 +472,19 @@ async def add_from_comicvine_prefetched( # ── Folder creation ─────────────────────────────────────────────── @staticmethod - async def _load_naming_config(session: AsyncSession) -> dict[str, str]: + async def _load_naming_config( + session: AsyncSession, + root: LibraryRoot | int | None = None, + ) -> dict[str, str]: """Load naming-related config keys from the database.""" - policy = await load_library_naming_policy(session) + policy = ( + await load_effective_library_ingest_policy(session, root) + if root is not None + else await load_library_naming_policy(session) + ) return { "series_folder_template": policy.series_folder_template, + "series_path_template": policy.series_path_template, "replace_illegal_characters": "true" if policy.replace_illegal_characters else "false", "colon_replacement": policy.colon_replacement, } @@ -373,47 +497,34 @@ async def _create_series_folder( comicvine_id: int, *, folder_series_type: SeriesType | None = None, - ) -> None: + ) -> SeriesFolderCreationResult: """Create a series folder on disk inside the given library root. - Sets ``series.path`` and ``series.library_root_id``. If a folder - with the same name already exists, appends ``[cv-{comicvine_id}]`` - to disambiguate. + Sets the current and preferred series library roots. If a folder with + the same name already exists, appends ``[cv-{comicvine_id}]`` to + disambiguate. """ root = await session.get(LibraryRoot, library_root_id) if root is None: raise ValidationError(f"Library root {library_root_id} not found") - if not root.enabled: - raise ValidationError(f"Library root '{root.name}' is disabled") + await validate_managed_library_root(root) # Load naming config - cfg = await self._load_naming_config(session) - template = cfg.get("series_folder_template", "{Series} ({Year})") - replace_illegal = cfg.get("replace_illegal_characters", "true") == "true" - colon_replacement = cfg.get("colon_replacement", "dash") + cfg = await self._load_naming_config(session, root) # Resolve publisher name for the template - publisher_name: str | None = None if series.publisher_id: await session.refresh(series, attribute_names=["publisher"]) - publisher_name = series.publisher.name if series.publisher else None - # Format folder name - folder_name = format_series_folder( - title=series.title, - year=series.year_start, - publisher=publisher_name, - comicvine_id=comicvine_id, - series_type=(folder_series_type or series.series_type).value - if (folder_series_type or series.series_type) - else None, - template=template, - replace_illegal=replace_illegal, - colon_replacement=colon_replacement, + folder_relative = build_series_relative_path( + series, + cfg, + series_type_override=folder_series_type.value if folder_series_type else None, + comicvine_id_override=comicvine_id, ) root_path = Path(root.path) - folder_path = root_path / folder_name + folder_path = root_path / folder_relative # Reclaim empty/progress-only collision folders left behind by prior interrupted work. if folder_path.exists(): @@ -427,20 +538,29 @@ async def _create_series_folder( # Handle collision: append [cv-{id}] if folder already exists if folder_path.exists(): - folder_name_cv = f"{folder_name} [cv-{comicvine_id}]" - folder_path = root_path / folder_name_cv + folder_name_cv = f"{folder_path.name} [cv-{comicvine_id}]" + folder_path = folder_path.with_name(folder_name_cv) logger.info( "series_folder_collision", - original=folder_name, + original=str(folder_relative), resolved=folder_name_cv, comicvine_id=comicvine_id, ) - # Create the directory (run in thread to avoid blocking event loop) - await asyncio.to_thread(folder_path.mkdir, parents=False, exist_ok=True) + # Create each missing segment independently so import rollback can + # distinguish directories created here from pre-existing user paths. + try: + created_directory_paths = await asyncio.to_thread( + _create_series_folder_directories, + folder_path, + root_path, + ) + except (OSError, ValueError) as exc: + raise ValidationError("Series folder must be created inside its library root") from exc series.path = str(folder_path) series.library_root_id = library_root_id + series.preferred_library_root_id = library_root_id logger.info( "series_folder_created", @@ -448,6 +568,11 @@ async def _create_series_folder( series_title=series.title, library_root=root.name, ) + return SeriesFolderCreationResult( + folder_path=folder_path.resolve(strict=True), + ownership_boundary_path=root_path.resolve(strict=True), + created_directory_paths=created_directory_paths, + ) @staticmethod async def _build_series_folder_name( @@ -458,26 +583,15 @@ async def _build_series_folder_name( if series.library_root_id is None: return None - cfg = await SeriesService._load_naming_config(session) - template = cfg.get("series_folder_template", "{Series} ({Year})") - replace_illegal = cfg.get("replace_illegal_characters", "true") == "true" - colon_replacement = cfg.get("colon_replacement", "dash") + root = await session.get(LibraryRoot, series.library_root_id) + if root is None: + return None + cfg = await SeriesService._load_naming_config(session, root) - publisher_name: str | None = None if series.publisher_id: await session.refresh(series, attribute_names=["publisher"]) - publisher_name = series.publisher.name if series.publisher else None - return format_series_folder( - title=series.title, - year=series.year_start, - publisher=publisher_name, - comicvine_id=series.comicvine_id, - series_type=series.series_type.value if series.series_type else None, - template=template, - replace_illegal=replace_illegal, - colon_replacement=colon_replacement, - ) + return str(build_series_relative_path(series, cfg)) # ── Folder renaming ───────────────────────────────────────────── @@ -501,39 +615,31 @@ async def rename_series_folder( root = await session.get(LibraryRoot, series.library_root_id) if not root: return None + await validate_managed_library_root(root) # Load naming config - cfg = await self._load_naming_config(session) - template = cfg.get("series_folder_template", "{Series} ({Year})") - replace_illegal = cfg.get("replace_illegal_characters", "true") == "true" - colon_replacement = cfg.get("colon_replacement", "dash") + cfg = await self._load_naming_config(session, root) # Resolve publisher - publisher_name: str | None = None if series.publisher_id: await session.refresh(series, attribute_names=["publisher"]) - publisher_name = series.publisher.name if series.publisher else None - - # Compute expected folder name - expected_name = format_series_folder( - title=series.title, - year=series.year_start, - publisher=publisher_name, - comicvine_id=series.comicvine_id, - series_type=series.series_type.value if series.series_type else None, - template=template, - replace_illegal=replace_illegal, - colon_replacement=colon_replacement, - ) root_path = Path(root.path) current_path = Path(series.path) - expected_path = root_path / expected_name + expected_path = root_path / build_series_relative_path(series, cfg) + expected_name = expected_path.name # No rename needed if already correct if current_path == expected_path: return None + await require_mutable_library_target( + session, + current_path, + include_descendants=True, + operation="renamed", + ) + # Don't overwrite an existing different folder if expected_path.exists() and expected_path != current_path: reclaimed = await asyncio.to_thread(reclaim_transient_series_folder, expected_path) @@ -547,7 +653,7 @@ async def rename_series_folder( if expected_path.exists() and expected_path != current_path: if series.comicvine_id: expected_name = f"{expected_name} [cv-{series.comicvine_id}]" - expected_path = root_path / expected_name + expected_path = expected_path.with_name(expected_name) if expected_path.exists() and expected_path != current_path: logger.warning( "series_folder_rename_collision", @@ -560,6 +666,7 @@ async def rename_series_folder( # Rename on disk if current_path.is_dir(): try: + await asyncio.to_thread(expected_path.parent.mkdir, parents=True, exist_ok=True) await asyncio.to_thread(current_path.rename, expected_path) series.path = str(expected_path) logger.info( @@ -839,6 +946,16 @@ async def delete( select(LibraryFile).where(LibraryFile.issue_id.in_(issue_ids_subq)) ) linked_files = list(file_result.scalars().all()) + managed_files = [ + library_file + for library_file in linked_files + if library_file.storage_mode == LibraryFileStorageMode.MANAGED + ] + referenced_files = [ + library_file + for library_file in linked_files + if library_file.storage_mode == LibraryFileStorageMode.REFERENCED + ] delete_target = await SeriesService.build_delete_target(session, series) folder_paths = ( @@ -849,6 +966,16 @@ async def delete( moved_folder_keys: set[str] = set() if delete_folder: for folder_path in folder_paths: + folder_references = await referenced_library_files_for_target( + session, folder_path, include_descendants=True + ) + if folder_references: + logger.info( + "referenced_folder_preserved", + series_id=series_id, + path=str(folder_path), + ) + continue if trash_dir is not None: try: move_path_to_utility_trash( @@ -872,6 +999,15 @@ async def delete( if effective_delete_files: for lf in linked_files: file_path = Path(lf.file_path) + if lf.storage_mode == LibraryFileStorageMode.REFERENCED: + await session.delete(lf) + logger.info( + "referenced_file_detached", + series_id=series_id, + library_file_id=lf.id, + path=str(file_path), + ) + continue if moved_folder_keys and any( path_key(folder_path) in moved_folder_keys and is_relative_to(file_path, folder_path) @@ -892,6 +1028,15 @@ async def delete( except OSError: logger.exception("file_delete_failed", path=str(file_path)) await session.delete(lf) + else: + for lf in referenced_files: + await session.delete(lf) + logger.info( + "referenced_file_detached", + series_id=series_id, + library_file_id=lf.id, + path=lf.file_path, + ) # ── 3. Delete series record (cascades to issues) ──────────── await purge_series_cover_cache(session, series_id) @@ -903,6 +1048,8 @@ async def delete( files_deleted=delete_files, folder_deleted=delete_folder, trashed=trash_dir is not None, + managed_files_deleted=len(managed_files) if effective_delete_files else 0, + referenced_files_detached=len(referenced_files), ) async def _apply_monitoring_on( diff --git a/src/pullbox/services/story_arc_catalog.py b/src/pullbox/services/story_arc_catalog.py new file mode 100644 index 00000000..e749cd7a --- /dev/null +++ b/src/pullbox/services/story_arc_catalog.py @@ -0,0 +1,513 @@ +"""Network-first catalog discovery and atomic, targeted story-arc adoption.""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Protocol + +from sqlalchemy import false, select, update +from sqlalchemy.exc import IntegrityError + +from pullbox.core.issue_numbers import parse_issue_number_text +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcExternalIdentity, + StoryArcLifecycle, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.services.story_arc_catalog_persistence import ( + canonical_root, + publisher_id, + seed_members, +) +from pullbox.services.story_arc_catalog_placement import initialize_catalog_placements +from pullbox.services.story_arc_catalog_types import ( + StoryArcCatalogError, + StoryArcCatalogPreview, + StoryArcCatalogRefreshPreview, + StoryArcCatalogRefreshResult, + catalog_snapshot, + exact_provider_id, + snapshot_fingerprint, +) +from pullbox.services.story_arc_placement_integration import ( + STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + StoryArcPlacementIntegrationError, + StoryArcPlacementPolicyInput, + validate_story_arc_placement_policy_input, +) +from pullbox.services.story_arc_service import StoryArcService +from pullbox.services.story_arc_sync_queue import enqueue_story_arc_sync_work + +if TYPE_CHECKING: + from collections.abc import Collection, Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.providers.base import IssueMetadata, SeriesMetadata + from pullbox.providers.story_arcs import StoryArcMetadata, StoryArcSearchResult + +__all__ = [ + "StoryArcCatalogError", + "StoryArcCatalogPreview", + "StoryArcCatalogRefreshPreview", + "StoryArcCatalogRefreshResult", + "StoryArcCatalogService", +] + +MAX_CATALOG_MEMBERS = 2_000 +MAX_CATALOG_PARENTS = 200 + + +class _CatalogProvider(Protocol): + async def search_story_arcs_page( + self, query: str, *, limit: int = 20, offset: int = 0 + ) -> tuple[list[StoryArcSearchResult], int]: ... + async def get_story_arc(self, provider_id: str) -> StoryArcMetadata: ... + async def get_story_arc_issues( + self, issue_provider_ids: Sequence[str] + ) -> list[IssueMetadata]: ... + async def get_series(self, provider_id: str) -> SeriesMetadata: ... + + +class StoryArcCatalogService: + """Fetch before opening a writer; flush within the caller's transaction only. + + Never fetch full parent catalogs, toggle parent monitoring, mutate files, or + dispatch searches. UI adapters commit adoption before scheduling acquisition. + """ + + def __init__(self, provider: _CatalogProvider) -> None: + self.provider = provider + self.domain = StoryArcService() + + async def search( + self, query: str, *, limit: int = 20, offset: int = 0 + ) -> tuple[list[StoryArcSearchResult], int]: + return await self.provider.search_story_arcs_page(query, limit=limit, offset=offset) + + async def find_existing( + self, session: AsyncSession, provider_ids: Sequence[str] + ) -> dict[str, int]: + ids = [exact_provider_id(value) for value in provider_ids] + rows = await session.execute( + select(StoryArc.comicvine_id, StoryArc.id).where(StoryArc.comicvine_id.in_(ids)) + ) + existing = {str(provider_id): arc_id for provider_id, arc_id in rows} + identities = await session.execute( + select( + StoryArcExternalIdentity.external_id, StoryArcExternalIdentity.story_arc_id + ).where( + StoryArcExternalIdentity.source == "comicvine", + StoryArcExternalIdentity.namespace == "story_arc", + StoryArcExternalIdentity.external_id.in_(provider_ids), + ) + ) + for provider_id, arc_id in identities: + if provider_id in existing and existing[provider_id] != arc_id: + raise StoryArcCatalogError( + "identity_conflict", "Provider story arc has conflicting local identities" + ) + existing[provider_id] = arc_id + return existing + + async def preview( + self, provider_id: str, *, known_series_provider_ids: Collection[str] = () + ) -> StoryArcCatalogPreview: + exact_provider_id(provider_id) + metadata = await self.provider.get_story_arc(provider_id) + if metadata.provider_id != provider_id: + raise StoryArcCatalogError( + "identity_conflict", "Provider returned a different story arc" + ) + ids = metadata.issue_provider_ids + self._validate_ids(ids) + preview = StoryArcCatalogPreview(metadata=metadata, issues=(), series=(), fingerprint="") + if metadata.membership_complete: + issues = tuple(await self.provider.get_story_arc_issues(ids)) if ids else () + self._validate_hydration(ids, issues) + parent_ids = tuple(dict.fromkeys(issue.series_provider_id for issue in issues)) + if len(parent_ids) > MAX_CATALOG_PARENTS: + raise StoryArcCatalogError( + "catalog_limit_exceeded", "This arc exceeds the supported parent-series limit" + ) + parents = [] + for parent_id in parent_ids: + if parent_id in known_series_provider_ids: + continue + parent = await self.provider.get_series(parent_id) + if parent.provider_id != parent_id or not parent.title.strip(): + raise StoryArcCatalogError( + "identity_conflict", "Provider returned a different parent series" + ) + parents.append(parent) + preview = replace(preview, issues=issues, series=tuple(parents)) + return replace(preview, fingerprint=snapshot_fingerprint(preview)) + + async def add( + self, + session: AsyncSession, + preview: StoryArcCatalogPreview, + *, + ordered_issue_provider_ids: Sequence[str], + library_root_id: int, + monitored: bool = False, + search_missing: bool = False, + include_upcoming: bool = False, + placement_policy: StoryArcPlacementPolicyInput | None = None, + skipped_issue_provider_ids: Collection[str] = (), + ) -> StoryArc: + self._validate_preview(preview) + order = tuple(ordered_issue_provider_ids) + if len(order) != len(set(order)) or set(order) != set(preview.metadata.issue_provider_ids): + raise StoryArcCatalogError( + "invalid_order", "Review a complete, nonduplicated member order" + ) + if not set(skipped_issue_provider_ids).issubset(order): + raise StoryArcCatalogError( + "invalid_skip", "Skipped members must belong to the reviewed arc" + ) + root = await canonical_root(session, library_root_id) + try: + if placement_policy is None: + from pullbox.services.story_arc_file_defaults import load_story_arc_file_defaults + + placement_policy = (await load_story_arc_file_defaults(session)).proposal() + policy = await validate_story_arc_placement_policy_input( + session, + placement_policy, + revision=1, + ) + except StoryArcPlacementIntegrationError as exc: + raise StoryArcCatalogError(exc.code, str(exc)) from exc + if await self.find_existing(session, [preview.metadata.provider_id]): + raise StoryArcCatalogError( + "already_added", "This provider story arc is already in the library" + ) + await self._enclose_savepoint(session) + try: + async with session.begin_nested(): + existing_identity = await session.scalar( + select(StoryArcExternalIdentity.id).where( + StoryArcExternalIdentity.source == "comicvine", + StoryArcExternalIdentity.namespace == "story_arc", + StoryArcExternalIdentity.external_id == preview.metadata.provider_id, + ) + ) + if existing_identity is not None: + raise StoryArcCatalogError( + "identity_conflict", "This provider story arc identity is already assigned" + ) + issues = await seed_members(session, preview, root, order) + arc = await self.domain.create( + session, + name=preview.metadata.title, + description=preview.metadata.description, + monitored=monitored, + search_missing=search_missing, + include_upcoming=include_upcoming, + sync_enabled=policy.synchronize, + source_kind=StoryArcSourceKind.PROVIDER, + ) + arc.comicvine_id = exact_provider_id(preview.metadata.provider_id) + arc.comicvine_url = preview.metadata.comicvine_url + arc.cover_url = preview.metadata.cover_url + arc.publisher_id = await publisher_id(session, preview.metadata.publisher) + arc.target_library_root_id = policy.target_library_root_id + arc.policy_schema_version = STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION + arc.policy_snapshot = policy.snapshot + session.add( + StoryArcExternalIdentity( + story_arc_id=arc.id, + source="comicvine", + namespace="story_arc", + external_id=preview.metadata.provider_id, + source_url=preview.metadata.comicvine_url, + evidence={"snapshot_fingerprint": preview.fingerprint}, + ) + ) + for position, provider_id in enumerate(order, start=1): + member = await self._member( + session, arc, preview, issues[provider_id], provider_id, position + ) + if provider_id in skipped_issue_provider_ids: + member.resolution_state = StoryArcResolutionState.SKIPPED + member.sync_eligible = False + self._diagnostics(arc, preview, root.id, (), ()) + await session.flush() + if arc.sync_enabled: + files = await session.scalars( + select(LibraryFile).where( + LibraryFile.issue_id.in_([issue.id for issue in issues.values()]) + ) + ) + for library_file in files: + await enqueue_story_arc_sync_work(session, library_file) + await initialize_catalog_placements(session, arc) + return arc + except IntegrityError as exc: + raise StoryArcCatalogError( + "identity_conflict", "Catalog identities changed; refresh the preview" + ) from exc + + async def preview_refresh( + self, session: AsyncSession, story_arc_id: int, preview: StoryArcCatalogPreview + ) -> StoryArcCatalogRefreshPreview: + self._validate_preview(preview) + arc = await self._arc(session, story_arc_id, preview) + rows = list( + ( + await session.scalars( + select(IssueStoryArc) + .where(IssueStoryArc.story_arc_id == arc.id) + .order_by( + IssueStoryArc.sequence_number, + IssueStoryArc.source_ordinal, + IssueStoryArc.id, + ) + ) + ).all() + ) + existing = { + row.source_issue_id + for row in rows + if row.source_kind is StoryArcSourceKind.PROVIDER + and row.source_arc_id == preview.metadata.provider_id + } + canonical_ids = await session.scalars( + select(Issue.comicvine_id) + .join(IssueStoryArc, IssueStoryArc.issue_id == Issue.id) + .where(IssueStoryArc.story_arc_id == arc.id, Issue.comicvine_id.is_not(None)) + ) + represented_ids = existing | {str(value) for value in canonical_ids} + incoming = set(preview.metadata.issue_provider_ids) + return StoryArcCatalogRefreshPreview( + arc.id, + arc.revision, + tuple( + value + for value in preview.metadata.issue_provider_ids + if value not in represented_ids + ), + tuple( + row.source_issue_id + for row in rows + if row.source_issue_id in existing + and row.source_issue_id not in incoming + and row.source_issue_id is not None + ), + ) + + async def refresh( + self, + session: AsyncSession, + story_arc_id: int, + preview: StoryArcCatalogPreview, + *, + expected_revision: int, + library_root_id: int | None = None, + ) -> StoryArcCatalogRefreshResult: + delta = await self.preview_refresh(session, story_arc_id, preview) + if isinstance(expected_revision, bool) or delta.revision != expected_revision: + raise StoryArcCatalogError("revision_conflict", "Story arc changed; refresh the review") + arc = await self._arc(session, story_arc_id, preview) + catalog = arc.diagnostics.get("provider_catalog", {}) + root_id = catalog.get("canonical_library_root_id") + if root_id is None: + root_id = library_root_id + root = await canonical_root(session, root_id) + await self._enclose_savepoint(session) + try: + async with session.begin_nested(): + claimed = await session.execute( + update(StoryArc) + .where(StoryArc.id == arc.id, StoryArc.revision == expected_revision) + .values(revision=expected_revision + 1) + ) + if claimed.rowcount != 1: # type: ignore[attr-defined] + raise StoryArcCatalogError( + "revision_conflict", "Story arc changed; refresh the review" + ) + issues = await seed_members( + session, preview, root, preview.metadata.issue_provider_ids + ) + rows = list( + ( + await session.scalars( + select(IssueStoryArc).where(IssueStoryArc.story_arc_id == arc.id) + ) + ).all() + ) + position = max((row.sequence_number for row in rows), default=0) + for row in rows: + if ( + row.source_kind is StoryArcSourceKind.PROVIDER + and row.source_arc_id == preview.metadata.provider_id + and row.source_issue_id in issues + and row.issue_id is not None + and row.issue_id != issues[row.source_issue_id].id + ): + raise StoryArcCatalogError( + "identity_conflict", + "Membership disagrees with its exact provider identity", + ) + created = [] + for offset, provider_id in enumerate(delta.added_issue_provider_ids, start=1): + member = await self._member( + session, arc, preview, issues[provider_id], provider_id, position + offset + ) + # Exact provider identity is sufficient for acquisition. + # Provider response order is not verified reading order, so + # keep placement paused until the user confirms this entry. + member.sync_eligible = False + member.evidence = {**member.evidence, "catalog_review_required": True} + created.append(member.id) + pending = tuple( + row.id for row in rows if row.evidence.get("catalog_review_required") is True + ) + tuple(created) + arc.cover_url = preview.metadata.cover_url + self._diagnostics(arc, preview, root.id, delta.removed_issue_provider_ids, pending) + await session.flush() + return StoryArcCatalogRefreshResult( + arc, tuple(created), delta.removed_issue_provider_ids + ) + except IntegrityError as exc: + raise StoryArcCatalogError( + "identity_conflict", "Catalog identities changed; refresh the preview" + ) from exc + + async def _arc( + self, session: AsyncSession, story_arc_id: int, preview: StoryArcCatalogPreview + ) -> StoryArc: + arc = await session.get(StoryArc, story_arc_id) + if arc is None or arc.lifecycle is not StoryArcLifecycle.ACTIVE: + raise StoryArcCatalogError("arc_unavailable", "Story arc is unavailable or archived") + if arc.comicvine_id != exact_provider_id(preview.metadata.provider_id): + raise StoryArcCatalogError( + "identity_conflict", "Provider snapshot belongs to a different story arc" + ) + return arc + + async def _member( + self, + session: AsyncSession, + arc: StoryArc, + preview: StoryArcCatalogPreview, + issue: Issue, + provider_id: str, + position: int, + ) -> IssueStoryArc: + metadata = next(value for value in preview.issues if value.provider_id == provider_id) + member = await self.domain.add_membership( + session, + arc.id, + issue_id=issue.id, + sequence_number=position, + source_ordinal=preview.metadata.issue_provider_ids.index(provider_id) + 1, + source_kind=StoryArcSourceKind.PROVIDER, + source_issue_number_text=metadata.issue_number_text or metadata.issue_number, + ) + member.source_entry_id = provider_id + member.source_issue_id = provider_id + member.source_series_id = metadata.series_provider_id + member.source_arc_id = preview.metadata.provider_id + member.source_issue_title = metadata.title + member.source_release_date_text = metadata.release_date + parent = next( + (value for value in preview.series if value.provider_id == metadata.series_provider_id), + None, + ) + if parent is not None: + member.source_series_name = parent.title + member.source_publisher = parent.publisher + member.resolution_method = "exact_comicvine_id" + member.resolution_confidence = 1.0 + member.evidence = { + "provider": "comicvine", + "snapshot_fingerprint": preview.fingerprint, + "order_basis": preview.order_basis, + "provider_response_ordinal": member.source_ordinal, + } + return member + + @staticmethod + async def _enclose_savepoint(session: AsyncSession) -> None: + """Keep SQLite's deferred BEGIN from committing the outer savepoint. + + Legacy sqlite3 transaction control starts BEGIN on DML, not SELECT or + SAVEPOINT. This zero-row statement establishes the caller-owned outer + transaction without changing records, so releasing our savepoint cannot + commit the caller's work. PostgreSQL already encloses savepoints correctly. + """ + if session.get_bind().dialect.name == "sqlite": + await session.execute( + update(StoryArc).where(false()).values(revision=StoryArc.revision) + ) + + @staticmethod + def _diagnostics( + arc: StoryArc, + preview: StoryArcCatalogPreview, + root_id: int, + removed: tuple[str, ...], + pending: tuple[int, ...], + ) -> None: + arc.diagnostics = { + **arc.diagnostics, + "provider_refresh_error": None, + "provider_catalog": { + "schema_version": 1, + "snapshot": catalog_snapshot(preview), + "snapshot_fingerprint": preview.fingerprint, + "order_basis": preview.order_basis, + "fetched_at": datetime.now(UTC).isoformat(), + "canonical_library_root_id": root_id, + "removed_issue_provider_ids": list(removed), + "pending_membership_ids": list(pending), + }, + } + + @staticmethod + def _validate_ids(ids: Sequence[str]) -> None: + if len(ids) > MAX_CATALOG_MEMBERS: + raise StoryArcCatalogError( + "catalog_limit_exceeded", "This arc exceeds the supported membership limit" + ) + if len(ids) != len(set(ids)): + raise StoryArcCatalogError( + "identity_conflict", "Provider returned duplicate arc members" + ) + for value in ids: + exact_provider_id(value) + + @staticmethod + def _validate_hydration(ids: Sequence[str], issues: Sequence[IssueMetadata]) -> None: + if tuple(issue.provider_id for issue in issues) != tuple(ids): + raise StoryArcCatalogError( + "incomplete_hydration", "Provider did not hydrate the exact complete arc membership" + ) + for issue in issues: + exact_provider_id(issue.series_provider_id) + try: + parse_issue_number_text(issue.issue_number_text or issue.issue_number) + except ValueError as exc: + raise StoryArcCatalogError( + "invalid_issue_number", "Provider returned an invalid exact issue number" + ) from exc + + def _validate_preview(self, preview: StoryArcCatalogPreview) -> None: + if snapshot_fingerprint(preview) != preview.fingerprint: + raise StoryArcCatalogError( + "snapshot_changed", "Catalog snapshot changed; review it again" + ) + if not preview.membership_complete: + raise StoryArcCatalogError( + "incomplete_membership", "Provider membership is incomplete; nothing was added" + ) + self._validate_ids(preview.metadata.issue_provider_ids) + self._validate_hydration(preview.metadata.issue_provider_ids, preview.issues) diff --git a/src/pullbox/services/story_arc_catalog_persistence.py b/src/pullbox/services/story_arc_catalog_persistence.py new file mode 100644 index 00000000..b7bcd268 --- /dev/null +++ b/src/pullbox/services/story_arc_catalog_persistence.py @@ -0,0 +1,207 @@ +"""Targeted canonical seeding for arcs without whole-series adoption side effects.""" + +from __future__ import annotations + +from datetime import date +from pathlib import Path +from typing import TYPE_CHECKING + +from sqlalchemy import func, select + +from pullbox.core.exceptions import ValidationError +from pullbox.core.issue_numbers import parse_issue_number_text +from pullbox.core.library_naming import build_series_relative_path +from pullbox.core.library_policy import load_effective_library_ingest_policy +from pullbox.core.naming import classify_series_type, detect_issue_type_from_metadata_title +from pullbox.core.type_semantics import canonical_issue_type_for_series_type +from pullbox.models.issue import Issue, IssueStatus, IssueType +from pullbox.models.library import LibraryRoot +from pullbox.models.publisher import Publisher +from pullbox.models.series import IssueCatalogState, Series, SeriesStatus, SeriesType +from pullbox.services.library_root_management import validate_managed_library_root +from pullbox.services.story_arc_catalog_types import StoryArcCatalogError, exact_provider_id + +if TYPE_CHECKING: + from collections.abc import Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + + from pullbox.providers.base import SeriesMetadata + from pullbox.services.story_arc_catalog_types import StoryArcCatalogPreview + + +async def canonical_root(session: AsyncSession, root_id: int) -> LibraryRoot: + if isinstance(root_id, bool) or not isinstance(root_id, int) or root_id < 1: + raise StoryArcCatalogError("canonical_root_required", "Select a canonical library root") + root = await session.get(LibraryRoot, root_id) + if root is None: + raise StoryArcCatalogError( + "canonical_root_unavailable", "Canonical library root is unavailable" + ) + try: + await validate_managed_library_root(root) + except ValidationError as exc: + raise StoryArcCatalogError( + "canonical_root_unavailable", "Canonical library root is unavailable" + ) from exc + return root + + +async def publisher_id(session: AsyncSession, name: str | None) -> int | None: + if not name: + return None + publisher = await session.scalar(select(Publisher).where(Publisher.name == name)) + if publisher is None: + publisher = Publisher(name=name) + session.add(publisher) + await session.flush() + return publisher.id + + +async def seed_members( + session: AsyncSession, + preview: StoryArcCatalogPreview, + root: LibraryRoot, + provider_ids: Sequence[str], +) -> dict[str, Issue]: + """Create only absent exact identities; all existing rows remain untouched. + + The caller's savepoint makes identity/path conflicts atomic. Paths are reserved + in the database only; normal acquisition creates canonical folders later. + """ + metadata_by_id = {issue.provider_id: issue for issue in preview.issues} + series_metadata = {series.provider_id: series for series in preview.series} + result: dict[str, Issue] = {} + parents: dict[str, Series] = {} + for provider_id in provider_ids: + metadata = metadata_by_id[provider_id] + parent_key = metadata.series_provider_id + parent = parents.get(parent_key) + if parent is None: + parent = await session.scalar( + select(Series).where(Series.comicvine_id == exact_provider_id(parent_key)) + ) + if parent is None: + parent_metadata = series_metadata.get(parent_key) + if parent_metadata is None: + raise StoryArcCatalogError( + "parent_metadata_missing", "Canonical parent changed; refresh the preview" + ) + parent = await _new_series(session, parent_metadata, root) + parents[parent_key] = parent + number, exact_number = parse_issue_number_text( + metadata.issue_number_text or metadata.issue_number + ) + issue = await session.scalar( + select(Issue).where(Issue.comicvine_id == exact_provider_id(provider_id)) + ) + if issue is not None: + if issue.series_id != parent.id or issue.effective_issue_number_text != exact_number: + raise StoryArcCatalogError( + "identity_conflict", + "Existing issue disagrees with the provider's exact identity", + ) + else: + sibling = await session.scalar( + select(Issue.id).where( + Issue.series_id == parent.id, Issue.issue_number_text == exact_number + ) + ) + if sibling is not None: + raise StoryArcCatalogError( + "identity_conflict", + "An issue with a different identity already uses this exact number", + ) + detected_type = IssueType(detect_issue_type_from_metadata_title(metadata.title)) + if detected_type is IssueType.ISSUE: + detected_type = canonical_issue_type_for_series_type(parent.series_type) + issue = Issue( + series_id=parent.id, + comicvine_id=exact_provider_id(provider_id), + issue_number=number, + issue_number_text=exact_number, + title=metadata.title, + description=metadata.description, + release_date=_date(metadata.release_date), + store_date=_date(metadata.store_date), + cover_url=metadata.cover_url, + comicvine_url=metadata.comicvine_url, + page_count=metadata.page_count, + metadata_source="comicvine", + issue_type=detected_type, + status=IssueStatus.SKIPPED, + manual_skip=False, + ) + session.add(issue) + await session.flush() + result[provider_id] = issue + return result + + +async def _new_series(session: AsyncSession, metadata: SeriesMetadata, root: LibraryRoot) -> Series: + series = Series( + comicvine_id=exact_provider_id(metadata.provider_id), + title=metadata.title, + sort_title=metadata.sort_title or metadata.title, + year_start=metadata.year_start, + year_end=metadata.year_end, + description=metadata.description, + cover_url=metadata.cover_url, + comicvine_url=metadata.comicvine_url, + issue_count=metadata.issue_count or 0, + status=SeriesStatus.ENDED if metadata.status == "ended" else SeriesStatus.CONTINUING, + metadata_source="comicvine_partial", + monitored=False, + issue_catalog_state=IssueCatalogState.PARTIAL, + series_type=SeriesType( + classify_series_type( + metadata.title, + description=metadata.description, + issue_count=metadata.issue_count or 0, + year_start=metadata.year_start, + ) + ), + library_root_id=root.id, + ) + identifier = await publisher_id(session, metadata.publisher) + series.publisher = await session.get(Publisher, identifier) if identifier is not None else None + session.add(series) + policy = await load_effective_library_ingest_policy(session, root) + path = Path(root.path) / build_series_relative_path(series, policy) + if await _path_claimed(session, path): + path = path.with_name(f"{path.name} [cv-{metadata.provider_id}]") + if await _path_claimed(session, path): + raise StoryArcCatalogError( + "canonical_path_collision", "Canonical series path is already in use" + ) + if not path.resolve().is_relative_to(Path(root.path).resolve()): + raise StoryArcCatalogError( + "canonical_path_unsafe", "Canonical series path escapes its library root" + ) + if len(str(path)) > 1000: + raise StoryArcCatalogError("canonical_path_unsafe", "Canonical series path is too long") + series.path = str(path) + session.add(series) + await session.flush() + return series + + +async def _path_claimed(session: AsyncSession, path: Path) -> bool: + return ( + path.exists() + or path.is_symlink() + or bool( + await session.scalar( + select(Series.id).where(func.lower(Series.path) == str(path).lower()).limit(1) + ) + ) + ) + + +def _date(value: str | None) -> date | None: + if not value: + return None + try: + return date.fromisoformat(value) + except ValueError: + return None diff --git a/src/pullbox/services/story_arc_catalog_placement.py b/src/pullbox/services/story_arc_catalog_placement.py new file mode 100644 index 00000000..f648ee57 --- /dev/null +++ b/src/pullbox/services/story_arc_catalog_placement.py @@ -0,0 +1,297 @@ +"""Explicit, resumable initial placements for a newly adopted provider arc. + +This is not a second placement executor. It records the exact creation-time +members and delegates each artifact to the existing safe manual sync service. +Nothing runs at startup; adapters invoke it after commit or explicit user retry. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from dataclasses import asdict, dataclass, replace +from typing import TYPE_CHECKING +from weakref import WeakValueDictionary + +from sqlalchemy import func, select + +from pullbox.database import get_session_factory +from pullbox.models.library import LibraryFile +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcResolutionState, +) +from pullbox.services.story_arc_catalog_types import StoryArcCatalogError +from pullbox.services.story_arc_placement_integration import ( + StoryArcPlacementIntegrationError, + StoryArcPlacementSyncService, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +_KEY = "catalog_initial_placements" +_LOCKS: WeakValueDictionary[int, asyncio.Lock] = WeakValueDictionary() +_MAX_ITEMS = 2_000 + + +@dataclass(frozen=True, slots=True) +class StoryArcCatalogPlacementResult: + total: int + completed: int + failed: int + pending: int + + +@dataclass(frozen=True, slots=True) +class _Item: + membership_id: int + library_file_id: int + source_hash: str + state: str = "pending" + error_code: str | None = None + + +@dataclass(frozen=True, slots=True) +class _Marker: + expected_revision: int + policy_hash: str + items: tuple[_Item, ...] + + @property + def result(self) -> StoryArcCatalogPlacementResult: + complete = sum(item.state == "complete" for item in self.items) + failed = sum(item.state == "failed" for item in self.items) + return StoryArcCatalogPlacementResult( + len(self.items), complete, failed, len(self.items) - complete - failed + ) + + def payload(self) -> dict[str, object]: + result = self.result + return { + "schema_version": 1, + "expected_revision": self.expected_revision, + "policy_hash": self.policy_hash, + "items": [asdict(item) for item in self.items], + "state": "pending" if result.pending else "failed" if result.failed else "complete", + **asdict(result), + } + + +async def initialize_catalog_placements(session: AsyncSession, arc: StoryArc) -> None: + """Record creation-time available members atomically with the new arc.""" + if arc.policy_snapshot.get("mode") not in {"copy", "hardlink", "symlink"}: + return + selected_file = ( + select(func.min(LibraryFile.id)) + .where(LibraryFile.issue_id == IssueStoryArc.issue_id) + .correlate(IssueStoryArc) + .scalar_subquery() + ) + rows = await session.execute( + select(IssueStoryArc.id, LibraryFile) + .join(LibraryFile, LibraryFile.id == selected_file) + .where( + IssueStoryArc.story_arc_id == arc.id, + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + ) + .order_by(IssueStoryArc.sequence_number, IssueStoryArc.source_ordinal, IssueStoryArc.id) + .limit(_MAX_ITEMS + 1) + ) + items = tuple(_Item(membership_id, file.id, _source_hash(file)) for membership_id, file in rows) + if len(items) > _MAX_ITEMS: + raise StoryArcCatalogError("catalog_limit_exceeded", "Too many initial placements") + _store(arc, _Marker(arc.revision, _hash(arc.policy_snapshot), items)) + await session.flush() + + +async def run_catalog_initial_placements( + story_arc_id: int, + *, + retry_failed: bool = False, + session_factory: async_sessionmaker[AsyncSession] | None = None, + batch_size: int = 25, +) -> StoryArcCatalogPlacementResult: + """Process bounded sequential pages after commit; explicit retry resumes work. + + Current/running work is idempotently resumable after a restart. Failed items + are retried only when the user explicitly requests it. Per-member failures + never remove the logical arc or turn canonical acquisition into a failure. + """ + if isinstance(batch_size, bool) or not 1 <= batch_size <= 100: + raise ValueError("Initial placement batch size must be between 1 and 100") + factory = session_factory or get_session_factory() + lock = _LOCKS.setdefault(story_arc_id, asyncio.Lock()) + async with lock: + attempted: set[int] = set() + while True: + async with factory() as session: + arc = await _arc(session, story_arc_id) + marker = _marker(arc) + pending = [ + item + for item in marker.items + if item.membership_id not in attempted + and ( + item.state in {"pending", "running"} + or (retry_failed and item.state == "failed") + ) + ][:batch_size] + if not pending: + return marker.result + for item in pending: + attempted.add(item.membership_id) + await _run_one(factory, story_arc_id, item.membership_id) + await asyncio.sleep(0) + + +async def _run_one( + factory: async_sessionmaker[AsyncSession], arc_id: int, membership_id: int +) -> None: + async with factory() as session: + arc = await _arc(session, arc_id) + marker = _marker(arc) + item = next(value for value in marker.items if value.membership_id == membership_id) + code: str | None = None + member = await session.get(IssueStoryArc, membership_id) + file = await session.get(LibraryFile, item.library_file_id) + selected_file_id = ( + await session.scalar( + select(func.min(LibraryFile.id)).where(LibraryFile.issue_id == member.issue_id) + ) + if member is not None + else None + ) + if ( + arc.lifecycle is not StoryArcLifecycle.ACTIVE + or arc.revision != marker.expected_revision + or _hash(arc.policy_snapshot) != marker.policy_hash + ): + code = "initial_placement_review_changed" + elif ( + member is None + or member.story_arc_id != arc_id + or member.resolution_state is not StoryArcResolutionState.RESOLVED + ): + code = "initial_placement_member_changed" + elif ( + file is None + or selected_file_id != item.library_file_id + or file.issue_id != member.issue_id + or _source_hash(file) != item.source_hash + ): + code = "initial_placement_source_changed" + if code is None: + _store(arc, _replace_item(marker, membership_id, "running", None)) + await session.commit() + try: + await StoryArcPlacementSyncService().sync_membership(session, arc_id, membership_id) + except StoryArcPlacementIntegrationError as exc: + await session.rollback() + code = exc.code + except Exception: + await session.rollback() + code = "initial_placement_failed" + # sync_membership commits its artifact journal and expires ORM objects. + # Re-read to preserve other diagnostic keys before checkpointing progress. + session.expire_all() + arc = await _arc(session, arc_id) + _store( + arc, _replace_item(_marker(arc), membership_id, "failed" if code else "complete", code) + ) + await session.commit() + + +async def _arc(session: AsyncSession, story_arc_id: int) -> StoryArc: + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise StoryArcCatalogError("arc_unavailable", "Story arc is unavailable") + return arc + + +def _replace_item(marker: _Marker, membership_id: int, state: str, code: str | None) -> _Marker: + return replace( + marker, + items=tuple( + replace(item, state=state, error_code=code) + if item.membership_id == membership_id + else item + for item in marker.items + ), + ) + + +def _store(arc: StoryArc, marker: _Marker) -> None: + arc.diagnostics = {**arc.diagnostics, _KEY: marker.payload()} + + +def _marker(arc: StoryArc) -> _Marker: + raw = arc.diagnostics.get(_KEY) + if raw is None: + return _Marker(arc.revision, _hash(arc.policy_snapshot), ()) + try: + if not isinstance(raw, dict) or raw["schema_version"] != 1: + raise ValueError + revision = raw["expected_revision"] + policy_hash = raw["policy_hash"] + values = raw["items"] + if ( + not _positive_int(revision) + or not _hash_string(policy_hash) + or not isinstance(values, list) + or len(values) > _MAX_ITEMS + ): + raise ValueError + items = [] + for value in values: + if not isinstance(value, dict): + raise ValueError + item = _Item(**value) + if ( + not _positive_int(item.membership_id) + or not _positive_int(item.library_file_id) + or not _hash_string(item.source_hash) + or item.state not in {"pending", "running", "failed", "complete"} + ): + raise ValueError + if item.error_code is not None and ( + not isinstance(item.error_code, str) or len(item.error_code) > 100 + ): + raise ValueError + items.append(item) + if len({item.membership_id for item in items}) != len(items): + raise ValueError + return _Marker(revision, policy_hash, tuple(items)) + except (KeyError, TypeError, ValueError) as exc: + raise StoryArcCatalogError( + "initial_placement_state_invalid", "Initial placement state needs review" + ) from exc + + +def _positive_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +def _hash_string(value: object) -> bool: + return ( + isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value) + ) + + +def _hash(value: object) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest() + + +def _source_hash(file: LibraryFile) -> str: + return _hash( + { + "path": file.file_path, + "size": file.file_size, + "modified": file.file_modified_at, + "hash": file.file_hash, + "signature": file.source_signature, + } + ) diff --git a/src/pullbox/services/story_arc_catalog_types.py b/src/pullbox/services/story_arc_catalog_types.py new file mode 100644 index 00000000..1b0cae6d --- /dev/null +++ b/src/pullbox/services/story_arc_catalog_types.py @@ -0,0 +1,92 @@ +"""Immutable provider-catalog review contracts; never persisted as new schema.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass +from typing import TYPE_CHECKING + +from pullbox.services.story_arc_service import StoryArcValidationError + +if TYPE_CHECKING: + from pullbox.models.story_arc import StoryArc + from pullbox.providers.base import IssueMetadata, SeriesMetadata + from pullbox.providers.story_arcs import StoryArcMetadata + + +class StoryArcCatalogError(StoryArcValidationError): + """A safe, actionable validation error at the catalog boundary.""" + + def __init__(self, code: str, message: str) -> None: + self.code = code + super().__init__(message) + + +@dataclass(frozen=True, slots=True) +class StoryArcCatalogPreview: + """A fully fetched snapshot; no ORM objects or open network work.""" + + metadata: StoryArcMetadata + issues: tuple[IssueMetadata, ...] + series: tuple[SeriesMetadata, ...] + fingerprint: str + + @property + def membership_complete(self) -> bool: + return self.metadata.membership_complete + + @property + def order_basis(self) -> str: + return self.metadata.order_basis + + +@dataclass(frozen=True, slots=True) +class StoryArcCatalogRefreshPreview: + story_arc_id: int + revision: int + added_issue_provider_ids: tuple[str, ...] + removed_issue_provider_ids: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class StoryArcCatalogRefreshResult: + story_arc: StoryArc + added_membership_ids: tuple[int, ...] + removed_issue_provider_ids: tuple[str, ...] + + +def catalog_snapshot(preview: StoryArcCatalogPreview) -> dict[str, object]: + """Only provider-public data is stored; no credentials or local file paths.""" + metadata = preview.metadata + return { + "provider": "comicvine", + "provider_id": metadata.provider_id, + "title": metadata.title, + "description": metadata.description, + "publisher": metadata.publisher, + "cover_url": metadata.cover_url, + "comicvine_url": metadata.comicvine_url, + "issue_provider_ids": list(metadata.issue_provider_ids), + "declared_issue_count": metadata.declared_issue_count, + "membership_complete": metadata.membership_complete, + "order_basis": metadata.order_basis, + "warnings": list(metadata.warnings), + "issues": [asdict(issue) for issue in preview.issues], + "series": [asdict(series) for series in preview.series], + } + + +def snapshot_fingerprint(preview: StoryArcCatalogPreview) -> str: + payload = json.dumps(catalog_snapshot(preview), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + +def exact_provider_id(value: str) -> int: + """Accept a canonical positive numeric CV ID, never a fuzzy or coerced key.""" + if not isinstance(value, str) or not value.isascii() or not value.isdecimal(): + raise StoryArcCatalogError("invalid_identity", "Provider identity must be a positive ID") + number = int(value) + if number < 1 or str(number) != value or number > 2**63 - 1: + raise StoryArcCatalogError("invalid_identity", "Provider identity must be a positive ID") + return number diff --git a/src/pullbox/services/story_arc_editing_policy.py b/src/pullbox/services/story_arc_editing_policy.py new file mode 100644 index 00000000..3e89b23a --- /dev/null +++ b/src/pullbox/services/story_arc_editing_policy.py @@ -0,0 +1,27 @@ +"""Release gate for user-authored overrides, separate from provider/import writes.""" + +from pullbox.config import get_settings +from pullbox.models.story_arc import StoryArc, StoryArcSourceKind +from pullbox.services.story_arc_service import StoryArcServiceError + + +class StoryArcManualEditingDisabledError(StoryArcServiceError): + """Provider metadata and membership cannot currently be edited manually.""" + + +def can_manually_edit_arc(arc: StoryArc) -> bool: + """Keep unlinked imported/custom lists editable, even when creation is off.""" + provider_managed = ( + arc.source_kind == StoryArcSourceKind.PROVIDER or arc.comicvine_id is not None + ) + return not provider_managed or get_settings().story_arc_manual_edit_enabled + + +def require_manual_arc_edit(arc: StoryArc) -> None: + """Check user-facing writes without blocking catalog refresh or imports.""" + if not can_manually_edit_arc(arc): + raise StoryArcManualEditingDisabledError( + "This story arc is managed by its metadata provider. Manual metadata and membership " + "edits are not enabled. Monitoring, reading order, and storage settings " + "remain available." + ) diff --git a/src/pullbox/services/story_arc_file_defaults.py b/src/pullbox/services/story_arc_file_defaults.py new file mode 100644 index 00000000..da9504b8 --- /dev/null +++ b/src/pullbox/services/story_arc_file_defaults.py @@ -0,0 +1,133 @@ +"""Global file defaults for new arcs; existing/imported policies never inherit live settings.""" + +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING, Literal + +from pydantic import BaseModel, Field, ValidationError +from sqlalchemy import select + +from pullbox.core.story_arc_naming import StoryArcNamingValues, render_story_arc_relative_path +from pullbox.models.config import DEFAULT_SYSTEM_CONFIG, SystemConfig +from pullbox.services.story_arc_placement_integration import ( + StoryArcPlacementIntegrationError, + StoryArcPlacementPolicyInput, + validate_story_arc_placement_policy_input, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +STORY_ARC_FILE_DEFAULT_KEYS = tuple( + key for key in DEFAULT_SYSTEM_CONFIG if key.startswith("story_arc_files_") +) + + +class StoryArcFileDefaults(BaseModel): + """Editable preferences, separate from the complete effective per-arc snapshot.""" + + enabled: bool + method: Literal["copy", "hardlink", "symlink"] + library_root_id: str = Field(max_length=20) + destination: str = Field(max_length=1000) + folder_template: str = Field(max_length=1024) + filename_style: Literal["original", "custom"] + prefix_reading_order: bool + reading_order_width: int = Field(ge=2, le=6) + file_template: str = Field(max_length=1024) + symlink_style: Literal["relative", "absolute"] + synchronize: bool + + @property + def fingerprint(self) -> str: + return hashlib.sha256(json.dumps(self.model_dump(), sort_keys=True).encode()).hexdigest() + + @property + def summary_label(self) -> str: + if not self.enabled: + return "No separate folder" + return { + "copy": "Copy issues into arc folders", + "hardlink": "Hardlink issues into arc folders", + "symlink": "Symlink issues into arc folders", + }[self.method] + + def proposal(self) -> StoryArcPlacementPolicyInput: + template = self.file_template + if self.filename_style == "original": + template = "{OriginalFilename}" + if self.prefix_reading_order: + template = f"{{ReadingOrder:0{self.reading_order_width}d}} - {template}" + try: + root_id = int(self.library_root_id) if self.library_root_id else None + except ValueError as exc: + raise StoryArcPlacementIntegrationError( + "invalid_root", "Choose a valid Story Arc library root." + ) from exc + return StoryArcPlacementPolicyInput( + mode=self.method if self.enabled else "logical", + target_library_root_id=root_id if self.enabled else None, + destination_root=(self.destination.strip() or None) if self.enabled else None, + folder_template=self.folder_template, + file_template=template, + symlink_style=self.symlink_style if self.enabled and self.method == "symlink" else None, + synchronize=self.enabled and self.synchronize, + ) + + def naming_preview(self) -> str: + proposal = self.proposal() + return str( + render_story_arc_relative_path( + StoryArcNamingValues( + story_arc="The Court of Owls", + reading_order=1, + series="Batman", + issue_number="001", + extension="cbz", + issue_title="Knife Trick", + year=2011, + start_year=2011, + end_year=2012, + publisher="DC Comics", + original_filename="Batman 001.cbz", + ), + folder_template=proposal.folder_template, + file_template=proposal.file_template, + ) + ) + + +def parse_story_arc_file_defaults(values: dict[str, str]) -> StoryArcFileDefaults: + try: + defaults = StoryArcFileDefaults.model_validate( + { + key.removeprefix("story_arc_files_"): values.get(key, DEFAULT_SYSTEM_CONFIG[key][0]) + for key in STORY_ARC_FILE_DEFAULT_KEYS + } + ) + defaults.naming_preview() + return defaults + except ValidationError as exc: + error = exc.errors()[0] + field = str(error["loc"][0]).replace("_", " ") + raise StoryArcPlacementIntegrationError( + "invalid_defaults", f"Story Arc {field}: {error['msg']}" + ) from exc + except ValueError as exc: + raise StoryArcPlacementIntegrationError( + "invalid_defaults", "Invalid Story Arc file defaults. " + str(exc) + ) from exc + + +async def load_story_arc_file_defaults(session: AsyncSession) -> StoryArcFileDefaults: + rows = await session.scalars( + select(SystemConfig).where(SystemConfig.key.in_(STORY_ARC_FILE_DEFAULT_KEYS)) + ) + return parse_story_arc_file_defaults({row.key: row.value for row in rows}) + + +async def validate_story_arc_file_defaults(session: AsyncSession, values: dict[str, str]) -> None: + defaults = parse_story_arc_file_defaults(values) + await validate_story_arc_placement_policy_input(session, defaults.proposal(), revision=1) diff --git a/src/pullbox/services/story_arc_managed_reorder.py b/src/pullbox/services/story_arc_managed_reorder.py new file mode 100644 index 00000000..f6ebcd81 --- /dev/null +++ b/src/pullbox/services/story_arc_managed_reorder.py @@ -0,0 +1,2126 @@ +"""Previewed, crash-truthful reordering for managed Story Arc placements. + +Adjacent UI moves affect at most two memberships. This service keeps that +boundary explicit: it renders a complete two-membership plan, signs it for +confirmation, durably reserves every managed artifact, and only then moves +files outside a database transaction. Canonical files and referenced +placements are observations only. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +from contextlib import suppress +from dataclasses import asdict, dataclass, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Literal, NoReturn, cast +from uuid import uuid4 + +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +from sqlalchemy import and_, or_, select +from sqlalchemy import update as sa_update + +from pullbox.core.config_resolver import get_application_secret +from pullbox.core.story_arc_naming import StoryArcOriginalFilenameError +from pullbox.models.library import LibraryFile +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcPlacement, + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcPlacementState, +) +from pullbox.services.story_arc_placement_integration import ( + STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + StoryArcPlacementPolicyMode, + _load_one_context, + _policy_from_arc, + _rendered_target_path, +) +from pullbox.services.story_arc_placement_service import ( + Fingerprint, + StoryArcPlacementError, + _case_only_collision, + _case_only_collision_at, + _entry_exists_at, + _fingerprint_target, + _fingerprint_target_at, + _fsync_directory, + _open_secure_parent_directory, + _path_exists, + _reject_canonical_destination, + _SecureParentDirectory, + _validate_path_limits, + _validate_removal_representation_at, + _validate_target_lexically, + _validated_root, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql import Select + +StoryArcReorderDirection = Literal["up", "down"] +StoryArcReorderAction = Literal[ + "rename", + "managed_unchanged", + "referenced_drift", + "referenced_unchanged", + "logical_reorder", +] + +_TOKEN_SALT = "pullbox-story-arc-managed-reorder-v1" +_TOKEN_MAX_AGE_SECONDS = 15 * 60 +_TOKEN_SCHEMA_VERSION = 1 +_JOURNAL_SCHEMA_VERSION = 1 +_MAX_PLACEMENTS_PER_ADJACENT_MOVE = 100 +_MAX_COLLISION_SCAN_PATHS = 204 +_UNCHANGED_MANAGED_STATUSES = frozenset({"complete", "rename_cancelled", "rename_failed"}) +_ACTIVE_REORDER_STATUSES = frozenset({"rename_prepared", "rename_recovery_required"}) + + +class StoryArcManagedReorderError(RuntimeError): + """Categorized failure that preserves safe UI and recovery semantics.""" + + def __init__(self, code: str, message: str, *, category: str = "validation") -> None: + self.code = code + self.category = category + super().__init__(message) + + +@dataclass(frozen=True, slots=True) +class StoryArcReorderPreviewItem: + """One complete old/new placement consequence in a signed preview.""" + + membership_id: int + placement_id: int | None + ownership: str + mode: str + old_reading_order: int + new_reading_order: int + old_path: str | None + new_path: str | None + rendered_path_after: str | None + temporary_path: str | None + action: StoryArcReorderAction + + +@dataclass(frozen=True, slots=True) +class StoryArcReorderPreview: + """Read-only adjacent-move preview requiring explicit confirmation.""" + + story_arc_id: int + membership_id: int + direction: StoryArcReorderDirection + expected_revision: int + preview_token: str + items: tuple[StoryArcReorderPreviewItem, ...] + managed_rename_count: int + referenced_drift_count: int + referenced_preserved_count: int + recovery_pending: bool + requires_confirmation: bool = True + filesystem_mutated: bool = False + + +@dataclass(frozen=True, slots=True) +class StoryArcReorderResult: + """Truthful completed reorder counts.""" + + story_arc_id: int + revision: int + managed_renamed: int + referenced_preserved: int + recovery_pending: bool = False + + +@dataclass(frozen=True, slots=True) +class _MembershipPlan: + membership_id: int + old_sequence_number: int + old_source_ordinal: int + new_sequence_number: int + new_source_ordinal: int + + +@dataclass(frozen=True, slots=True) +class _PlacementPlan: + placement_id: int + membership_id: int + ownership: str + mode: str + old_reading_order: int + new_reading_order: int + old_path: str + new_path: str + temporary_path: str | None + rendered_path_after: str + action: StoryArcReorderAction + canonical_path: str | None + source_fingerprint: Fingerprint + target_fingerprint: Fingerprint + placement_state: str | None + last_result_snapshot: dict[str, object] + + +@dataclass(frozen=True, slots=True) +class _SignedPlan: + story_arc_id: int + selected_membership_id: int + direction: StoryArcReorderDirection + expected_revision: int + operation_token: str + plan_digest: str + destination_root: str | None + memberships: tuple[_MembershipPlan, ...] + placements: tuple[_PlacementPlan, ...] + + +@dataclass(frozen=True, slots=True) +class _FilesystemResult: + fingerprints: dict[int, Fingerprint] + + +class _CancelledDuringReorderError(RuntimeError): + pass + + +class StoryArcManagedReorderService: + """Coordinate signed preview, durable preparation, and safe adjacent moves.""" + + async def preview_adjacent_move( + self, + session: AsyncSession, + story_arc_id: int, + membership_id: int, + *, + direction: StoryArcReorderDirection, + expected_revision: int, + ) -> StoryArcReorderPreview: + """Render a complete bounded plan without mutating the DB or filesystem.""" + plan = await self._build_plan( + session, + story_arc_id=story_arc_id, + membership_id=membership_id, + direction=direction, + expected_revision=expected_revision, + ) + return self._preview_from_plan(plan) + + async def confirm_adjacent_move( + self, + session: AsyncSession, + *, + story_arc_id: int, + membership_id: int, + direction: StoryArcReorderDirection, + expected_revision: int, + preview_token: str, + cancellation_requested: Callable[[], bool] | None = None, + ) -> StoryArcReorderResult: + """Confirm one signed plan, with restart-safe durable filesystem truth.""" + plan = self._decode_plan(preview_token) + _assert_plan_binding( + plan, + story_arc_id=story_arc_id, + membership_id=membership_id, + direction=direction, + expected_revision=expected_revision, + ) + try: + await self._verify_and_prepare(session, plan) + except StoryArcManagedReorderError as exc: + await session.rollback() + if not await self._has_active_journal(session, plan): + await session.rollback() + raise + # A recovered prepared operation can outlive its original browser + # request. If current DB truth no longer accepts that plan, end + # the inspection transaction before restoring its old paths. + await session.rollback() + await self._raise_after_semantic_failure(session, plan, exc) + try: + filesystem_result = await asyncio.to_thread( + _complete_filesystem_plan, + plan, + cancellation_requested, + ) + except _CancelledDuringReorderError as exc: + await self._record_rolled_back( + session, + plan, + status="rename_cancelled", + ) + raise StoryArcManagedReorderError( + "reorder_cancelled", + "Story Arc reorder was cancelled and its old paths were restored", + category="cancelled", + ) from exc + except (OSError, StoryArcPlacementError, StoryArcManagedReorderError) as exc: + rollback_complete = await asyncio.to_thread(_restore_old_paths, plan) + if rollback_complete: + await self._record_rolled_back(session, plan, status="rename_failed") + raise StoryArcManagedReorderError( + "reorder_failed", + "Story Arc reorder failed and its old paths were restored", + category="filesystem", + ) from exc + await self._record_recovery_required(session, plan, exc) + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Story Arc reorder needs recovery before another move can begin", + category="recovery", + ) from exc + + try: + revision = await self._reconcile_success( + session, + plan, + filesystem_result, + ) + except StoryArcManagedReorderError as exc: + await session.rollback() + # Files are already published. A semantic DB conflict cannot be + # retried forever (notably after an unrelated revision bump), so + # restore the old paths outside a transaction and retire the + # prepared journal. Incomplete restoration remains discoverable. + await self._raise_after_semantic_failure(session, plan, exc) + except Exception: + await session.rollback() + # The committed prepared journal remains authoritative. A retry + # with the same signed token observes the final files and completes + # the database reconciliation without moving them again. + raise + + return StoryArcReorderResult( + story_arc_id=plan.story_arc_id, + revision=revision, + managed_renamed=sum(item.action == "rename" for item in plan.placements), + # The result is scoped to the bounded adjacent-move plan. An + # affected referenced row remains preserved even when fresh + # reconciliation observes concurrent drift and skips it: this + # operation never mutates referenced placements or their files. + referenced_preserved=sum( + item.ownership == StoryArcPlacementOwnership.REFERENCED.value + for item in plan.placements + ), + ) + + def inspect_preview_token( + self, + *, + story_arc_id: int, + membership_id: int, + direction: StoryArcReorderDirection, + expected_revision: int, + preview_token: str, + recovery_pending: bool = False, + ) -> StoryArcReorderPreview: + """Rehydrate signed preview truth without touching the DB or filesystem.""" + plan = self._decode_plan(preview_token) + _assert_plan_binding( + plan, + story_arc_id=story_arc_id, + membership_id=membership_id, + direction=direction, + expected_revision=expected_revision, + ) + preview = self._preview_from_plan(plan) + return replace( + preview, + preview_token=preview_token, + recovery_pending=recovery_pending, + ) + + async def load_pending_preview( + self, + session: AsyncSession, + story_arc_id: int, + ) -> StoryArcReorderPreview | None: + """Discover one durable prepared reorder without a browser-held token.""" + coordinator = await session.scalar(_pending_coordinator_statement(story_arc_id)) + if coordinator is None: + return None + journal = dict(coordinator.last_result or {}) + recovery_token = journal.get("recovery_token") + if not isinstance(recovery_token, str): + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Prepared Story Arc reorder is missing its recovery payload", + category="recovery", + ) + plan = self._decode_durable_plan(recovery_token) + managed = [item for item in plan.placements if item.action == "rename"] + if ( + plan.story_arc_id != story_arc_id + or coordinator.operation_token != plan.operation_token + or journal.get("plan_digest") != plan.plan_digest + or journal.get("coordinator_placement_id") != coordinator.id + or not managed + or coordinator.id != min(item.placement_id for item in managed) + ): + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Prepared Story Arc reorder recovery payload does not match its journal", + category="recovery", + ) + managed_rows = { + row.id: row + for row in ( + await session.scalars( + select(StoryArcPlacement).where( + StoryArcPlacement.id.in_(tuple(item.placement_id for item in managed)) + ) + ) + ).all() + } + if len(managed_rows) != len(managed): + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Prepared Story Arc reorder journal is incomplete", + category="recovery", + ) + for item in managed: + row = managed_rows[item.placement_id] + row_journal = dict(row.last_result or {}) + if ( + row.operation_token != plan.operation_token + or row_journal.get("status") not in _ACTIVE_REORDER_STATUSES + or row_journal.get("operation") != "story_arc_reorder" + or row_journal.get("plan_digest") != plan.plan_digest + or row_journal.get("coordinator_placement_id") != coordinator.id + or row_journal.get("old_path") != item.old_path + or row_journal.get("new_path") != item.new_path + or row_journal.get("temporary_path") != item.temporary_path + ): + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Prepared Story Arc reorder journal changed before recovery", + category="recovery", + ) + return replace(self._preview_from_plan(plan), recovery_pending=True) + + async def _build_plan( + self, + session: AsyncSession, + *, + story_arc_id: int, + membership_id: int, + direction: StoryArcReorderDirection, + expected_revision: int, + ) -> _SignedPlan: + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise StoryArcManagedReorderError( + "story_arc_not_found", "Story arc was not found", category="not_found" + ) + if arc.lifecycle is StoryArcLifecycle.ARCHIVED: + raise StoryArcManagedReorderError( + "story_arc_archived", "Archived story arcs cannot be reordered" + ) + if arc.revision != expected_revision: + raise StoryArcManagedReorderError( + "revision_conflict", + "Story arc changed after the page was loaded", + category="conflict", + ) + selected = await session.scalar( + select(IssueStoryArc).where( + IssueStoryArc.id == membership_id, + IssueStoryArc.story_arc_id == story_arc_id, + ) + ) + if selected is None: + raise StoryArcManagedReorderError( + "membership_not_found", + "Story-arc membership was not found", + category="not_found", + ) + neighbour = await _load_adjacent_membership( + session, + selected=selected, + direction=direction, + ) + if neighbour is None: + edge = "already_first" if direction == "up" else "already_last" + raise StoryArcManagedReorderError(edge, f"Membership is {edge.replace('_', ' ')}") + + membership_plans = ( + _MembershipPlan( + membership_id=selected.id, + old_sequence_number=selected.sequence_number, + old_source_ordinal=selected.source_ordinal, + new_sequence_number=neighbour.sequence_number, + new_source_ordinal=neighbour.source_ordinal, + ), + _MembershipPlan( + membership_id=neighbour.id, + old_sequence_number=neighbour.sequence_number, + old_source_ordinal=neighbour.source_ordinal, + new_sequence_number=selected.sequence_number, + new_source_ordinal=selected.source_ordinal, + ), + ) + operation_token = uuid4().hex + policy = _policy_from_arc(arc) + placement_plans = await self._build_placement_plans( + session, + arc=arc, + policy_mode=policy.mode, + destination_root=policy.destination_root, + membership_plans=membership_plans, + ) + core: dict[str, object] = { + "story_arc_id": story_arc_id, + "selected_membership_id": membership_id, + "direction": direction, + "expected_revision": expected_revision, + "destination_root": policy.destination_root, + "memberships": [asdict(item) for item in membership_plans], + "placements": [asdict(item) for item in placement_plans], + } + digest = _plan_digest(core) + # Temp names are derived only after the semantic plan is frozen, so the + # digest cannot depend recursively on its own filename. + placement_plans = tuple( + replace( + item, + temporary_path=( + str( + Path(item.old_path).with_name( + ".pullbox-story-arc-reorder-" + f"{operation_token[:12]}-{item.placement_id}.tmp" + ) + ) + if item.action == "rename" + else None + ), + ) + for item in placement_plans + ) + _validate_complete_preview_paths( + placement_plans, + destination_root=policy.destination_root, + ) + await _validate_database_preview_paths(session, placement_plans) + return _SignedPlan( + story_arc_id=story_arc_id, + selected_membership_id=membership_id, + direction=direction, + expected_revision=expected_revision, + operation_token=operation_token, + plan_digest=digest, + destination_root=policy.destination_root, + memberships=membership_plans, + placements=placement_plans, + ) + + async def _build_placement_plans( + self, + session: AsyncSession, + *, + arc: StoryArc, + policy_mode: StoryArcPlacementPolicyMode, + destination_root: str | None, + membership_plans: tuple[_MembershipPlan, _MembershipPlan], + ) -> tuple[_PlacementPlan, ...]: + policy = _policy_from_arc(arc) + contexts = { + item.membership_id: await _load_one_context(session, arc.id, item.membership_id) + for item in membership_plans + } + membership_ids = tuple(contexts) + rows = list( + ( + await session.scalars( + select(StoryArcPlacement) + .where(StoryArcPlacement.issue_story_arc_id.in_(membership_ids)) + .order_by(StoryArcPlacement.id.asc()) + .limit(_MAX_PLACEMENTS_PER_ADJACENT_MOVE + 1) + ) + ).all() + ) + if len(rows) > _MAX_PLACEMENTS_PER_ADJACENT_MOVE: + raise StoryArcManagedReorderError( + "placement_limit", + "Adjacent reorder exceeds the bounded placement safety limit", + category="safety", + ) + by_membership: dict[int, list[StoryArcPlacement]] = { + membership_id: [] for membership_id in membership_ids + } + for row in rows: + by_membership[row.issue_story_arc_id].append(row) + + plans: list[_PlacementPlan] = [] + for membership in membership_plans: + context = contexts[membership.membership_id] + placement_rows = by_membership[membership.membership_id] + if not placement_rows: + plans.append( + _PlacementPlan( + placement_id=0, + membership_id=membership.membership_id, + ownership="logical", + mode="logical", + old_reading_order=membership.old_sequence_number, + new_reading_order=membership.new_sequence_number, + old_path="", + new_path="", + temporary_path=None, + rendered_path_after="", + action="logical_reorder", + canonical_path=None, + source_fingerprint={}, + target_fingerprint={}, + placement_state=None, + last_result_snapshot={}, + ) + ) + continue + try: + old_rendered = ( + _rendered_target_path(context, policy) + if policy_mode is not StoryArcPlacementPolicyMode.LOGICAL + and destination_root is not None + else None + ) + new_context = replace( + context, + sequence_number=membership.new_sequence_number, + ) + new_rendered = ( + _rendered_target_path(new_context, policy) + if policy_mode is not StoryArcPlacementPolicyMode.LOGICAL + and destination_root is not None + else None + ) + except StoryArcOriginalFilenameError as exc: + raise StoryArcManagedReorderError( + "original_filename_unsafe", str(exc), category="safety" + ) from exc + for row in placement_rows: + plans.append( + _placement_plan_from_row( + row, + membership=membership, + old_rendered=old_rendered, + new_rendered=new_rendered, + canonical_path=context.canonical_path, + policy_mode=policy_mode, + arc_policy_schema_version=arc.policy_schema_version, + ) + ) + return tuple(plans) + + def _preview_from_plan(self, plan: _SignedPlan) -> StoryArcReorderPreview: + token = self._serializer().dumps(_plan_to_payload(plan)) + items = tuple( + StoryArcReorderPreviewItem( + membership_id=item.membership_id, + placement_id=item.placement_id or None, + ownership=item.ownership, + mode=item.mode, + old_reading_order=item.old_reading_order, + new_reading_order=item.new_reading_order, + old_path=item.old_path or None, + new_path=item.new_path or None, + rendered_path_after=item.rendered_path_after or None, + temporary_path=item.temporary_path, + action=item.action, + ) + for item in plan.placements + ) + return StoryArcReorderPreview( + story_arc_id=plan.story_arc_id, + membership_id=plan.selected_membership_id, + direction=plan.direction, + expected_revision=plan.expected_revision, + preview_token=token, + items=items, + managed_rename_count=sum(item.action == "rename" for item in items), + referenced_drift_count=sum(item.action == "referenced_drift" for item in items), + referenced_preserved_count=sum( + item.ownership == StoryArcPlacementOwnership.REFERENCED.value for item in items + ), + recovery_pending=False, + ) + + async def _verify_and_prepare( + self, + session: AsyncSession, + plan: _SignedPlan, + ) -> None: + arc = await session.get(StoryArc, plan.story_arc_id) + if arc is None: + raise StoryArcManagedReorderError( + "story_arc_not_found", "Story arc was not found", category="not_found" + ) + if arc.revision != plan.expected_revision: + raise StoryArcManagedReorderError( + "revision_conflict", + "Story arc changed after the preview was generated", + category="conflict", + ) + memberships = { + row.id: row + for row in ( + await session.scalars( + select(IssueStoryArc).where( + IssueStoryArc.id.in_( + tuple(item.membership_id for item in plan.memberships) + ), + IssueStoryArc.story_arc_id == plan.story_arc_id, + ) + ) + ).all() + } + if len(memberships) != len(plan.memberships): + raise StoryArcManagedReorderError( + "membership_not_found", + "A previewed Story Arc membership no longer exists", + category="not_found", + ) + for expected in plan.memberships: + current = memberships[expected.membership_id] + if ( + current.sequence_number != expected.old_sequence_number + or current.source_ordinal != expected.old_source_ordinal + ): + raise StoryArcManagedReorderError( + "revision_conflict", + "Story Arc order changed after the preview was generated", + category="conflict", + ) + + persisted_plans = [item for item in plan.placements if item.placement_id > 0] + managed_plans = [item for item in persisted_plans if item.action == "rename"] + placement_ids = tuple(item.placement_id for item in persisted_plans) + placements = { + row.id: row + for row in ( + await session.scalars( + select(StoryArcPlacement).where(StoryArcPlacement.id.in_(placement_ids)) + ) + ).all() + } + if len(placements) != len(persisted_plans): + raise StoryArcManagedReorderError( + "placement_changed", + "A Story Arc placement no longer matches the preview", + category="ownership", + ) + for placement_plan in persisted_plans: + row = placements[placement_plan.placement_id] + if placement_plan.action == "rename": + continue + _validate_nonrenamed_row_matches_plan(row, placement_plan) + if placement_plan.action == "managed_unchanged": + _validate_artifact_at_old_path(placement_plan) + + if not managed_plans: + await session.rollback() + return + already_prepared = True + for placement_plan in managed_plans: + row = placements[placement_plan.placement_id] + journal = dict(row.last_result or {}) + is_same_prepared = ( + row.operation_token == plan.operation_token + and journal.get("status") in _ACTIVE_REORDER_STATUSES + and journal.get("plan_digest") == plan.plan_digest + and journal.get("operation_token") == plan.operation_token + and journal.get("old_path") == placement_plan.old_path + and journal.get("new_path") == placement_plan.new_path + and journal.get("temporary_path") == placement_plan.temporary_path + and journal.get("old_reading_order") == placement_plan.old_reading_order + and journal.get("new_reading_order") == placement_plan.new_reading_order + and journal.get("target_fingerprint") == placement_plan.target_fingerprint + and row.issue_story_arc_id == placement_plan.membership_id + and row.ownership is StoryArcPlacementOwnership.MANAGED + and row.mode.value == placement_plan.mode + and _normal_path(row.placement_path) == _normal_path(placement_plan.old_path) + ) + if not is_same_prepared: + already_prepared = False + _validate_managed_row_matches_plan(row, placement_plan) + _validate_artifact_at_old_path(placement_plan) + + if already_prepared: + # End the verification transaction before resuming any filesystem + # work from the durable prepared journal. + await session.rollback() + return + if any(placements[item.placement_id].operation_token for item in managed_plans): + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "A managed placement has an unfinished operation", + category="recovery", + ) + + now = datetime.now(UTC) + coordinator_placement_id = min(item.placement_id for item in managed_plans) + recovery_token = self._serializer().dumps(_plan_to_payload(plan)) + for placement_plan in managed_plans: + row = placements[placement_plan.placement_id] + previous = dict(row.last_result or {}) + journal_payload: dict[str, object] = { + "schema_version": _JOURNAL_SCHEMA_VERSION, + "status": "rename_prepared", + "operation": "story_arc_reorder", + "operation_token": plan.operation_token, + "plan_digest": plan.plan_digest, + "coordinator": placement_plan.placement_id == coordinator_placement_id, + "coordinator_placement_id": coordinator_placement_id, + "old_path": placement_plan.old_path, + "new_path": placement_plan.new_path, + "temporary_path": placement_plan.temporary_path, + "old_reading_order": placement_plan.old_reading_order, + "new_reading_order": placement_plan.new_reading_order, + "target_fingerprint": dict(placement_plan.target_fingerprint), + "previous_result": previous, + } + if placement_plan.placement_id == coordinator_placement_id: + # The browser-held confirmation token is not restart truth. + # Keep one signed, bounded coordinator payload in the durable + # journal so a fresh process can discover and reissue it. + journal_payload["recovery_token"] = recovery_token + result = await session.execute( + sa_update(StoryArcPlacement) + .where( + StoryArcPlacement.id == placement_plan.placement_id, + StoryArcPlacement.issue_story_arc_id == placement_plan.membership_id, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.MANAGED, + StoryArcPlacement.state == StoryArcPlacementState.CURRENT, + StoryArcPlacement.operation_token.is_(None), + StoryArcPlacement.placement_path == placement_plan.old_path, + StoryArcPlacement.rendered_reading_order == placement_plan.old_reading_order, + ) + .values( + operation_token=plan.operation_token, + last_checked_at=now, + last_result=journal_payload, + ) + .execution_options(synchronize_session=False) + ) + if result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcManagedReorderError( + "reorder_operation_superseded", + "Managed placement was reserved by another operation", + category="conflict", + ) + await session.commit() + + async def _reconcile_success( + self, + session: AsyncSession, + plan: _SignedPlan, + filesystem_result: _FilesystemResult, + ) -> int: + arc = await session.get(StoryArc, plan.story_arc_id) + if arc is None or arc.revision != plan.expected_revision: + raise StoryArcManagedReorderError( + "revision_conflict", + "Story Arc changed before filesystem reconciliation", + category="conflict", + ) + placement_ids = tuple( + item.placement_id for item in plan.placements if item.placement_id > 0 + ) + placements = { + row.id: row + for row in ( + await session.scalars( + select(StoryArcPlacement).where(StoryArcPlacement.id.in_(placement_ids)) + ) + ).all() + } + changed_nonrenamed: set[int] = set() + for item in plan.placements: + if item.placement_id == 0: + continue + row = placements.get(item.placement_id) + if row is None: + if item.action == "rename": + raise StoryArcManagedReorderError( + "managed_placement_changed", + "A managed placement disappeared before reconciliation", + category="conflict", + ) + changed_nonrenamed.add(item.placement_id) + continue + if item.action == "rename": + journal = dict(row.last_result or {}) + if ( + row.operation_token != plan.operation_token + or journal.get("status") not in _ACTIVE_REORDER_STATUSES + or journal.get("plan_digest") != plan.plan_digest + or journal.get("operation_token") != plan.operation_token + or journal.get("old_path") != item.old_path + or journal.get("new_path") != item.new_path + or journal.get("temporary_path") != item.temporary_path + or _normal_path(row.placement_path) != _normal_path(item.old_path) + ): + raise StoryArcManagedReorderError( + "reorder_operation_superseded", + "Managed placement operation was superseded", + category="conflict", + ) + else: + try: + _validate_nonrenamed_row_matches_plan(row, item) + except StoryArcManagedReorderError: + # The reorder never owns referenced or path-unchanged rows. + # Preserve a concurrent inspection/repair/delete verbatim; + # its stale rendered-order evidence remains truthful drift + # for normal synchronization to revisit. + changed_nonrenamed.add(item.placement_id) + + # Break database path uniqueness inside the same post-filesystem + # transaction, then publish the actual final paths before commit. + for item in plan.placements: + if item.action == "rename": + row = placements[item.placement_id] + if item.temporary_path is None: + raise StoryArcManagedReorderError( + "invalid_preview_token", "Reorder temporary path is missing" + ) + row.placement_path = item.temporary_path + await session.flush() + + now = datetime.now(UTC) + for item in plan.placements: + if item.placement_id == 0: + continue + if item.placement_id in changed_nonrenamed: + continue + row = placements[item.placement_id] + if item.action == "rename": + row.placement_path = item.new_path + row.rendered_reading_order = item.new_reading_order + row.policy_schema_version = STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION + row.operation_token = None + row.state = StoryArcPlacementState.CURRENT + row.last_checked_at = now + row.last_result = { + "schema_version": _JOURNAL_SCHEMA_VERSION, + "status": "complete", + "outcome": "reordered", + "plan_digest": plan.plan_digest, + "target_fingerprint": dict(filesystem_result.fingerprints[item.placement_id]), + } + elif item.action == "managed_unchanged": + row.rendered_reading_order = item.new_reading_order + row.state = StoryArcPlacementState.CURRENT + row.last_checked_at = now + row.last_result = { + "schema_version": _JOURNAL_SCHEMA_VERSION, + "status": "complete", + "outcome": "reading_order_updated", + "plan_digest": plan.plan_digest, + "target_fingerprint": dict(item.target_fingerprint), + } + elif item.action in {"referenced_drift", "referenced_unchanged"}: + row.rendered_reading_order = item.new_reading_order + if item.action == "referenced_drift": + row.state = StoryArcPlacementState.DRIFTED + prior = dict(row.last_result or {}) + row.last_result = { + **prior, + "schema_version": _JOURNAL_SCHEMA_VERSION, + "status": "referenced_preserved", + "desired_path": item.rendered_path_after, + "artifact_mutated": False, + } + + memberships = { + row.id: row + for row in ( + await session.scalars( + select(IssueStoryArc).where( + IssueStoryArc.id.in_(tuple(item.membership_id for item in plan.memberships)) + ) + ) + ).all() + } + for membership_plan in plan.memberships: + membership = memberships[membership_plan.membership_id] + membership.sequence_number = membership_plan.new_sequence_number + membership.source_ordinal = membership_plan.new_source_ordinal + arc.revision += 1 + await session.commit() + return arc.revision + + async def _has_active_journal( + self, + session: AsyncSession, + plan: _SignedPlan, + ) -> bool: + row = await session.scalar( + select(StoryArcPlacement) + .where(StoryArcPlacement.operation_token == plan.operation_token) + .order_by(StoryArcPlacement.id) + .limit(1) + ) + if row is None: + return False + journal = dict(row.last_result or {}) + return ( + journal.get("status") in _ACTIVE_REORDER_STATUSES + and journal.get("operation") == "story_arc_reorder" + and journal.get("operation_token") == plan.operation_token + and journal.get("plan_digest") == plan.plan_digest + ) + + async def _raise_after_semantic_failure( + self, + session: AsyncSession, + plan: _SignedPlan, + exc: StoryArcManagedReorderError, + ) -> NoReturn: + restored = await asyncio.to_thread(_restore_old_paths, plan) + if restored: + await self._record_rolled_back(session, plan, status="rename_failed") + raise StoryArcManagedReorderError( + exc.code, + f"{exc}; managed placement paths were restored", + category=exc.category, + ) from exc + await self._record_recovery_required(session, plan, exc) + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Story Arc reorder needs recovery before another move can begin", + category="recovery", + ) from exc + + async def _record_rolled_back( + self, + session: AsyncSession, + plan: _SignedPlan, + *, + status: Literal["rename_cancelled", "rename_failed"], + ) -> None: + rows = list( + ( + await session.scalars( + select(StoryArcPlacement).where( + StoryArcPlacement.operation_token == plan.operation_token + ) + ) + ).all() + ) + now = datetime.now(UTC) + by_id = {item.placement_id: item for item in plan.placements} + for row in rows: + expected = by_id.get(row.id) + if expected is None: + continue + row.operation_token = None + row.state = StoryArcPlacementState.CURRENT + row.last_checked_at = now + row.last_result = { + "schema_version": _JOURNAL_SCHEMA_VERSION, + "status": status, + "plan_digest": plan.plan_digest, + "target_fingerprint": dict(expected.target_fingerprint), + "retryable": True, + } + await session.commit() + + async def _record_recovery_required( + self, + session: AsyncSession, + plan: _SignedPlan, + exc: BaseException, + ) -> None: + rows = list( + ( + await session.scalars( + select(StoryArcPlacement).where( + StoryArcPlacement.operation_token == plan.operation_token + ) + ) + ).all() + ) + for row in rows: + previous = dict(row.last_result or {}) + row.state = StoryArcPlacementState.DRIFTED + row.last_checked_at = datetime.now(UTC) + row.last_result = { + **previous, + "schema_version": _JOURNAL_SCHEMA_VERSION, + "status": "rename_recovery_required", + "plan_digest": plan.plan_digest, + "error_type": type(exc).__name__, + "retry_with_same_preview": True, + } + await session.commit() + + def _decode_plan(self, token: str) -> _SignedPlan: + if not token: + raise StoryArcManagedReorderError( + "confirmation_required", "A reorder preview must be confirmed" + ) + try: + raw = self._serializer().loads(token, max_age=_TOKEN_MAX_AGE_SECONDS) + except SignatureExpired as exc: + raise StoryArcManagedReorderError( + "preview_expired", "The reorder preview expired; generate a new preview" + ) from exc + except BadSignature as exc: + raise StoryArcManagedReorderError( + "invalid_preview_token", "The reorder preview token is invalid" + ) from exc + try: + return _plan_from_payload(raw) + except (KeyError, TypeError, ValueError) as exc: + raise StoryArcManagedReorderError( + "invalid_preview_token", "The reorder preview token is invalid" + ) from exc + + def _decode_durable_plan(self, token: str) -> _SignedPlan: + """Decode a DB-held signed recovery token without browser TTL expiry.""" + try: + raw = self._serializer().loads(token) + except BadSignature as exc: + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Prepared Story Arc reorder recovery signature is invalid", + category="recovery", + ) from exc + try: + return _plan_from_payload(raw) + except (KeyError, TypeError, ValueError) as exc: + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Prepared Story Arc reorder recovery payload is invalid", + category="recovery", + ) from exc + + @staticmethod + def _serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(get_application_secret(), salt=_TOKEN_SALT) + + +def _pending_coordinator_statement( + story_arc_id: int, +) -> Select[tuple[StoryArcPlacement]]: + """Return the bounded, portable durable-reorder discovery query.""" + return ( + select(StoryArcPlacement) + .join( + IssueStoryArc, + StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id, + ) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + StoryArcPlacement.operation_token.is_not(None), + StoryArcPlacement.last_result["operation"].as_string() == "story_arc_reorder", + StoryArcPlacement.last_result["coordinator"].as_boolean().is_(True), + ) + .order_by(StoryArcPlacement.id) + .limit(1) + ) + + +async def _load_adjacent_membership( + session: AsyncSession, + *, + selected: IssueStoryArc, + direction: StoryArcReorderDirection, +) -> IssueStoryArc | None: + key = ( + IssueStoryArc.sequence_number, + IssueStoryArc.source_ordinal, + IssueStoryArc.id, + ) + before = or_( + IssueStoryArc.sequence_number < selected.sequence_number, + and_( + IssueStoryArc.sequence_number == selected.sequence_number, + IssueStoryArc.source_ordinal < selected.source_ordinal, + ), + and_( + IssueStoryArc.sequence_number == selected.sequence_number, + IssueStoryArc.source_ordinal == selected.source_ordinal, + IssueStoryArc.id < selected.id, + ), + ) + after = or_( + IssueStoryArc.sequence_number > selected.sequence_number, + and_( + IssueStoryArc.sequence_number == selected.sequence_number, + IssueStoryArc.source_ordinal > selected.source_ordinal, + ), + and_( + IssueStoryArc.sequence_number == selected.sequence_number, + IssueStoryArc.source_ordinal == selected.source_ordinal, + IssueStoryArc.id > selected.id, + ), + ) + statement = select(IssueStoryArc).where( + IssueStoryArc.story_arc_id == selected.story_arc_id, + before if direction == "up" else after, + ) + statement = statement.order_by( + *(column.desc() for column in key) + if direction == "up" + else (column.asc() for column in key) + ) + return cast("IssueStoryArc | None", await session.scalar(statement.limit(1))) + + +def _placement_plan_from_row( + row: StoryArcPlacement, + *, + membership: _MembershipPlan, + old_rendered: str | None, + new_rendered: str | None, + canonical_path: str | None, + policy_mode: StoryArcPlacementPolicyMode, + arc_policy_schema_version: int | None, +) -> _PlacementPlan: + ownership = row.ownership.value + if row.ownership is StoryArcPlacementOwnership.REFERENCED: + desired = new_rendered or row.placement_path + action: StoryArcReorderAction = ( + "referenced_unchanged" + if os.path.normcase(os.path.abspath(row.placement_path)) + == os.path.normcase(os.path.abspath(desired)) + else "referenced_drift" + ) + return _PlacementPlan( + placement_id=row.id, + membership_id=row.issue_story_arc_id, + ownership=ownership, + mode=row.mode.value, + old_reading_order=membership.old_sequence_number, + new_reading_order=membership.new_sequence_number, + old_path=row.placement_path, + new_path=row.placement_path, + temporary_path=None, + rendered_path_after=desired, + action=action, + canonical_path=canonical_path, + source_fingerprint=dict(row.source_fingerprint or {}), + target_fingerprint=_stored_target_fingerprint(row), + placement_state=row.state.value, + last_result_snapshot=dict(row.last_result or {}), + ) + + if old_rendered is None or new_rendered is None: + raise StoryArcManagedReorderError( + "managed_policy_missing", + "Managed placement has no active rendered policy", + category="ownership", + ) + if ( + row.mode.value != policy_mode.value + or row.policy_schema_version != arc_policy_schema_version + or row.rendered_reading_order != membership.old_sequence_number + or _normal_path(row.placement_path) != _normal_path(old_rendered) + ): + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement no longer matches its rendered policy", + category="ownership", + ) + target_fingerprint = _stored_target_fingerprint(row) + if ( + row.state is not StoryArcPlacementState.CURRENT + or row.operation_token is not None + or not row.source_fingerprint + or not target_fingerprint + or dict(row.last_result or {}).get("status") not in _UNCHANGED_MANAGED_STATUSES + ): + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement lacks unchanged action-owned evidence", + category="ownership", + ) + action = ( + "managed_unchanged" + if _normal_path(old_rendered) == _normal_path(new_rendered) + else "rename" + ) + _validate_artifact_fingerprint( + Path(row.placement_path), + target_fingerprint, + canonical_path=Path(canonical_path) if canonical_path else None, + mode=row.mode, + ) + return _PlacementPlan( + placement_id=row.id, + membership_id=row.issue_story_arc_id, + ownership=ownership, + mode=row.mode.value, + old_reading_order=membership.old_sequence_number, + new_reading_order=membership.new_sequence_number, + old_path=row.placement_path, + new_path=new_rendered, + temporary_path=None, + rendered_path_after=new_rendered, + action=action, + canonical_path=canonical_path, + source_fingerprint=dict(row.source_fingerprint), + target_fingerprint=target_fingerprint, + placement_state=row.state.value, + last_result_snapshot=dict(row.last_result or {}), + ) + + +def _validate_complete_preview_paths( + plans: Sequence[_PlacementPlan], + *, + destination_root: str | None, +) -> None: + managed = [item for item in plans if item.action == "rename"] + if not managed: + return + if destination_root is None: + raise StoryArcManagedReorderError( + "managed_policy_missing", "Managed reorder destination root is missing" + ) + root = Path(destination_root) + old_paths = {_normal_path(item.old_path) for item in managed} + new_paths: set[str] = set() + canonical_paths = { + _normal_path(item.canonical_path) for item in managed if item.canonical_path is not None + } + if len(managed) > _MAX_COLLISION_SCAN_PATHS: + raise StoryArcManagedReorderError( + "placement_limit", "Reorder collision scan exceeds its bounded limit" + ) + for item in managed: + old = Path(item.old_path) + new = Path(item.new_path) + _validate_target_lexically(root, old) + _validate_target_lexically(root, new) + _validate_path_limits(old) + _validate_path_limits(new) + if _normal_path(old) in canonical_paths or _normal_path(new) in canonical_paths: + raise StoryArcManagedReorderError( + "canonical_destination", + "Reorder cannot mutate a canonical library artifact", + category="safety", + ) + if item.canonical_path is not None: + _reject_canonical_destination(Path(item.canonical_path), old) + _reject_canonical_destination(Path(item.canonical_path), new) + if item.temporary_path is not None: + temporary = Path(item.temporary_path) + _validate_target_lexically(root, temporary) + _validate_path_limits(temporary) + if item.canonical_path is not None: + _reject_canonical_destination(Path(item.canonical_path), temporary) + if _path_exists(temporary) or _case_only_collision(temporary) is not None: + raise StoryArcManagedReorderError( + "temporary_collision", + "A reorder recovery checkpoint path is already occupied", + category="collision", + ) + normalized_new = _normal_path(new) + if normalized_new in new_paths: + raise StoryArcManagedReorderError( + "destination_collision", + "Two managed placements render to the same destination", + category="collision", + ) + new_paths.add(normalized_new) + if normalized_new not in old_paths and _path_exists(new): + raise StoryArcManagedReorderError( + "destination_collision", + "A rendered reorder destination already exists", + category="collision", + ) + case_collision = _case_only_collision(new) + if case_collision is not None and _normal_path(case_collision) not in old_paths: + raise StoryArcManagedReorderError( + "case_only_collision", + "A case-only reorder destination collision exists", + category="collision", + ) + + +async def _validate_database_preview_paths( + session: AsyncSession, + plans: Sequence[_PlacementPlan], +) -> None: + """Reject canonical or independently tracked paths with two bounded queries.""" + managed = [item for item in plans if item.action == "rename"] + if not managed: + return + candidate_paths = tuple( + dict.fromkeys( + raw_path + for item in managed + for raw_path in (item.old_path, item.new_path, item.temporary_path) + if raw_path is not None + ) + ) + canonical = await session.scalar( + select(LibraryFile.file_path).where(LibraryFile.file_path.in_(candidate_paths)).limit(1) + ) + if canonical is not None: + raise StoryArcManagedReorderError( + "canonical_destination", + "Reorder cannot mutate a canonical library artifact", + category="safety", + ) + planned_ids = {item.placement_id for item in managed} + occupied_rows = ( + await session.execute( + select(StoryArcPlacement.id, StoryArcPlacement.placement_path).where( + StoryArcPlacement.placement_path.in_( + tuple( + raw_path + for item in managed + for raw_path in (item.new_path, item.temporary_path) + if raw_path is not None + ) + ) + ) + ) + ).all() + temporary_paths = {item.temporary_path for item in managed if item.temporary_path is not None} + if any( + row_path in temporary_paths or row_id not in planned_ids + for row_id, row_path in occupied_rows + ): + raise StoryArcManagedReorderError( + "destination_collision", + "A rendered reorder destination belongs to another placement", + category="collision", + ) + + +def _validate_managed_row_matches_plan( + row: StoryArcPlacement, + expected: _PlacementPlan, +) -> None: + if ( + row.issue_story_arc_id != expected.membership_id + or row.ownership is not StoryArcPlacementOwnership.MANAGED + or row.mode.value != expected.mode + or row.state is not StoryArcPlacementState.CURRENT + or row.operation_token is not None + or row.rendered_reading_order != expected.old_reading_order + or _normal_path(row.placement_path) != _normal_path(expected.old_path) + or dict(row.source_fingerprint or {}) != expected.source_fingerprint + or _stored_target_fingerprint(row) != expected.target_fingerprint + or dict(row.last_result or {}).get("status") not in _UNCHANGED_MANAGED_STATUSES + ): + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement no longer matches its action-owned preview evidence", + category="ownership", + ) + + +def _validate_nonrenamed_row_matches_plan( + row: StoryArcPlacement, + expected: _PlacementPlan, +) -> None: + if ( + row.issue_story_arc_id != expected.membership_id + or row.ownership.value != expected.ownership + or row.mode.value != expected.mode + or row.operation_token is not None + or _normal_path(row.placement_path) != _normal_path(expected.old_path) + or dict(row.source_fingerprint or {}) != expected.source_fingerprint + or _stored_target_fingerprint(row) != expected.target_fingerprint + or row.state.value != expected.placement_state + or dict(row.last_result or {}) != expected.last_result_snapshot + ): + raise StoryArcManagedReorderError( + "placement_changed", + "Story Arc placement changed after the preview was generated", + category="ownership", + ) + if expected.action == "managed_unchanged" and ( + row.ownership is not StoryArcPlacementOwnership.MANAGED + or row.state is not StoryArcPlacementState.CURRENT + or row.rendered_reading_order != expected.old_reading_order + or dict(row.last_result or {}).get("status") not in _UNCHANGED_MANAGED_STATUSES + ): + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement lacks unchanged action-owned evidence", + category="ownership", + ) + + +def _validate_artifact_at_old_path(expected: _PlacementPlan) -> None: + _validate_artifact_fingerprint( + Path(expected.old_path), + expected.target_fingerprint, + canonical_path=( + Path(expected.canonical_path) if expected.canonical_path is not None else None + ), + mode=StoryArcPlacementMode(expected.mode), + ) + + +def _validate_artifact_fingerprint( + path: Path, + expected: Fingerprint, + *, + canonical_path: Path | None, + mode: StoryArcPlacementMode, +) -> None: + try: + if not _path_exists(path): + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement is missing from its previewed path", + category="ownership", + ) + _validate_removal_representation(path, mode=mode) + actual = _fingerprint_target(path) + except StoryArcManagedReorderError: + raise + except (OSError, StoryArcPlacementError) as exc: + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement could not be validated safely", + category="ownership", + ) from exc + if actual != expected: + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement changed after its ownership evidence was recorded", + category="ownership", + ) + if canonical_path is not None: + _reject_canonical_destination(canonical_path, path) + + +def _validate_removal_representation(path: Path, *, mode: StoryArcPlacementMode) -> None: + root_guard = _validated_root(_common_destination_root(path)) + with _open_secure_parent_directory(root_guard, path.parent, create=False) as parent: + _validate_removal_representation_at(mode, parent, path.name) + + +def _common_destination_root(path: Path) -> Path: + # This helper is used only for a read-only type check before the signed plan + # has its policy root attached. Opening the immediate real parent still + # rejects a symlink parent and pins the exact entry for fingerprinting. + return path.parent + + +def _complete_filesystem_plan( + plan: _SignedPlan, + cancellation_requested: Callable[[], bool] | None, +) -> _FilesystemResult: + managed = [item for item in plan.placements if item.action == "rename"] + if not managed: + return _FilesystemResult(fingerprints={}) + if plan.destination_root is None: + raise StoryArcManagedReorderError( + "managed_policy_missing", "Managed reorder destination root is missing" + ) + _validate_plan_candidates(plan, managed) + try: + if cancellation_requested is not None and cancellation_requested(): + raise _CancelledDuringReorderError + locations = _locate_managed_artifacts( + managed, + root=Path(plan.destination_root), + prefer="new", + ) + for item in managed: + if cancellation_requested is not None and cancellation_requested(): + raise _CancelledDuringReorderError + location = locations[item.placement_id] + if location == item.old_path: + if item.temporary_path is None: + raise StoryArcManagedReorderError( + "invalid_preview_token", "Reorder temporary path is missing" + ) + _exclusive_move( + root=Path(plan.destination_root), + source=Path(item.old_path), + destination=Path(item.temporary_path), + expected=item.target_fingerprint, + canonical_path=( + Path(item.canonical_path) if item.canonical_path is not None else None + ), + mode=StoryArcPlacementMode(item.mode), + create_destination_parent=False, + ) + locations[item.placement_id] = item.temporary_path + + if cancellation_requested is not None and cancellation_requested(): + raise _CancelledDuringReorderError + + for item in managed: + if cancellation_requested is not None and cancellation_requested(): + raise _CancelledDuringReorderError + location = locations[item.placement_id] + if location == item.new_path: + continue + if location != item.temporary_path or item.temporary_path is None: + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Managed reorder artifact has an unexpected recovery location", + category="recovery", + ) + _exclusive_move( + root=Path(plan.destination_root), + source=Path(item.temporary_path), + destination=Path(item.new_path), + expected=item.target_fingerprint, + canonical_path=( + Path(item.canonical_path) if item.canonical_path is not None else None + ), + mode=StoryArcPlacementMode(item.mode), + create_destination_parent=True, + ) + locations[item.placement_id] = item.new_path + except _CancelledDuringReorderError as exc: + if not _restore_old_paths(plan): + raise StoryArcManagedReorderError( + "reorder_recovery_required", + "Cancelled reorder could not restore every old placement path", + category="recovery", + ) from exc + raise + + final_fingerprints: dict[int, Fingerprint] = {} + for item in managed: + final_fingerprint = _fingerprint_target(Path(item.new_path)) + if final_fingerprint != item.target_fingerprint: + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement changed before reorder reconciliation", + category="recovery", + ) + final_fingerprints[item.placement_id] = final_fingerprint + return _FilesystemResult(fingerprints=final_fingerprints) + + +def _validate_plan_candidates(plan: _SignedPlan, managed: Sequence[_PlacementPlan]) -> None: + if not managed: + return + if plan.destination_root is None: + raise StoryArcManagedReorderError( + "managed_policy_missing", "Managed reorder destination root is missing" + ) + root = Path(plan.destination_root) + root_guard = _validated_root(root) + del root_guard + candidate_paths: list[Path] = [] + for item in managed: + if item.temporary_path is None: + raise StoryArcManagedReorderError( + "invalid_preview_token", "Reorder temporary path is missing" + ) + for raw_path in (item.old_path, item.temporary_path, item.new_path): + path = Path(raw_path) + _validate_target_lexically(root, path) + _validate_path_limits(path) + candidate_paths.append(path) + if item.canonical_path is not None: + _reject_canonical_destination(Path(item.canonical_path), path) + normalized = [_normal_path(path) for path in candidate_paths] + if len(normalized) != len(set(normalized)): + # Old/new overlap between distinct placements is valid; a temp path is + # never allowed to overlap anything else. + temporary = {_normal_path(cast("str", item.temporary_path)) for item in managed} + non_temporary = { + _normal_path(path) for item in managed for path in (item.old_path, item.new_path) + } + if temporary & non_temporary or len(temporary) != len(managed): + raise StoryArcManagedReorderError( + "invalid_preview_token", "Reorder paths overlap unsafely" + ) + + +def _locate_managed_artifacts( + plans: Sequence[_PlacementPlan], + *, + root: Path, + prefer: Literal["new", "old"], +) -> dict[int, str]: + candidate_paths = { + path + for item in plans + for path in (item.old_path, item.temporary_path, item.new_path) + if path is not None + } + fingerprints: dict[str, Fingerprint] = {} + for raw_path in candidate_paths: + path = Path(raw_path) + if _path_exists(path): + fingerprints[raw_path] = _fingerprint_target(path) + locations: dict[int, str] = {} + claimed: set[str] = set() + for item in plans: + candidates = [ + raw_path + for raw_path in (item.old_path, item.temporary_path, item.new_path) + if raw_path is not None and fingerprints.get(raw_path) == item.target_fingerprint + ] + if not candidates: + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement is missing or changed during recovery", + category="recovery", + ) + preference = ( + (item.new_path, item.temporary_path, item.old_path) + if prefer == "new" + else (item.old_path, item.temporary_path, item.new_path) + ) + selected = next(path for path in preference if path in candidates) + if selected in claimed: + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Two managed placements claim the same recovery artifact", + category="recovery", + ) + # ``_exclusive_move`` publishes by an exclusive hard-link followed by + # unlink. A process death between those syscalls leaves two names for + # the exact recorded inode. The signed plan and exact fingerprint make + # it safe to finish that interrupted step by keeping the furthest-safe + # name for the requested direction and unlinking only its duplicates. + for duplicate in candidates: + if duplicate == selected: + continue + _unlink_exact_duplicate( + root=root, + path=Path(duplicate), + expected=item.target_fingerprint, + canonical_path=( + Path(item.canonical_path) if item.canonical_path is not None else None + ), + mode=StoryArcPlacementMode(item.mode), + ) + fingerprints.pop(duplicate, None) + locations[item.placement_id] = selected + claimed.add(selected) + return locations + + +def _unlink_exact_duplicate( + *, + root: Path, + path: Path, + expected: Fingerprint, + canonical_path: Path | None, + mode: StoryArcPlacementMode, +) -> None: + root_guard = _validated_root(root) + _validate_target_lexically(root, path) + if canonical_path is not None: + _reject_canonical_destination(canonical_path, path) + with _open_secure_parent_directory( + root_guard, + path.parent, + create=False, + ) as parent: + _validate_removal_representation_at(mode, parent, path.name) + actual = _fingerprint_target_at( + parent, + path.name, + canonical_path=canonical_path, + ) + if actual != expected: + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Duplicate recovery artifact changed before reconciliation", + category="recovery", + ) + os.unlink(path.name, dir_fd=parent.parent_fd) + _fsync_directory(parent.parent_fd) + + +def _exclusive_move( + *, + root: Path, + source: Path, + destination: Path, + expected: Fingerprint, + canonical_path: Path | None, + mode: StoryArcPlacementMode, + create_destination_parent: bool, +) -> None: + root_guard = _validated_root(root) + _validate_target_lexically(root, source) + _validate_target_lexically(root, destination) + if canonical_path is not None: + _reject_canonical_destination(canonical_path, source) + _reject_canonical_destination(canonical_path, destination) + with _open_secure_parent_directory(root_guard, source.parent, create=False) as source_parent: + _validate_removal_representation_at(mode, source_parent, source.name) + actual = _fingerprint_target_at( + source_parent, + source.name, + canonical_path=canonical_path, + ) + if actual != expected: + raise StoryArcManagedReorderError( + "managed_placement_changed", + "Managed placement changed while the reorder was executing", + category="ownership", + ) + with _open_secure_parent_directory( + root_guard, + destination.parent, + create=create_destination_parent, + ) as destination_parent: + if _entry_exists_at(destination_parent.parent_fd, destination.name): + raise StoryArcManagedReorderError( + "destination_collision", + "Reorder destination was occupied before publication", + category="collision", + ) + case_collision = _case_only_collision_at( + destination_parent.parent_fd, + destination.name, + ) + if case_collision is not None: + raise StoryArcManagedReorderError( + "case_only_collision", + "A case-only reorder destination collision exists", + category="collision", + ) + link_published = False + source_unlinked = False + try: + os.link( + source.name, + destination.name, + src_dir_fd=source_parent.parent_fd, + dst_dir_fd=destination_parent.parent_fd, + follow_symlinks=False, + ) + link_published = True + _fsync_directory(destination_parent.parent_fd) + os.unlink(source.name, dir_fd=source_parent.parent_fd) + source_unlinked = True + _fsync_directory(source_parent.parent_fd) + except BaseException: + # Never remove a destination merely because it now exists. A + # foreign entry may have won the race after our precheck. We + # may undo only our own successfully published hard-link, and + # only while the exact source inode still exists as the + # authoritative copy. Once source unlink succeeds, the + # destination is the sole durable artifact and must be left for + # journal-based restart reconciliation. + if link_published and not source_unlinked: + with suppress(OSError, StoryArcPlacementError): + _cleanup_published_link_if_source_authoritative( + source_parent=source_parent, + source_name=source.name, + destination_parent=destination_parent, + destination_name=destination.name, + expected=expected, + canonical_path=canonical_path, + mode=mode, + ) + raise + + +def _cleanup_published_link_if_source_authoritative( + *, + source_parent: _SecureParentDirectory, + source_name: str, + destination_parent: _SecureParentDirectory, + destination_name: str, + expected: Fingerprint, + canonical_path: Path | None, + mode: StoryArcPlacementMode, +) -> None: + """Undo only a proven duplicate link while the exact source still exists.""" + if not _entry_exists_at(source_parent.parent_fd, source_name) or not _entry_exists_at( + destination_parent.parent_fd, destination_name + ): + return + _validate_removal_representation_at(mode, source_parent, source_name) + _validate_removal_representation_at(mode, destination_parent, destination_name) + source_fingerprint = _fingerprint_target_at( + source_parent, + source_name, + canonical_path=canonical_path, + ) + destination_fingerprint = _fingerprint_target_at( + destination_parent, + destination_name, + canonical_path=canonical_path, + ) + if ( + source_fingerprint != expected + or destination_fingerprint != expected + or source_fingerprint.get("device") != destination_fingerprint.get("device") + or source_fingerprint.get("inode") != destination_fingerprint.get("inode") + ): + return + os.unlink(destination_name, dir_fd=destination_parent.parent_fd) + _fsync_directory(destination_parent.parent_fd) + + +def _restore_old_paths(plan: _SignedPlan) -> bool: + managed = [item for item in plan.placements if item.action == "rename"] + if not managed or plan.destination_root is None: + return True + try: + locations = _locate_managed_artifacts( + managed, + root=Path(plan.destination_root), + prefer="old", + ) + # First vacate every final destination. Only then restore old paths, + # which is safe even when the plan is a direct A/B swap. + for item in reversed(managed): + location = locations[item.placement_id] + if location != item.new_path: + continue + if item.temporary_path is None: + return False + _exclusive_move( + root=Path(plan.destination_root), + source=Path(item.new_path), + destination=Path(item.temporary_path), + expected=item.target_fingerprint, + canonical_path=( + Path(item.canonical_path) if item.canonical_path is not None else None + ), + mode=StoryArcPlacementMode(item.mode), + create_destination_parent=False, + ) + locations[item.placement_id] = item.temporary_path + for item in reversed(managed): + location = locations[item.placement_id] + if location == item.old_path: + continue + if location != item.temporary_path or item.temporary_path is None: + return False + _exclusive_move( + root=Path(plan.destination_root), + source=Path(item.temporary_path), + destination=Path(item.old_path), + expected=item.target_fingerprint, + canonical_path=( + Path(item.canonical_path) if item.canonical_path is not None else None + ), + mode=StoryArcPlacementMode(item.mode), + create_destination_parent=True, + ) + return True + except (OSError, StoryArcPlacementError, StoryArcManagedReorderError): + return False + + +def _stored_target_fingerprint(row: StoryArcPlacement) -> Fingerprint: + raw = dict(row.last_result or {}).get("target_fingerprint") + return dict(raw) if isinstance(raw, dict) else {} + + +def _plan_digest(value: dict[str, object]) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _normal_path(path: str | Path) -> str: + return os.path.normcase(os.path.abspath(path)) + + +def _assert_plan_binding( + plan: _SignedPlan, + *, + story_arc_id: int, + membership_id: int, + direction: StoryArcReorderDirection, + expected_revision: int, +) -> None: + if ( + plan.story_arc_id != story_arc_id + or plan.selected_membership_id != membership_id + or plan.direction != direction + or plan.expected_revision != expected_revision + ): + raise StoryArcManagedReorderError( + "invalid_preview_token", + "The reorder preview does not match this Story Arc move", + ) + + +def _plan_to_payload(plan: _SignedPlan) -> dict[str, object]: + return { + "schema_version": _TOKEN_SCHEMA_VERSION, + "story_arc_id": plan.story_arc_id, + "selected_membership_id": plan.selected_membership_id, + "direction": plan.direction, + "expected_revision": plan.expected_revision, + "operation_token": plan.operation_token, + "plan_digest": plan.plan_digest, + "destination_root": plan.destination_root, + "memberships": [asdict(item) for item in plan.memberships], + "placements": [asdict(item) for item in plan.placements], + } + + +def _plan_from_payload(raw: object) -> _SignedPlan: + if not isinstance(raw, dict) or raw.get("schema_version") != _TOKEN_SCHEMA_VERSION: + raise ValueError("invalid schema") + memberships_raw = raw["memberships"] + placements_raw = raw["placements"] + if not isinstance(memberships_raw, list) or len(memberships_raw) != 2: + raise ValueError("invalid memberships") + if ( + not isinstance(placements_raw, list) + or not placements_raw + or len(placements_raw) > _MAX_PLACEMENTS_PER_ADJACENT_MOVE + 2 + ): + raise ValueError("invalid placements") + memberships = tuple(_membership_plan_from_raw(item) for item in memberships_raw) + placements = tuple(_placement_plan_from_raw(item) for item in placements_raw) + direction = raw["direction"] + if direction not in {"up", "down"}: + raise ValueError("invalid direction") + destination_root = raw["destination_root"] + if destination_root is not None and not isinstance(destination_root, str): + raise ValueError("invalid root") + plan = _SignedPlan( + story_arc_id=_positive_int(raw["story_arc_id"]), + selected_membership_id=_positive_int(raw["selected_membership_id"]), + direction=cast("StoryArcReorderDirection", direction), + expected_revision=_positive_int(raw["expected_revision"]), + operation_token=_fixed_string(raw["operation_token"], length=32), + plan_digest=_fixed_string(raw["plan_digest"], length=64), + destination_root=destination_root, + memberships=memberships, + placements=placements, + ) + semantic: dict[str, object] = { + "story_arc_id": plan.story_arc_id, + "selected_membership_id": plan.selected_membership_id, + "direction": plan.direction, + "expected_revision": plan.expected_revision, + "destination_root": plan.destination_root, + "memberships": [asdict(item) for item in plan.memberships], + "placements": [{**asdict(item), "temporary_path": None} for item in plan.placements], + } + if _plan_digest(semantic) != plan.plan_digest: + raise ValueError("digest mismatch") + return plan + + +def _membership_plan_from_raw(raw: object) -> _MembershipPlan: + if not isinstance(raw, dict) or set(raw) != { + "membership_id", + "old_sequence_number", + "old_source_ordinal", + "new_sequence_number", + "new_source_ordinal", + }: + raise ValueError("invalid membership plan") + return _MembershipPlan( + membership_id=_positive_int(raw["membership_id"]), + old_sequence_number=_nonnegative_int(raw["old_sequence_number"]), + old_source_ordinal=_nonnegative_int(raw["old_source_ordinal"]), + new_sequence_number=_nonnegative_int(raw["new_sequence_number"]), + new_source_ordinal=_nonnegative_int(raw["new_source_ordinal"]), + ) + + +def _placement_plan_from_raw(raw: object) -> _PlacementPlan: + expected_keys = { + "placement_id", + "membership_id", + "ownership", + "mode", + "old_reading_order", + "new_reading_order", + "old_path", + "new_path", + "temporary_path", + "rendered_path_after", + "action", + "canonical_path", + "source_fingerprint", + "target_fingerprint", + "placement_state", + "last_result_snapshot", + } + if not isinstance(raw, dict) or set(raw) != expected_keys: + raise ValueError("invalid placement plan") + action = raw["action"] + if action not in { + "rename", + "managed_unchanged", + "referenced_drift", + "referenced_unchanged", + "logical_reorder", + }: + raise ValueError("invalid action") + temporary_path = raw["temporary_path"] + canonical_path = raw["canonical_path"] + if temporary_path is not None and not isinstance(temporary_path, str): + raise ValueError("invalid temporary path") + if canonical_path is not None and not isinstance(canonical_path, str): + raise ValueError("invalid canonical path") + source_fingerprint = raw["source_fingerprint"] + target_fingerprint = raw["target_fingerprint"] + last_result_snapshot = raw["last_result_snapshot"] + placement_state = raw["placement_state"] + if ( + not isinstance(source_fingerprint, dict) + or not isinstance(target_fingerprint, dict) + or not isinstance(last_result_snapshot, dict) + or (placement_state is not None and not isinstance(placement_state, str)) + ): + raise ValueError("invalid fingerprint") + return _PlacementPlan( + placement_id=_nonnegative_int(raw["placement_id"]), + membership_id=_positive_int(raw["membership_id"]), + ownership=_string(raw["ownership"]), + mode=_string(raw["mode"]), + old_reading_order=_nonnegative_int(raw["old_reading_order"]), + new_reading_order=_nonnegative_int(raw["new_reading_order"]), + old_path=_string(raw["old_path"]), + new_path=_string(raw["new_path"]), + temporary_path=temporary_path, + rendered_path_after=_string(raw["rendered_path_after"]), + action=cast("StoryArcReorderAction", action), + canonical_path=canonical_path, + source_fingerprint=dict(source_fingerprint), + target_fingerprint=dict(target_fingerprint), + placement_state=placement_state, + last_result_snapshot=dict(last_result_snapshot), + ) + + +def _positive_int(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError("positive integer required") + return value + + +def _nonnegative_int(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError("nonnegative integer required") + return value + + +def _string(value: object) -> str: + if not isinstance(value, str): + raise ValueError("string required") + return value + + +def _fixed_string(value: object, *, length: int) -> str: + result = _string(value) + if len(result) != length: + raise ValueError("fixed string length required") + return result diff --git a/src/pullbox/services/story_arc_membership_policy.py b/src/pullbox/services/story_arc_membership_policy.py new file mode 100644 index 00000000..638eaba6 --- /dev/null +++ b/src/pullbox/services/story_arc_membership_policy.py @@ -0,0 +1,25 @@ +"""Reading-order review never changes a member's exact canonical identity.""" + +from sqlalchemy import func +from sqlalchemy.sql.elements import ColumnElement + +from pullbox.models.story_arc import IssueStoryArc, StoryArcSourceKind + + +def provider_issue_identity(member: IssueStoryArc) -> str | None: + """Return the exact provider identity, without interpreting import-local IDs.""" + if ( + member.source_kind == StoryArcSourceKind.PROVIDER + or (member.evidence or {}).get("provider") == "comicvine" + ): + return member.source_issue_id + return None + + +def requires_order_review(member: IssueStoryArc) -> bool: + return (member.evidence or {}).get("catalog_review_required") is True + + +def order_review_filter() -> ColumnElement[bool]: + """Portable SQLite/PostgreSQL JSON boolean with an absent-key default.""" + return func.coalesce(IssueStoryArc.evidence["catalog_review_required"].as_boolean(), False) diff --git a/src/pullbox/services/story_arc_placement_integration.py b/src/pullbox/services/story_arc_placement_integration.py new file mode 100644 index 00000000..7502db46 --- /dev/null +++ b/src/pullbox/services/story_arc_placement_integration.py @@ -0,0 +1,3202 @@ +"""Short-transaction database boundary for story-arc placement operations. + +The filesystem service is intentionally synchronous. This adapter freezes a +complete per-arc policy, commits a prepared placement row, performs filesystem +work in a worker thread with no database transaction open, and then reconciles +the durable row. Canonical issues and library files are read-only throughout. +""" + +from __future__ import annotations + +import asyncio +import enum +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING +from uuid import uuid4 + +from sqlalchemy import and_, delete, func, select +from sqlalchemy import update as sa_update +from sqlalchemy.exc import IntegrityError + +from pullbox.core.exceptions import ValidationError +from pullbox.core.story_arc_naming import ( + DEFAULT_STORY_ARC_FILE_TEMPLATE, + DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + StoryArcNamingValues, + StoryArcOriginalFilenameError, + render_story_arc_relative_path, + validate_story_arc_file_template, + validate_story_arc_folder_template, +) +from pullbox.models.import_job import ( + ImportControlRequest, + ImportJob, + ImportJobAction, + ImportJobActionStatus, + ImportJobStatus, +) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile, LibraryRoot +from pullbox.models.publisher import Publisher +from pullbox.models.series import Series +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcPlacement, + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcPlacementState, + StoryArcResolutionState, + StoryArcSourceKind, + StoryArcSymlinkStyle, +) +from pullbox.services.library_root_management import ( + validate_managed_library_root, + validate_reference_library_root, +) +from pullbox.services.story_arc_membership_policy import order_review_filter, requires_order_review +from pullbox.services.story_arc_placement_preview import ( + StoryArcCollisionKind, + StoryArcPlacementPreviewState, +) +from pullbox.services.story_arc_placement_service import ( + ManagedStoryArcPlacementEvidence, + PreparedManagedStoryArcPlacementEvidence, + StoryArcPlacementCancellationError, + StoryArcPlacementCollisionError, + StoryArcPlacementError, + StoryArcPlacementInspection, + StoryArcPlacementInspectionEvidence, + StoryArcPlacementInspectionState, + StoryArcPlacementJournalEvent, + StoryArcPlacementOwnershipError, + StoryArcPlacementPlan, + StoryArcPlacementPreparation, + StoryArcPlacementRemovalResult, + StoryArcPlacementResult, + StoryArcPlacementSafetyError, + execute_story_arc_placement, + inspect_story_arc_placement, + prepare_story_arc_placement, + recover_prepared_story_arc_placement, + remove_managed_story_arc_placement, +) + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Awaitable, Callable, Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + +STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION = 1 +MAX_STORY_ARC_PLACEMENT_PAGE_SIZE = 200 +_PLACEMENT_HEARTBEAT_SECONDS = 10.0 +_PLACEMENT_OPERATION_LEASE = timedelta(minutes=5) +_POLICY_SNAPSHOT_KEYS = frozenset( + { + "schema_version", + "mode", + "target_library_root_id", + "destination_root", + "folder_template", + "file_template", + "symlink_style", + "synchronize", + } +) +_IMPORT_PLACEMENT_ACTION_TYPE = "story_arc_managed_placement_requested" +_IMPORT_PLACEMENT_PHASE = "story_arc_placements" +_IMPORT_PLACEMENT_PAYLOAD_KEYS = frozenset( + { + "schema_version", + "sync_work_id", + "membership_id", + "desired_generation", + "imported_story_arc_id", + "imported_story_arc_entry_id", + "source_import_job_id", + } +) + + +class StoryArcPlacementPolicyMode(enum.StrEnum): + """User-facing policy mode, including a truly logical-only arc.""" + + LOGICAL = "logical" + REFERENCE_ONLY = "reference_only" + COPY = "copy" + HARDLINK = "hardlink" + SYMLINK = "symlink" + + +class StoryArcPlacementIntegrationError(RuntimeError): + """Safe categorized failure exposed by the database/API adapter.""" + + def __init__(self, code: str, message: str, *, category: str = "validation") -> None: + self.code = code + self.category = category + super().__init__(message) + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPolicyInput: + """Complete candidate policy supplied by an API or import adapter.""" + + mode: StoryArcPlacementPolicyMode | str + target_library_root_id: int | None + destination_root: str | None + folder_template: str = DEFAULT_STORY_ARC_FOLDER_TEMPLATE + file_template: str = DEFAULT_STORY_ARC_FILE_TEMPLATE + symlink_style: StoryArcSymlinkStyle | str | None = None + synchronize: bool = False + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPolicy: + """Validated immutable effective policy for one story arc.""" + + configured: bool + revision: int + mode: StoryArcPlacementPolicyMode + target_library_root_id: int | None + destination_root: str | None + folder_template: str + file_template: str + symlink_style: StoryArcSymlinkStyle | None + synchronize: bool + + @property + def snapshot(self) -> dict[str, object]: + """Return the complete versioned JSON representation persisted on the arc.""" + return { + "schema_version": STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + "mode": self.mode.value, + "target_library_root_id": self.target_library_root_id, + "destination_root": self.destination_root, + "folder_template": self.folder_template, + "file_template": self.file_template, + "symlink_style": self.symlink_style.value if self.symlink_style else None, + "synchronize": self.synchronize, + } + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPreviewItem: + """One bounded membership preview without ORM or filesystem ownership claims.""" + + membership_id: int + sequence_number: int + issue_id: int | None + issue_number_text: str + mode: str + state: str + target_path: str | None + collision: str + reason: str | None + required_bytes: int + proposed_ownership: str + overwrite_allowed: bool + classification: str + placement_id: int | None + current_ownership: str | None + inspection_code: str | None + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPreviewPage: + """Bounded preview page with deterministic pagination metadata.""" + + items: tuple[StoryArcPlacementPreviewItem, ...] + total: int + limit: int + offset: int + has_more: bool + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementView: + """Detached API-safe view of one durable placement row.""" + + id: int + issue_story_arc_id: int + library_file_id: int | None + library_root_id: int | None + placement_path: str + mode: StoryArcPlacementMode + ownership: StoryArcPlacementOwnership + symlink_style: StoryArcSymlinkStyle | None + rendered_reading_order: int | None + policy_schema_version: int | None + source_fingerprint: dict[str, object] + target_fingerprint: dict[str, object] + state: StoryArcPlacementState + last_result: dict[str, object] + last_checked_at: datetime | None + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPage: + """Bounded durable placement page.""" + + items: tuple[StoryArcPlacementView, ...] + total: int + limit: int + offset: int + has_more: bool + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementSyncResult: + """Truthful result from one logical or filesystem synchronization.""" + + membership_id: int + outcome: str + placement: StoryArcPlacementView | None + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementRemovalView: + """Truthful ownership-aware result of explicitly removing placement evidence.""" + + placement_id: int + ownership: StoryArcPlacementOwnership + artifact_removed: bool + canonical_preserved: bool = True + referenced_artifact_preserved: bool = False + automatic_sync_disabled: bool = True + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementImportProvenance: + """Immutable import job/action ownership stamped only on a new managed row.""" + + import_job_id: int + import_action_id: int + + +@dataclass(frozen=True, slots=True) +class _PlacementContext: + membership_id: int + story_arc_id: int + sequence_number: int + issue_id: int | None + issue_number_text: str + story_arc_name: str + series_name: str + publisher_name: str | None + issue_title: str | None + year: int | None + series_start_year: int | None + series_end_year: int | None + library_file_id: int | None + canonical_path: str | None + extension: str + + def naming_values(self) -> StoryArcNamingValues: + return StoryArcNamingValues( + story_arc=self.story_arc_name, + reading_order=self.sequence_number, + series=self.series_name, + publisher=self.publisher_name, + issue_number=self.issue_number_text, + issue_title=self.issue_title, + year=self.year, + start_year=self.series_start_year, + end_year=self.series_end_year, + extension=self.extension, + original_filename=Path(self.canonical_path).name if self.canonical_path else None, + ) + + +@dataclass(frozen=True, slots=True) +class _PreviewPlacementEvidence: + """Detached evidence for the one placement matching a rendered page target.""" + + id: int + issue_story_arc_id: int + placement_path: str + mode: StoryArcPlacementMode + ownership: StoryArcPlacementOwnership + symlink_style: StoryArcSymlinkStyle | None + source_fingerprint: dict[str, object] + target_fingerprint: dict[str, object] + creating_action_id: int | None + + +@dataclass(slots=True) +class _MembershipSyncLock: + """One reference-counted process-local membership operation lock.""" + + lock: asyncio.Lock + users: int = 0 + + +_sync_locks: dict[tuple[int, int], _MembershipSyncLock] = {} + + +@asynccontextmanager +async def _membership_sync_lock(lock_key: tuple[int, int]) -> AsyncIterator[None]: + """Serialize sync/removal without dropping a lock that still has waiters.""" + entry = _sync_locks.setdefault(lock_key, _MembershipSyncLock(lock=asyncio.Lock())) + entry.users += 1 + try: + async with entry.lock: + yield + finally: + entry.users -= 1 + if entry.users == 0 and _sync_locks.get(lock_key) is entry: + del _sync_locks[lock_key] + + +class StoryArcPlacementSyncService: + """Freeze policies and synchronize one membership with short transactions.""" + + async def get_policy( + self, + session: AsyncSession, + story_arc_id: int, + ) -> StoryArcPlacementPolicy: + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise _not_found("story_arc_not_found", "Story arc was not found") + return _policy_from_arc(arc) + + async def validate_policy( + self, + session: AsyncSession, + story_arc_id: int, + proposal: StoryArcPlacementPolicyInput, + ) -> StoryArcPlacementPolicy: + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise _not_found("story_arc_not_found", "Story arc was not found") + if arc.lifecycle is StoryArcLifecycle.ARCHIVED: + raise StoryArcPlacementIntegrationError( + "story_arc_archived", + "Archived story arcs cannot change placement policy", + ) + return await validate_story_arc_placement_policy_input( + session, + proposal, + revision=arc.revision, + ) + + async def update_policy( + self, + session: AsyncSession, + story_arc_id: int, + *, + expected_revision: int, + proposal: StoryArcPlacementPolicyInput, + ) -> StoryArcPlacementPolicy: + if isinstance(expected_revision, bool) or expected_revision < 1: + raise StoryArcPlacementIntegrationError( + "invalid_revision", + "Expected story-arc revision must be a positive integer", + ) + policy = await self.validate_policy(session, story_arc_id, proposal) + current_arc = await session.get(StoryArc, story_arc_id) + if current_arc is None: # pragma: no cover - validate_policy loaded it + raise _not_found("story_arc_not_found", "Story arc was not found") + if current_arc.revision != expected_revision: + raise StoryArcPlacementIntegrationError( + "revision_conflict", + ( + "Story arc revision changed: expected " + f"{expected_revision}, current {current_arc.revision}" + ), + category="conflict", + ) + current_policy = _policy_from_arc(current_arc) + if ( + current_policy.configured + and _placement_policy_shape(current_policy) != _placement_policy_shape(policy) + and await _arc_has_managed_placements(session, story_arc_id) + ): + raise StoryArcPlacementIntegrationError( + "managed_policy_change_requires_migration", + ( + "Move or remove current managed placements before changing their " + "destination policy" + ), + category="conflict", + ) + next_revision = expected_revision + 1 + result = await session.execute( + sa_update(StoryArc) + .where( + StoryArc.id == story_arc_id, + StoryArc.revision == expected_revision, + StoryArc.lifecycle == StoryArcLifecycle.ACTIVE, + ) + .values( + target_library_root_id=policy.target_library_root_id, + policy_schema_version=STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + policy_snapshot=policy.snapshot, + sync_enabled=policy.synchronize, + revision=next_revision, + ) + ) + if result.rowcount != 1: # type: ignore[attr-defined] + current = await session.get(StoryArc, story_arc_id) + if current is None: + raise _not_found("story_arc_not_found", "Story arc was not found") + if current.lifecycle is StoryArcLifecycle.ARCHIVED: + raise StoryArcPlacementIntegrationError( + "story_arc_archived", + "Archived story arcs cannot change placement policy", + ) + raise StoryArcPlacementIntegrationError( + "revision_conflict", + ( + "Story arc revision changed: expected " + f"{expected_revision}, current {current.revision}" + ), + category="conflict", + ) + await session.execute( + sa_update(IssueStoryArc) + .where(IssueStoryArc.story_arc_id == story_arc_id) + .values( + sync_eligible=( + and_( + IssueStoryArc.issue_id.is_not(None), + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + ~order_review_filter(), + ) + if policy.synchronize + else False + ) + ) + ) + await session.commit() + return StoryArcPlacementPolicy( + configured=True, + revision=next_revision, + mode=policy.mode, + target_library_root_id=policy.target_library_root_id, + destination_root=policy.destination_root, + folder_template=policy.folder_template, + file_template=policy.file_template, + symlink_style=policy.symlink_style, + synchronize=policy.synchronize, + ) + + async def preview_arc( + self, + session: AsyncSession, + story_arc_id: int, + *, + limit: int, + offset: int, + proposal: StoryArcPlacementPolicyInput | None = None, + ) -> StoryArcPlacementPreviewPage: + limit, offset = _bounded_page(limit, offset) + policy = ( + await self.validate_policy(session, story_arc_id, proposal) + if proposal is not None + else await self.get_policy(session, story_arc_id) + ) + total, contexts = await _load_context_page( + session, + story_arc_id, + limit=limit, + offset=offset, + ) + target_paths = _rendered_target_paths(contexts, policy) + evidence_by_path = await _load_matching_preview_evidence(session, target_paths) + # Close the read transaction before filesystem inspection in the worker. + await session.rollback() + items = await asyncio.to_thread( + _preview_contexts, + contexts, + policy, + evidence_by_path, + ) + return StoryArcPlacementPreviewPage( + items=items, + total=total, + limit=limit, + offset=offset, + has_more=(offset + limit) < total, + ) + + async def list_placements( + self, + session: AsyncSession, + story_arc_id: int, + *, + limit: int, + offset: int, + ) -> StoryArcPlacementPage: + limit, offset = _bounded_page(limit, offset) + if await session.get(StoryArc, story_arc_id) is None: + raise _not_found("story_arc_not_found", "Story arc was not found") + arc_filter = IssueStoryArc.story_arc_id == story_arc_id + total = int( + await session.scalar( + select(func.count(StoryArcPlacement.id)) + .join( + IssueStoryArc, + StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id, + ) + .where(arc_filter) + ) + or 0 + ) + rows = list( + ( + await session.scalars( + select(StoryArcPlacement) + .join( + IssueStoryArc, + StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id, + ) + .where(arc_filter) + .order_by( + IssueStoryArc.sequence_number.asc(), + StoryArcPlacement.id.asc(), + ) + .limit(limit) + .offset(offset) + ) + ).all() + ) + return StoryArcPlacementPage( + items=tuple(_placement_view(row) for row in rows), + total=total, + limit=limit, + offset=offset, + has_more=(offset + limit) < total, + ) + + async def sync_membership( + self, + session: AsyncSession, + story_arc_id: int, + membership_id: int, + *, + adopt_identical_existing: bool = False, + cancellation_requested: Callable[[], bool] | None = None, + import_provenance: StoryArcPlacementImportProvenance | None = None, + ) -> StoryArcPlacementSyncResult: + lock_key = (story_arc_id, membership_id) + async with _membership_sync_lock(lock_key): + try: + return await self._sync_membership_locked( + session, + story_arc_id, + membership_id, + adopt_identical_existing=adopt_identical_existing, + cancellation_requested=cancellation_requested, + import_provenance=import_provenance, + ) + except StoryArcOriginalFilenameError as exc: + raise StoryArcPlacementIntegrationError( + "original_filename_unsafe", str(exc), category="safety" + ) from exc + + async def retry_placement( + self, + session: AsyncSession, + story_arc_id: int, + placement_id: int, + *, + adopt_identical_existing: bool = False, + ) -> StoryArcPlacementSyncResult: + placement = await _require_placement(session, story_arc_id, placement_id) + return await self.sync_membership( + session, + story_arc_id, + placement.issue_story_arc_id, + adopt_identical_existing=adopt_identical_existing, + ) + + async def repair_placement( + self, + session: AsyncSession, + story_arc_id: int, + placement_id: int, + ) -> StoryArcPlacementSyncResult: + placement = await _require_placement(session, story_arc_id, placement_id) + if placement.ownership is not StoryArcPlacementOwnership.MANAGED: + raise StoryArcPlacementIntegrationError( + "referenced_placement_immutable", + "Referenced story-arc placements cannot be repaired or changed", + category="ownership", + ) + return await self.sync_membership( + session, + story_arc_id, + placement.issue_story_arc_id, + ) + + async def remove_placement( + self, + session: AsyncSession, + story_arc_id: int, + placement_id: int, + *, + confirm_managed_artifact_removal: bool = False, + abandoned_published_operation_token: str | None = None, + ) -> StoryArcPlacementRemovalView: + """Remove only owned placement evidence, never a canonical or referenced file.""" + placement = await _require_placement(session, story_arc_id, placement_id) + ownership = placement.ownership + membership_id = placement.issue_story_arc_id + if ownership is StoryArcPlacementOwnership.REFERENCED: + # Close the initial read transaction before waiting for an active + # synchronization. The durable operation token below remains the + # cross-process fence; this lock provides deterministic local UX. + await session.commit() + async with _membership_sync_lock((story_arc_id, membership_id)): + current = await _require_placement(session, story_arc_id, placement_id) + if current.ownership is not StoryArcPlacementOwnership.REFERENCED: + raise StoryArcPlacementIntegrationError( + "placement_ownership_changed", + "Story-arc placement ownership changed before it could be forgotten", + category="ownership", + ) + reference_token_filter: ColumnElement[bool] = StoryArcPlacement.operation_token.is_( + None + ) + if abandoned_published_operation_token is not None: + _require_published_operation_token( + current, + abandoned_published_operation_token, + ) + reference_token_filter = ( + StoryArcPlacement.operation_token == abandoned_published_operation_token + ) + reference_removal_token = uuid4().hex + previous_result = dict(current.last_result or {}) + reserve = await session.execute( + sa_update(StoryArcPlacement) + .where( + StoryArcPlacement.id == placement_id, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.REFERENCED, + reference_token_filter, + ) + .values( + operation_token=reference_removal_token, + last_result={ + **previous_result, + "schema_version": 1, + "status": "remove_prepared", + "operation_token": reference_removal_token, + }, + ) + ) + if reserve.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_in_progress", + "Referenced placement evidence is being updated by another operation", + category="conflict", + ) + result = await session.execute( + delete(StoryArcPlacement).where( + StoryArcPlacement.id == placement_id, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.REFERENCED, + StoryArcPlacement.operation_token == reference_removal_token, + ) + ) + if result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_in_progress", + "Referenced placement evidence is being updated by another operation", + category="conflict", + ) + await session.execute( + sa_update(IssueStoryArc) + .where(IssueStoryArc.id == membership_id) + .values( + sync_eligible=False, + last_materialization_result={ + "schema_version": 1, + "status": "placement_reference_removed", + "placement_id": placement_id, + "artifact_removed": False, + "canonical_preserved": True, + "referenced_artifact_preserved": True, + }, + ) + ) + await session.commit() + return StoryArcPlacementRemovalView( + placement_id=placement_id, + ownership=ownership, + artifact_removed=False, + referenced_artifact_preserved=True, + ) + + if not confirm_managed_artifact_removal: + raise StoryArcPlacementIntegrationError( + "managed_removal_confirmation_required", + ( + "Confirm removal of the Pullbox-managed arc artifact; " + "the canonical comic will be preserved" + ), + ) + + evidence = _managed_removal_evidence(placement) + if evidence is None: + raise StoryArcPlacementIntegrationError( + "managed_ownership_evidence_missing", + "Managed placement cannot be removed without durable ownership evidence", + category="ownership", + ) + policy = await self.get_policy(session, story_arc_id) + if policy.destination_root is None or policy.mode in { + StoryArcPlacementPolicyMode.LOGICAL, + StoryArcPlacementPolicyMode.REFERENCE_ONLY, + }: + raise StoryArcPlacementIntegrationError( + "managed_policy_evidence_missing", + "Managed placement has no valid destination policy for safe removal", + category="safety", + ) + canonical_path_raw = ( + await session.scalar( + select(LibraryFile.file_path).where(LibraryFile.id == placement.library_file_id) + ) + if placement.library_file_id is not None + else None + ) + observed_token = placement.operation_token + previous_result = dict(placement.last_result or {}) + if abandoned_published_operation_token is not None: + _require_published_operation_token( + placement, + abandoned_published_operation_token, + ) + if ( + observed_token is not None + and abandoned_published_operation_token is None + and placement.updated_at > datetime.now(UTC) - _PLACEMENT_OPERATION_LEASE + ): + raise StoryArcPlacementIntegrationError( + "placement_operation_in_progress", + "A story-arc placement operation is already in progress", + category="conflict", + ) + + operation_token = uuid4().hex + managed_token_filter: ColumnElement[bool] = ( + StoryArcPlacement.operation_token.is_(None) + if observed_token is None + else StoryArcPlacement.operation_token == observed_token + ) + reserve = await session.execute( + sa_update(StoryArcPlacement) + .where( + StoryArcPlacement.id == placement_id, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.MANAGED, + managed_token_filter, + ) + .values( + operation_token=operation_token, + last_result={ + **previous_result, + "schema_version": 1, + "status": "remove_prepared", + "operation_token": operation_token, + }, + ) + ) + if reserve.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_superseded", + "Another worker reserved this story-arc placement", + category="conflict", + ) + await session.commit() + + caller_cancelled = False + try: + removed, caller_cancelled = await _run_filesystem_call( + partial( + remove_managed_story_arc_placement, + evidence, + destination_root=Path(policy.destination_root), + canonical_path=( + Path(canonical_path_raw) if canonical_path_raw is not None else None + ), + ), + heartbeat=partial( + _refresh_operation_lease, + session, + placement_id=placement_id, + operation_token=operation_token, + ), + ) + except StoryArcPlacementError as exc: + if ( + not evidence.target_fingerprint + and exc.code == "filesystem_error" + and isinstance(exc.__cause__, FileNotFoundError) + ): + # The secure, root-anchored walk proved that an intermediate + # target directory is absent. This is the same idempotent + # absence represented by the removal-only ``{}`` sentinel. + removed = StoryArcPlacementRemovalResult( + placement_path=evidence.placement_path, + removed=False, + ) + else: + await _persist_removal_failure( + session, + placement_id=placement_id, + operation_token=operation_token, + error=exc, + ) + raise _translate_filesystem_error(exc) from exc + except (OSError, ValueError) as exc: + error = StoryArcPlacementIntegrationError( + "placement_removal_failed", + "Story-arc placement removal failed safely", + category="safety", + ) + await _persist_removal_failure( + session, + placement_id=placement_id, + operation_token=operation_token, + error=error, + ) + raise error from exc + + await _delete_removed_placement_checkpoint( + session, + placement_id=placement_id, + membership_id=membership_id, + operation_token=operation_token, + artifact_removed=removed.removed, + ) + if caller_cancelled: + raise asyncio.CancelledError + return StoryArcPlacementRemovalView( + placement_id=placement_id, + ownership=ownership, + artifact_removed=removed.removed, + ) + + async def _sync_membership_locked( + self, + session: AsyncSession, + story_arc_id: int, + membership_id: int, + *, + adopt_identical_existing: bool, + cancellation_requested: Callable[[], bool] | None, + import_provenance: StoryArcPlacementImportProvenance | None, + ) -> StoryArcPlacementSyncResult: + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise _not_found("story_arc_not_found", "Story arc was not found") + if arc.lifecycle is StoryArcLifecycle.ARCHIVED: + raise StoryArcPlacementIntegrationError( + "story_arc_archived", + "Archived story arcs cannot synchronize placements", + ) + policy = _policy_from_arc(arc) + if not policy.configured: + raise StoryArcPlacementIntegrationError( + "placement_policy_not_configured", + "Configure the story-arc placement policy before synchronization", + ) + if not await _policy_root_is_available(session, policy): + raise StoryArcPlacementIntegrationError( + "target_library_root_unavailable", + "Selected story-arc library root is no longer available", + category="safety", + ) + context = await _load_one_context(session, story_arc_id, membership_id) + membership = await session.get(IssueStoryArc, membership_id) + if membership is None: # pragma: no cover - guarded by context loader + raise _not_found("membership_not_found", "Story-arc membership was not found") + if context.issue_id is None or context.library_file_id is None: + raise StoryArcPlacementIntegrationError( + "canonical_file_unavailable", + "The resolved story-arc membership has no canonical library file", + category="safety", + ) + if requires_order_review(membership): + raise StoryArcPlacementIntegrationError( + "reading_order_review_required", + "Confirm this member's reading order before synchronizing its arc file", + ) + if membership.resolution_state is not StoryArcResolutionState.RESOLVED: + raise StoryArcPlacementIntegrationError( + "membership_not_resolved", + "Only a resolved story-arc membership can synchronize a placement", + ) + if import_provenance is not None: + await _validate_import_provenance( + session, + provenance=import_provenance, + story_arc_id=story_arc_id, + membership_id=membership_id, + ) + if ( + policy.mode + not in { + StoryArcPlacementPolicyMode.COPY, + StoryArcPlacementPolicyMode.HARDLINK, + StoryArcPlacementPolicyMode.SYMLINK, + } + or adopt_identical_existing + ): + raise StoryArcPlacementIntegrationError( + "import_placement_requires_managed_mode", + "Import-origin placement work requires copy, hardlink, or symlink mode", + category="ownership", + ) + + if policy.mode is StoryArcPlacementPolicyMode.LOGICAL: + outcome = StoryArcPlacementPolicyMode.LOGICAL.value + membership.last_materialization_result = { + "schema_version": 1, + "status": "complete", + "outcome": outcome, + } + await session.commit() + return StoryArcPlacementSyncResult( + membership_id=membership_id, + outcome=outcome, + placement=None, + ) + if policy.mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY: + return await _sync_reference_only( + session, + context=context, + policy=policy, + membership=membership, + adopt_identical_existing=adopt_identical_existing, + cancellation_requested=cancellation_requested, + ) + if adopt_identical_existing: + # Explicit adoption is a read-only referenced operation. Route it + # through the fail-closed reference path so a disappearing target + # can never turn the confirmation into an unjournaled managed copy. + return await _sync_reference_only( + session, + context=context, + policy=policy, + membership=membership, + adopt_identical_existing=True, + cancellation_requested=cancellation_requested, + ) + + plan = _build_plan( + context, + policy, + adopt_identical_existing=adopt_identical_existing, + ) + if plan.destination_root is None: # pragma: no cover - complete policy invariant + raise StoryArcPlacementIntegrationError( + "destination_root_required", + "Managed placement policy requires a destination root", + ) + relative_path = render_story_arc_relative_path( + plan.values, + folder_template=policy.folder_template, + file_template=policy.file_template, + ) + target_path = plan.destination_root / relative_path + existing = await session.scalar( + select(StoryArcPlacement) + .where( + StoryArcPlacement.issue_story_arc_id == membership_id, + StoryArcPlacement.placement_path == str(target_path), + ) + .limit(1) + ) + other_managed = await session.scalar( + select(StoryArcPlacement) + .where( + StoryArcPlacement.issue_story_arc_id == membership_id, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.MANAGED, + StoryArcPlacement.placement_path != str(target_path), + ) + .order_by(StoryArcPlacement.id.asc()) + .limit(1) + ) + if existing is not None and existing.ownership is StoryArcPlacementOwnership.REFERENCED: + if import_provenance is not None: + raise StoryArcPlacementIntegrationError( + "import_placement_reference_not_owned", + "Import-origin work cannot adopt or replace a referenced artifact", + category="ownership", + ) + # A prior explicit adoption or crash-safe ownership downgrade stays + # referenced. Validate it read-only and never silently promote it + # to a Pullbox-managed artifact. + return await _sync_reference_only( + session, + context=context, + policy=policy, + membership=membership, + adopt_identical_existing=False, + cancellation_requested=cancellation_requested, + ) + if import_provenance is not None and other_managed is not None: + raise StoryArcPlacementIntegrationError( + "import_placement_existing_managed_not_owned", + "Import-origin work cannot change an existing managed placement", + category="ownership", + ) + if other_managed is not None: + other_managed.state = StoryArcPlacementState.DRIFTED + other_managed.last_result = { + **dict(other_managed.last_result or {}), + "status": "drifted", + "error_code": "policy_destination_changed", + } + membership.last_materialization_result = { + "schema_version": 1, + "status": "drifted", + "error_code": "policy_destination_changed", + } + await session.commit() + raise StoryArcPlacementIntegrationError( + "policy_destination_changed", + "Existing managed placement requires an explicit policy-change repair", + category="conflict", + ) + if ( + import_provenance is not None + and existing is not None + and ( + existing.source_import_job_id != import_provenance.import_job_id + or existing.creating_action_id != import_provenance.import_action_id + ) + ): + raise StoryArcPlacementIntegrationError( + "import_placement_existing_managed_not_owned", + "Import-origin work cannot retrofit ownership onto an existing placement", + category="ownership", + ) + + existing_id = existing.id if existing is not None else None + existing_operation_token = existing.operation_token if existing is not None else None + existing_last_result = dict(existing.last_result or {}) if existing is not None else {} + if ( + existing is not None + and existing_operation_token is not None + and existing_last_result.get("status") != "published_pending_reconcile" + and existing.updated_at > datetime.now(UTC) - _PLACEMENT_OPERATION_LEASE + ): + raise StoryArcPlacementIntegrationError( + "placement_operation_in_progress", + "A story-arc placement operation is already in progress", + category="conflict", + ) + evidence = _managed_evidence(existing) if existing is not None else None + prepared_evidence = _prepared_managed_evidence(existing) if existing is not None else None + + # Close the read transaction before hashing the canonical source or + # inspecting the destination root. Only detached scalar context is + # retained across this boundary. + await session.commit() + if cancellation_requested is not None and cancellation_requested(): + cancellation = StoryArcPlacementIntegrationError( + "cancelled", + "Story-arc placement was cancelled", + category="cancelled", + ) + await _persist_preflight_failure( + session, + membership_id=membership_id, + placement_id=existing_id, + observed_operation_token=existing_operation_token, + error=cancellation, + ) + raise cancellation + try: + recovered = ( + await asyncio.to_thread( + recover_prepared_story_arc_placement, + plan, + prepared_evidence, + ) + if prepared_evidence is not None + else None + ) + preparation = ( + await asyncio.to_thread( + prepare_story_arc_placement, + plan, + existing_managed=evidence, + ) + if recovered is None + else None + ) + except StoryArcPlacementError as exc: + if existing_id is not None and prepared_evidence is not None: + await _persist_failure( + session, + membership_id=membership_id, + placement_id=existing_id, + operation_token=( + prepared_evidence.operation_token + if prepared_evidence is not None + else uuid4().hex + ), + error=exc, + ) + else: + await _persist_preflight_failure( + session, + membership_id=membership_id, + placement_id=existing_id, + observed_operation_token=existing_operation_token, + error=exc, + ) + raise _translate_filesystem_error(exc) from exc + + if recovered is not None: + if existing_id is None or prepared_evidence is None: # pragma: no cover + raise StoryArcPlacementIntegrationError( + "prepared_placement_missing", + "Prepared story-arc placement record disappeared during recovery", + category="conflict", + ) + synchronized, reconcile_cancelled = await _run_published_reconciliation( + session, + context=context, + policy=policy, + prepared_placement_id=existing_id, + operation_token=prepared_evidence.operation_token, + result=recovered, + ) + if reconcile_cancelled: + raise asyncio.CancelledError + return synchronized + + if preparation is None: # pragma: no cover - recovered returned above + raise StoryArcPlacementIntegrationError( + "placement_preparation_missing", + "Story-arc placement preparation did not produce durable evidence", + category="safety", + ) + + session.expire_all() + current_arc = await session.get(StoryArc, story_arc_id) + if current_arc is None: + raise _not_found("story_arc_not_found", "Story arc was not found") + if _policy_from_arc(current_arc) != policy: + raise StoryArcPlacementIntegrationError( + "placement_policy_changed", + "Story-arc placement policy changed during preparation", + category="conflict", + ) + current_context = await _load_one_context(session, story_arc_id, membership_id) + if current_context != context: + raise StoryArcPlacementIntegrationError( + "canonical_context_changed", + "Canonical issue or library-file context changed during preparation", + category="conflict", + ) + if import_provenance is not None: + await _validate_import_provenance( + session, + provenance=import_provenance, + story_arc_id=story_arc_id, + membership_id=membership_id, + ) + if cancellation_requested is not None and cancellation_requested(): + raise StoryArcPlacementIntegrationError( + "cancelled", + "Import-origin story-arc placement was cancelled before reservation", + category="cancelled", + ) + existing = ( + await session.get(StoryArcPlacement, existing_id) + if existing_id is not None + else await session.scalar( + select(StoryArcPlacement).where( + StoryArcPlacement.placement_path == str(preparation.target_path) + ) + ) + ) + if existing_id is None and existing is not None: + raise StoryArcPlacementIntegrationError( + "placement_concurrency_conflict", + "Another placement operation reserved this destination", + category="conflict", + ) + if existing_id is not None and existing is None: + raise StoryArcPlacementIntegrationError( + "prepared_placement_missing", + "Prepared story-arc placement record disappeared during preparation", + category="conflict", + ) + + operation_token = uuid4().hex + if existing is None: + existing = StoryArcPlacement( + issue_story_arc_id=membership_id, + library_file_id=context.library_file_id, + library_root_id=policy.target_library_root_id, + placement_path=str(preparation.target_path), + mode=_filesystem_mode(policy.mode), + ownership=StoryArcPlacementOwnership.MANAGED, + symlink_style=policy.symlink_style, + source_kind=StoryArcSourceKind.PULLBOX, + source_import_job_id=( + import_provenance.import_job_id if import_provenance is not None else None + ), + creating_action_id=( + import_provenance.import_action_id if import_provenance is not None else None + ), + rendered_reading_order=context.sequence_number, + policy_schema_version=STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + source_fingerprint=dict(preparation.source_fingerprint), + state=StoryArcPlacementState.MISSING, + last_result={}, + operation_token=operation_token, + ) + session.add(existing) + existing.last_result = { + "schema_version": 1, + "status": "prepared", + "operation_token": operation_token, + "prepared_evidence": _preparation_snapshot(preparation), + } + await session.flush() + prepared_placement_id = existing.id + else: + token_filter = ( + StoryArcPlacement.operation_token.is_(None) + if existing_operation_token is None + else StoryArcPlacement.operation_token == existing_operation_token + ) + reserve_result = await session.execute( + sa_update(StoryArcPlacement) + .where(StoryArcPlacement.id == existing.id, token_filter) + .values( + source_fingerprint=dict(preparation.source_fingerprint), + operation_token=operation_token, + last_result={ + **existing_last_result, + "schema_version": 1, + "status": "prepared", + "operation_token": operation_token, + "prepared_evidence": _preparation_snapshot(preparation), + }, + ) + ) + if reserve_result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_superseded", + "Another worker reserved this story-arc placement", + category="conflict", + ) + prepared_placement_id = existing.id + try: + await session.commit() + except IntegrityError as exc: + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_concurrency_conflict", + "Another placement operation reserved this destination", + category="conflict", + ) from exc + + caller_cancelled = False + try: + result, caller_cancelled = await _run_filesystem_call( + partial( + execute_story_arc_placement, + plan, + existing_managed=evidence, + preparation=preparation, + cancellation_requested=cancellation_requested, + ), + heartbeat=partial( + _refresh_operation_lease, + session, + placement_id=prepared_placement_id, + operation_token=operation_token, + ), + ) + except StoryArcPlacementError as exc: + await _persist_failure( + session, + membership_id=membership_id, + placement_id=prepared_placement_id, + operation_token=operation_token, + error=exc, + ) + raise _translate_filesystem_error(exc) from exc + except (OSError, ValueError) as exc: + error = StoryArcPlacementIntegrationError( + "placement_execution_failed", + "Story-arc placement execution failed safely", + category="safety", + ) + await _persist_failure( + session, + membership_id=membership_id, + placement_id=prepared_placement_id, + operation_token=operation_token, + error=error, + ) + raise error from exc + + synchronized, reconcile_cancelled = await _run_published_reconciliation( + session, + context=context, + policy=policy, + prepared_placement_id=prepared_placement_id, + operation_token=operation_token, + result=result, + ) + if caller_cancelled or reconcile_cancelled: + raise asyncio.CancelledError + return synchronized + + +async def validate_story_arc_placement_policy_input( + session: AsyncSession, + proposal: StoryArcPlacementPolicyInput, + *, + revision: int, +) -> StoryArcPlacementPolicy: + """Validate a complete policy for normal management or staged import confirmation.""" + try: + mode = StoryArcPlacementPolicyMode(proposal.mode) + except ValueError as exc: + raise StoryArcPlacementIntegrationError( + "unsupported_mode", + "Unsupported story-arc placement policy mode", + ) from exc + try: + validate_story_arc_folder_template(proposal.folder_template) + except ValueError as exc: + raise StoryArcPlacementIntegrationError( + "invalid_folder_template", + str(exc), + ) from exc + try: + validate_story_arc_file_template(proposal.file_template) + except ValueError as exc: + raise StoryArcPlacementIntegrationError( + "invalid_file_template", + str(exc), + ) from exc + if not isinstance(proposal.synchronize, bool): + raise StoryArcPlacementIntegrationError( + "invalid_synchronize_flag", + "Story-arc synchronize must be true or false", + ) + symlink_style: StoryArcSymlinkStyle | None = None + if proposal.symlink_style is not None: + try: + symlink_style = StoryArcSymlinkStyle(proposal.symlink_style) + except ValueError as exc: + raise StoryArcPlacementIntegrationError( + "unsupported_symlink_style", + "Unsupported story-arc symlink style", + ) from exc + if mode is StoryArcPlacementPolicyMode.SYMLINK and symlink_style is None: + raise StoryArcPlacementIntegrationError( + "symlink_style_required", + "Symlink placement policy requires an absolute or relative style", + ) + if mode is not StoryArcPlacementPolicyMode.SYMLINK and symlink_style is not None: + raise StoryArcPlacementIntegrationError( + "symlink_style_not_allowed", + "Only symlink placement policy may specify a symlink style", + ) + + if mode is StoryArcPlacementPolicyMode.LOGICAL: + if proposal.synchronize: + raise StoryArcPlacementIntegrationError( + "logical_policy_cannot_synchronize", + "Logical-only story arcs do not synchronize filesystem placements", + ) + if proposal.target_library_root_id is not None or proposal.destination_root is not None: + raise StoryArcPlacementIntegrationError( + "logical_policy_has_root", + "Logical-only story arcs must not configure a placement root", + ) + return StoryArcPlacementPolicy( + configured=True, + revision=revision, + mode=mode, + target_library_root_id=None, + destination_root=None, + folder_template=proposal.folder_template, + file_template=proposal.file_template, + symlink_style=None, + synchronize=proposal.synchronize, + ) + + root_id = proposal.target_library_root_id + if isinstance(root_id, bool) or not isinstance(root_id, int) or root_id < 1: + raise StoryArcPlacementIntegrationError( + "target_library_root_required", + "Placement policy requires a selected library root", + ) + library_root = await session.get(LibraryRoot, root_id) + if library_root is None: + raise StoryArcPlacementIntegrationError( + "target_library_root_unavailable", + "Selected story-arc library root is unavailable", + category="safety", + ) + try: + if mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY: + await validate_reference_library_root(library_root) + else: + await validate_managed_library_root(library_root) + except ValidationError as exc: + raise StoryArcPlacementIntegrationError( + "target_library_root_unavailable", + "Selected story-arc library root is unavailable", + category="safety", + ) from exc + raw_destination = proposal.destination_root + if raw_destination is None or not raw_destination.strip(): + raise StoryArcPlacementIntegrationError( + "destination_root_required", + "Placement policy requires an approved destination root", + ) + destination = Path(raw_destination) + if not destination.is_absolute(): + raise StoryArcPlacementIntegrationError( + "destination_root_not_absolute", + "Story-arc destination root must be absolute", + category="safety", + ) + if destination.is_symlink(): + raise StoryArcPlacementIntegrationError( + "symlink_root", + "Story-arc destination root cannot be a symbolic link", + category="safety", + ) + try: + resolved_destination = destination.resolve(strict=True) + except OSError as exc: + raise StoryArcPlacementIntegrationError( + "destination_root_unavailable", + "Story-arc destination root is unavailable", + category="safety", + ) from exc + if not resolved_destination.is_dir(): + raise StoryArcPlacementIntegrationError( + "destination_root_unavailable", + "Story-arc destination root is not a directory", + category="safety", + ) + try: + resolved_library_root = Path(library_root.path).resolve(strict=True) + except OSError as exc: + raise StoryArcPlacementIntegrationError( + "target_library_root_unavailable", + "Selected story-arc library root is unavailable", + category="safety", + ) from exc + if not resolved_library_root.is_dir(): + raise StoryArcPlacementIntegrationError( + "target_library_root_unavailable", + "Selected story-arc library root is unavailable", + category="safety", + ) + if not resolved_destination.is_relative_to(resolved_library_root): + raise StoryArcPlacementIntegrationError( + "destination_root_outside_library_root", + "Story-arc destination root must be within the selected library root", + category="safety", + ) + return StoryArcPlacementPolicy( + configured=True, + revision=revision, + mode=mode, + target_library_root_id=root_id, + destination_root=str(resolved_destination), + folder_template=proposal.folder_template, + file_template=proposal.file_template, + symlink_style=symlink_style, + synchronize=proposal.synchronize, + ) + + +def _is_positive_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +async def _validate_import_provenance( + session: AsyncSession, + *, + provenance: StoryArcPlacementImportProvenance, + story_arc_id: int, + membership_id: int, +) -> None: + """Fail closed unless an active import action exactly owns this request.""" + if not _is_positive_int(provenance.import_job_id) or not _is_positive_int( + provenance.import_action_id + ): + raise StoryArcPlacementIntegrationError( + "import_placement_provenance_invalid", + "Import placement provenance requires positive job and action identifiers", + category="ownership", + ) + action = await session.get(ImportJobAction, provenance.import_action_id) + job = await session.get(ImportJob, provenance.import_job_id) + if ( + action is None + or job is None + or action.import_job_id != job.id + or action.phase != _IMPORT_PLACEMENT_PHASE + or action.action_type != _IMPORT_PLACEMENT_ACTION_TYPE + or action.status is not ImportJobActionStatus.COMPLETED + ): + raise StoryArcPlacementIntegrationError( + "import_placement_provenance_invalid", + "Import placement action is missing, inactive, or belongs to another job", + category="ownership", + ) + payload = dict(action.payload or {}) + if ( + set(payload) != _IMPORT_PLACEMENT_PAYLOAD_KEYS + or payload.get("schema_version") != 1 + or payload.get("membership_id") != membership_id + or payload.get("source_import_job_id") != job.id + or not _is_positive_int(payload.get("sync_work_id")) + or not _is_positive_int(payload.get("imported_story_arc_id")) + or not _is_positive_int(payload.get("imported_story_arc_entry_id")) + or not isinstance(payload.get("desired_generation"), str) + or len(str(payload.get("desired_generation"))) != 64 + ): + raise StoryArcPlacementIntegrationError( + "import_placement_payload_invalid", + "Import placement action payload does not match the requested membership", + category="ownership", + ) + if ( + job.status is not ImportJobStatus.IMPORTING + or job.control_request is not ImportControlRequest.NONE + or dict(job.progress_snapshot or {}).get("phase") != _IMPORT_PLACEMENT_PHASE + ): + raise StoryArcPlacementIntegrationError( + "import_placement_job_inactive", + "Import job is not actively publishing Story Arc placements", + category="cancelled", + ) + membership_arc_id = await session.scalar( + select(IssueStoryArc.story_arc_id).where(IssueStoryArc.id == membership_id) + ) + if membership_arc_id != story_arc_id: + raise StoryArcPlacementIntegrationError( + "import_placement_membership_changed", + "Import placement membership no longer belongs to the requested Story Arc", + category="ownership", + ) + + +def _policy_from_arc(arc: StoryArc) -> StoryArcPlacementPolicy: + raw = dict(arc.policy_snapshot or {}) + if arc.policy_schema_version != STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION: + return _logical_default(arc.revision) + if set(raw) != _POLICY_SNAPSHOT_KEYS or raw.get("schema_version") != 1: + return _logical_default(arc.revision) + try: + mode = StoryArcPlacementPolicyMode(str(raw["mode"])) + raw_style = raw["symlink_style"] + style = StoryArcSymlinkStyle(str(raw_style)) if raw_style is not None else None + root_id_raw = raw["target_library_root_id"] + if root_id_raw is not None and ( + isinstance(root_id_raw, bool) or not isinstance(root_id_raw, int) + ): + raise ValueError + root_id = root_id_raw + destination_raw = raw["destination_root"] + if destination_raw is not None and not isinstance(destination_raw, str): + raise ValueError + destination = destination_raw + folder_template_raw = raw["folder_template"] + file_template_raw = raw["file_template"] + if not isinstance(folder_template_raw, str) or not isinstance(file_template_raw, str): + raise ValueError + folder_template = folder_template_raw + file_template = file_template_raw + synchronize_raw = raw["synchronize"] + if not isinstance(synchronize_raw, bool): + raise ValueError + validate_story_arc_folder_template(folder_template) + validate_story_arc_file_template(file_template) + if mode is StoryArcPlacementPolicyMode.SYMLINK and style is None: + raise ValueError + if mode is not StoryArcPlacementPolicyMode.SYMLINK and style is not None: + raise ValueError + if mode is StoryArcPlacementPolicyMode.LOGICAL: + if root_id is not None or destination is not None or synchronize_raw: + raise ValueError + elif root_id is None or destination is None or not Path(destination).is_absolute(): + raise ValueError + except (KeyError, TypeError, ValueError): + return _logical_default(arc.revision) + return StoryArcPlacementPolicy( + configured=True, + revision=arc.revision, + mode=mode, + target_library_root_id=root_id, + destination_root=destination, + folder_template=folder_template, + file_template=file_template, + symlink_style=style, + synchronize=synchronize_raw, + ) + + +def _logical_default(revision: int) -> StoryArcPlacementPolicy: + return StoryArcPlacementPolicy( + configured=False, + revision=revision, + mode=StoryArcPlacementPolicyMode.LOGICAL, + target_library_root_id=None, + destination_root=None, + folder_template=DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + file_template=DEFAULT_STORY_ARC_FILE_TEMPLATE, + symlink_style=None, + synchronize=False, + ) + + +async def _policy_root_is_available( + session: AsyncSession, + policy: StoryArcPlacementPolicy, +) -> bool: + if policy.mode is StoryArcPlacementPolicyMode.LOGICAL: + return True + if policy.target_library_root_id is None: + return False + root = await session.get(LibraryRoot, policy.target_library_root_id) + if root is None: + return False + try: + if policy.mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY: + await validate_reference_library_root(root) + else: + await validate_managed_library_root(root) + except ValidationError: + return False + return True + + +def _placement_policy_shape( + policy: StoryArcPlacementPolicy, +) -> tuple[object, ...]: + return ( + policy.mode, + policy.target_library_root_id, + policy.destination_root, + policy.folder_template, + policy.file_template, + policy.symlink_style, + ) + + +async def _arc_has_managed_placements( + session: AsyncSession, + story_arc_id: int, +) -> bool: + placement_id = await session.scalar( + select(StoryArcPlacement.id) + .join(IssueStoryArc, StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.MANAGED, + ) + .limit(1) + ) + return placement_id is not None + + +async def _load_context_page( + session: AsyncSession, + story_arc_id: int, + *, + limit: int, + offset: int, +) -> tuple[int, tuple[_PlacementContext, ...]]: + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise _not_found("story_arc_not_found", "Story arc was not found") + total = int( + await session.scalar( + select(func.count(IssueStoryArc.id)).where(IssueStoryArc.story_arc_id == story_arc_id) + ) + or 0 + ) + rows = ( + await session.execute( + select(IssueStoryArc, Issue, Series, Publisher) + .outerjoin(Issue, IssueStoryArc.issue_id == Issue.id) + .outerjoin(Series, Issue.series_id == Series.id) + .outerjoin(Publisher, Series.publisher_id == Publisher.id) + .where(IssueStoryArc.story_arc_id == story_arc_id) + .order_by( + IssueStoryArc.sequence_number.asc(), + IssueStoryArc.source_ordinal.asc(), + IssueStoryArc.id.asc(), + ) + .limit(limit) + .offset(offset) + ) + ).all() + issue_ids = [issue.id for _membership, issue, _series, _publisher in rows if issue is not None] + files_by_issue: dict[int, LibraryFile] = {} + if issue_ids: + files = list( + ( + await session.scalars( + select(LibraryFile) + .where(LibraryFile.issue_id.in_(issue_ids)) + .order_by(LibraryFile.issue_id.asc(), LibraryFile.id.asc()) + ) + ).all() + ) + for library_file in files: + if library_file.issue_id is not None: + files_by_issue.setdefault(library_file.issue_id, library_file) + return total, tuple( + _context_from_row( + arc, + membership, + issue, + series, + publisher, + files_by_issue.get(issue.id) if issue is not None else None, + ) + for membership, issue, series, publisher in rows + ) + + +async def _load_one_context( + session: AsyncSession, + story_arc_id: int, + membership_id: int, +) -> _PlacementContext: + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise _not_found("story_arc_not_found", "Story arc was not found") + row = ( + await session.execute( + select(IssueStoryArc, Issue, Series, Publisher) + .outerjoin(Issue, IssueStoryArc.issue_id == Issue.id) + .outerjoin(Series, Issue.series_id == Series.id) + .outerjoin(Publisher, Series.publisher_id == Publisher.id) + .where( + IssueStoryArc.id == membership_id, + IssueStoryArc.story_arc_id == story_arc_id, + ) + ) + ).one_or_none() + if row is None: + raise _not_found("membership_not_found", "Story-arc membership was not found") + membership, issue, series, publisher = row + library_file = None + if issue is not None: + library_file = await session.scalar( + select(LibraryFile) + .where(LibraryFile.issue_id == issue.id) + .order_by(LibraryFile.id.asc()) + .limit(1) + ) + return _context_from_row(arc, membership, issue, series, publisher, library_file) + + +def _context_from_row( + arc: StoryArc, + membership: IssueStoryArc, + issue: Issue | None, + series: Series | None, + publisher: Publisher | None, + library_file: LibraryFile | None, +) -> _PlacementContext: + exact_number = ( + issue.effective_issue_number_text + if issue is not None + else membership.source_issue_number_text or "unknown" + ) + extension = ( + library_file.file_format.value + if library_file is not None + else Path(library_file.file_path).suffix.lstrip(".").lower() + if library_file is not None + else "cbz" + ) + return _PlacementContext( + membership_id=membership.id, + story_arc_id=membership.story_arc_id, + sequence_number=membership.sequence_number, + issue_id=issue.id if issue is not None else None, + issue_number_text=exact_number, + story_arc_name=arc.name, + series_name=( + series.title + if series is not None + else membership.source_series_name or "Unknown Series" + ), + publisher_name=(publisher.name if publisher is not None else membership.source_publisher), + issue_title=issue.title if issue is not None else membership.source_issue_title, + year=( + issue.release_date.year + if issue is not None and issue.release_date is not None + else series.year_start + if series is not None + else None + ), + series_start_year=series.year_start if series is not None else None, + series_end_year=series.year_end if series is not None else None, + library_file_id=library_file.id if library_file is not None else None, + canonical_path=library_file.file_path if library_file is not None else None, + extension=extension, + ) + + +def _rendered_target_paths( + contexts: Sequence[_PlacementContext], + policy: StoryArcPlacementPolicy, +) -> dict[int, str]: + """Render only the targets represented by the already-bounded context page.""" + if policy.mode is StoryArcPlacementPolicyMode.LOGICAL or policy.destination_root is None: + return {} + return { + context.membership_id: target + for context in contexts + if (target := _rendered_preview_target_path(context, policy)) is not None + } + + +def _rendered_preview_target_path( + context: _PlacementContext, + policy: StoryArcPlacementPolicy, +) -> str | None: + try: + return _rendered_target_path(context, policy) + except StoryArcOriginalFilenameError: + # The filesystem inspection reports missing/unsafe canonical names as + # blocked rows, without guessing a filename before acquisition. + return None + + +def _rendered_target_path( + context: _PlacementContext, + policy: StoryArcPlacementPolicy, +) -> str: + if policy.destination_root is None: + raise ValueError("Placement policy has no destination root") + return str( + Path(policy.destination_root) + / render_story_arc_relative_path( + context.naming_values(), + folder_template=policy.folder_template, + file_template=policy.file_template, + ) + ) + + +async def _load_matching_preview_evidence( + session: AsyncSession, + target_paths: dict[int, str], +) -> dict[str, _PreviewPlacementEvidence]: + """Load at most one unique placement row per target on the bounded page.""" + if not target_paths: + return {} + rows = list( + ( + await session.scalars( + select(StoryArcPlacement) + .where(StoryArcPlacement.placement_path.in_(tuple(target_paths.values()))) + .order_by(StoryArcPlacement.id.asc()) + ) + ).all() + ) + return { + row.placement_path: _PreviewPlacementEvidence( + id=row.id, + issue_story_arc_id=row.issue_story_arc_id, + placement_path=row.placement_path, + mode=row.mode, + ownership=row.ownership, + symlink_style=row.symlink_style, + source_fingerprint=dict(row.source_fingerprint or {}), + target_fingerprint=_stored_target_fingerprint(row), + creating_action_id=row.creating_action_id, + ) + for row in rows + } + + +def _stored_target_fingerprint(row: StoryArcPlacement) -> dict[str, object]: + target_fingerprint = dict(row.last_result or {}).get("target_fingerprint") + return dict(target_fingerprint) if isinstance(target_fingerprint, dict) else {} + + +def _preview_contexts( + contexts: Sequence[_PlacementContext], + policy: StoryArcPlacementPolicy, + evidence_by_path: dict[str, _PreviewPlacementEvidence], +) -> tuple[StoryArcPlacementPreviewItem, ...]: + items: list[StoryArcPlacementPreviewItem] = [] + for context in contexts: + if policy.mode is StoryArcPlacementPolicyMode.LOGICAL: + items.append( + StoryArcPlacementPreviewItem( + membership_id=context.membership_id, + sequence_number=context.sequence_number, + issue_id=context.issue_id, + issue_number_text=context.issue_number_text, + mode=policy.mode.value, + state=StoryArcPlacementPreviewState.LOGICAL_ONLY.value, + target_path=None, + collision=StoryArcCollisionKind.NONE.value, + reason=None, + required_bytes=0, + proposed_ownership=StoryArcPlacementOwnership.REFERENCED.value, + overwrite_allowed=False, + classification="logical_only", + placement_id=None, + current_ownership=None, + inspection_code=None, + ) + ) + continue + preview_mode = ( + StoryArcPlacementMode.COPY + if policy.mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY + else _filesystem_mode(policy.mode) + ) + plan = StoryArcPlacementPlan( + issue_story_arc_id=context.membership_id, + library_file_id=context.library_file_id, + canonical_path=Path(context.canonical_path) if context.canonical_path else None, + destination_root=Path(policy.destination_root) if policy.destination_root else None, + values=context.naming_values(), + mode=preview_mode, + symlink_style=policy.symlink_style, + folder_template=policy.folder_template, + file_template=policy.file_template, + ) + rendered_target = _rendered_preview_target_path(context, policy) + evidence = evidence_by_path.get(rendered_target) if rendered_target else None + if evidence is not None and evidence.issue_story_arc_id != context.membership_id: + items.append(_preview_destination_conflict(context, policy, evidence)) + continue + inspection = inspect_story_arc_placement( + plan, + existing=_inspection_evidence(evidence), + ) + items.append(_preview_item_from_inspection(context, policy, evidence, inspection)) + return tuple(items) + + +def _inspection_evidence( + evidence: _PreviewPlacementEvidence | None, +) -> StoryArcPlacementInspectionEvidence | None: + if evidence is None: + return None + return StoryArcPlacementInspectionEvidence( + placement_path=Path(evidence.placement_path), + mode=evidence.mode, + ownership=evidence.ownership, + symlink_style=evidence.symlink_style, + source_fingerprint=dict(evidence.source_fingerprint), + target_fingerprint=dict(evidence.target_fingerprint), + ) + + +def _preview_item_from_inspection( + context: _PlacementContext, + policy: StoryArcPlacementPolicy, + evidence: _PreviewPlacementEvidence | None, + inspection: StoryArcPlacementInspection, +) -> StoryArcPlacementPreviewItem: + classification = inspection.state.value + if inspection.state is StoryArcPlacementInspectionState.FREE: + classification = ( + "reference_missing" + if policy.mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY + else "will_materialize" + ) + state = _legacy_inspection_state(inspection.state, policy) + collision = _inspection_collision(inspection, policy) + reason = inspection.reason + if classification == "reference_missing": + reason = "No existing artifact is available to reference" + elif inspection.state is StoryArcPlacementInspectionState.UNTRACKED_IDENTICAL: + reason = "An identical user artifact requires explicit referenced-placement review" + return StoryArcPlacementPreviewItem( + membership_id=context.membership_id, + sequence_number=context.sequence_number, + issue_id=context.issue_id, + issue_number_text=context.issue_number_text, + mode=policy.mode.value, + state=state, + target_path=str(inspection.target_path) if inspection.target_path is not None else None, + collision=collision, + reason=reason, + required_bytes=( + 0 + if policy.mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY + else inspection.required_bytes + ), + proposed_ownership=( + StoryArcPlacementOwnership.REFERENCED.value + if policy.mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY + or inspection.state is StoryArcPlacementInspectionState.UNTRACKED_IDENTICAL + else inspection.proposed_ownership + ), + overwrite_allowed=False, + classification=classification, + placement_id=evidence.id if evidence is not None else None, + current_ownership=(evidence.ownership.value if evidence is not None else None), + inspection_code=inspection.code, + ) + + +def _legacy_inspection_state( + state: StoryArcPlacementInspectionState, + policy: StoryArcPlacementPolicy, +) -> str: + if state in { + StoryArcPlacementInspectionState.MANAGED_CURRENT, + StoryArcPlacementInspectionState.REFERENCED_CURRENT, + }: + return StoryArcPlacementPreviewState.ALREADY_REPRESENTED.value + if state in { + StoryArcPlacementInspectionState.MANAGED_MISSING, + StoryArcPlacementInspectionState.REFERENCED_MISSING, + } or ( + state is StoryArcPlacementInspectionState.FREE + and policy.mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY + ): + return "missing" + if state in { + StoryArcPlacementInspectionState.MANAGED_DRIFTED, + StoryArcPlacementInspectionState.REFERENCED_DRIFTED, + }: + return "drifted" + if state is StoryArcPlacementInspectionState.FREE: + return StoryArcPlacementPreviewState.READY.value + return StoryArcPlacementPreviewState.BLOCKED.value + + +def _inspection_collision( + inspection: StoryArcPlacementInspection, + policy: StoryArcPlacementPolicy, +) -> str: + if inspection.state in { + StoryArcPlacementInspectionState.MANAGED_CURRENT, + StoryArcPlacementInspectionState.REFERENCED_CURRENT, + StoryArcPlacementInspectionState.FREE, + }: + if ( + inspection.state is StoryArcPlacementInspectionState.FREE + and policy.mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY + ): + return "reference_missing" + return StoryArcCollisionKind.NONE.value + if inspection.state is StoryArcPlacementInspectionState.MANAGED_MISSING: + return "managed_missing" + if inspection.state is StoryArcPlacementInspectionState.REFERENCED_MISSING: + return "reference_missing" + if inspection.state is StoryArcPlacementInspectionState.UNTRACKED_IDENTICAL: + return "identical_unconfirmed" + if ( + inspection.state + in { + StoryArcPlacementInspectionState.MANAGED_DRIFTED, + StoryArcPlacementInspectionState.REFERENCED_DRIFTED, + } + and inspection.source_fingerprint is not None + and inspection.target_fingerprint is not None + and not _inspection_content_matches(inspection) + ): + return StoryArcCollisionKind.DIFFERENT_CONTENT.value + if ( + inspection.state + in { + StoryArcPlacementInspectionState.MANAGED_DRIFTED, + StoryArcPlacementInspectionState.REFERENCED_DRIFTED, + } + and inspection.collision + in { + StoryArcCollisionKind.NONE, + StoryArcCollisionKind.SAME_INODE_REFERENCED, + } + and inspection.code is not None + ): + return inspection.code + return inspection.collision.value + + +def _inspection_content_matches(inspection: StoryArcPlacementInspection) -> bool: + source = inspection.source_fingerprint or {} + target = inspection.target_fingerprint or {} + target_content = target.get("content") if target.get("kind") == "symlink" else target + return bool( + isinstance(target_content, dict) + and source.get("sha256") is not None + and source.get("sha256") == target_content.get("sha256") + ) + + +def _preview_destination_conflict( + context: _PlacementContext, + policy: StoryArcPlacementPolicy, + evidence: _PreviewPlacementEvidence, +) -> StoryArcPlacementPreviewItem: + return StoryArcPlacementPreviewItem( + membership_id=context.membership_id, + sequence_number=context.sequence_number, + issue_id=context.issue_id, + issue_number_text=context.issue_number_text, + mode=policy.mode.value, + state=StoryArcPlacementPreviewState.BLOCKED.value, + target_path=evidence.placement_path, + collision="placement_destination_conflict", + reason="Story-arc destination is already tracked by another membership", + required_bytes=0, + proposed_ownership=( + StoryArcPlacementOwnership.REFERENCED.value + if policy.mode is StoryArcPlacementPolicyMode.REFERENCE_ONLY + else StoryArcPlacementOwnership.MANAGED.value + ), + overwrite_allowed=False, + classification="blocked", + placement_id=evidence.id, + current_ownership=evidence.ownership.value, + inspection_code="placement_destination_conflict", + ) + + +async def _run_filesystem_call[FilesystemResultT]( + call: Callable[[], FilesystemResultT], + *, + heartbeat: Callable[[], Awaitable[None]] | None = None, +) -> tuple[FilesystemResultT, bool]: + """Let a worker reach a safe boundary before propagating task cancellation. + + Cancelling an await on ``asyncio.to_thread`` does not stop its thread. A + copy could otherwise publish after the coroutine had abandoned its + prepared row. Shield the worker, remember cancellation, and let the caller + persist publish evidence and reconcile before cancellation is re-raised. + """ + worker = asyncio.create_task(asyncio.to_thread(call)) + caller_cancelled = False + while True: + try: + done, _pending = await asyncio.wait( + {worker}, + timeout=_PLACEMENT_HEARTBEAT_SECONDS, + ) + if done: + return worker.result(), caller_cancelled + if heartbeat is not None: + await heartbeat() + except asyncio.CancelledError: + caller_cancelled = True + + +async def _run_published_reconciliation( + session: AsyncSession, + *, + context: _PlacementContext, + policy: StoryArcPlacementPolicy, + prepared_placement_id: int, + operation_token: str, + result: StoryArcPlacementResult, +) -> tuple[StoryArcPlacementSyncResult, bool]: + """Finish durable checkpoint/reconcile before propagating cancellation.""" + + async def finish() -> StoryArcPlacementSyncResult: + await _persist_published_checkpoint( + session, + context=context, + policy=policy, + prepared_placement_id=prepared_placement_id, + operation_token=operation_token, + result=result, + ) + return await _reconcile_result( + session, + context=context, + policy=policy, + prepared_placement_id=prepared_placement_id, + operation_token=operation_token, + result=result, + ) + + worker = asyncio.create_task(finish()) + caller_cancelled = False + while True: + try: + return await asyncio.shield(worker), caller_cancelled + except asyncio.CancelledError: + caller_cancelled = True + + +async def _sync_reference_only( + session: AsyncSession, + *, + context: _PlacementContext, + policy: StoryArcPlacementPolicy, + membership: IssueStoryArc, + adopt_identical_existing: bool, + cancellation_requested: Callable[[], bool] | None, +) -> StoryArcPlacementSyncResult: + """Validate and explicitly adopt an existing artifact without creating one.""" + if policy.destination_root is None: + raise StoryArcPlacementIntegrationError( + "destination_root_required", + "Reference-only placement policy requires a destination root", + ) + target_path = Path(policy.destination_root) / render_story_arc_relative_path( + context.naming_values(), + folder_template=policy.folder_template, + file_template=policy.file_template, + ) + existing = await session.scalar( + select(StoryArcPlacement).where(StoryArcPlacement.placement_path == str(target_path)) + ) + if existing is not None and existing.issue_story_arc_id != context.membership_id: + raise StoryArcPlacementIntegrationError( + "placement_destination_conflict", + "Story-arc destination is already tracked by another membership", + category="conflict", + ) + if existing is not None and existing.ownership is StoryArcPlacementOwnership.MANAGED: + raise StoryArcPlacementIntegrationError( + "managed_placement_requires_repair", + "A managed placement cannot be converted to referenced implicitly", + category="ownership", + ) + if existing is None and not adopt_identical_existing: + raise StoryArcPlacementIntegrationError( + "reference_adoption_required", + "Adopting an existing user artifact requires explicit confirmation", + category="conflict", + ) + + reference_operation_token: str | None = None + if existing is not None: + observed_token = existing.operation_token + if ( + observed_token is not None + and existing.updated_at > datetime.now(UTC) - _PLACEMENT_OPERATION_LEASE + ): + raise StoryArcPlacementIntegrationError( + "placement_operation_in_progress", + "Referenced placement evidence is being updated by another operation", + category="conflict", + ) + reference_operation_token = uuid4().hex + token_filter = ( + StoryArcPlacement.operation_token.is_(None) + if observed_token is None + else StoryArcPlacement.operation_token == observed_token + ) + previous_result = dict(existing.last_result or {}) + reserve = await session.execute( + sa_update(StoryArcPlacement) + .where( + StoryArcPlacement.id == existing.id, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.REFERENCED, + token_filter, + ) + .values( + operation_token=reference_operation_token, + last_result={ + **previous_result, + "schema_version": 1, + "status": "reference_validation_prepared", + "operation_token": reference_operation_token, + }, + ) + ) + if reserve.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_superseded", + "Another worker reserved this referenced placement", + category="conflict", + ) + + # No database transaction remains open during read-only filesystem validation. + await session.commit() + + def target_exists() -> bool: + return target_path.exists() or target_path.is_symlink() + + if not await asyncio.to_thread(target_exists): + error = StoryArcPlacementIntegrationError( + "reference_target_missing", + "No existing artifact is available to reference", + category="safety", + ) + await _persist_reference_failure( + session, + membership.id, + error, + placement_id=existing.id if existing is not None else None, + operation_token=reference_operation_token, + ) + raise error + + def adoption_cancelled() -> bool: + return bool( + (cancellation_requested is not None and cancellation_requested()) or not target_exists() + ) + + def adoption_journal(event: StoryArcPlacementJournalEvent) -> None: + # Existing-target resolution returns before a publish journal event. If + # execution ever reaches preparation, the target changed and this + # reference-only operation must fail before creating anything. + if event.stage == "prepared": + raise StoryArcPlacementSafetyError( + "reference_target_changed", + "Referenced story-arc artifact changed during validation", + ) + + plan = StoryArcPlacementPlan( + issue_story_arc_id=context.membership_id, + library_file_id=context.library_file_id, + canonical_path=Path(context.canonical_path) if context.canonical_path else None, + destination_root=Path(policy.destination_root), + values=context.naming_values(), + mode=StoryArcPlacementMode.COPY, + folder_template=policy.folder_template, + file_template=policy.file_template, + adopt_identical_existing=True, + ) + caller_cancelled = False + try: + result, caller_cancelled = await _run_filesystem_call( + partial( + execute_story_arc_placement, + plan, + cancellation_requested=adoption_cancelled, + journal=adoption_journal, + ) + ) + except StoryArcPlacementError as exc: + await _persist_reference_failure( + session, + membership.id, + exc, + placement_id=existing.id if existing is not None else None, + operation_token=reference_operation_token, + ) + raise _translate_filesystem_error(exc) from exc + if result.ownership is not StoryArcPlacementOwnership.REFERENCED: + error = StoryArcPlacementIntegrationError( + "reference_materialization_forbidden", + "Reference-only synchronization refused to create an artifact", + category="safety", + ) + await _persist_reference_failure( + session, + membership.id, + error, + placement_id=existing.id if existing is not None else None, + operation_token=reference_operation_token, + ) + raise error + + placement = await session.scalar( + select(StoryArcPlacement).where(StoryArcPlacement.placement_path == str(target_path)) + ) + if existing is not None and ( + placement is None + or placement.id != existing.id + or placement.operation_token != reference_operation_token + ): + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_superseded", + "Referenced placement validation lost its ownership lease", + category="conflict", + ) + if placement is None: + placement = StoryArcPlacement( + issue_story_arc_id=context.membership_id, + library_file_id=context.library_file_id, + library_root_id=policy.target_library_root_id, + placement_path=str(target_path), + mode=StoryArcPlacementMode.REFERENCE_ONLY, + ownership=StoryArcPlacementOwnership.REFERENCED, + symlink_style=None, + source_kind=StoryArcSourceKind.PULLBOX, + rendered_reading_order=context.sequence_number, + policy_schema_version=STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + source_fingerprint=dict(result.source_fingerprint), + state=StoryArcPlacementState.CURRENT, + last_result={}, + ) + session.add(placement) + placement.library_file_id = context.library_file_id + placement.library_root_id = policy.target_library_root_id + placement.source_fingerprint = dict(result.source_fingerprint) + placement.state = StoryArcPlacementState.CURRENT + placement.last_checked_at = datetime.now(UTC) + placement.operation_token = None + placement.last_result = { + "schema_version": 1, + "status": "complete", + "outcome": result.state.value, + "target_fingerprint": dict(result.target_fingerprint), + } + await session.execute( + sa_update(IssueStoryArc) + .where(IssueStoryArc.id == context.membership_id) + .values( + sync_eligible=policy.synchronize, + last_materialization_result={ + "schema_version": 1, + "status": "complete", + "outcome": result.state.value, + }, + ) + ) + await session.commit() + synchronized = StoryArcPlacementSyncResult( + membership_id=context.membership_id, + outcome=result.state.value, + placement=_placement_view(placement), + ) + if caller_cancelled: + raise asyncio.CancelledError + return synchronized + + +async def _persist_reference_failure( + session: AsyncSession, + membership_id: int, + error: StoryArcPlacementError | StoryArcPlacementIntegrationError, + *, + placement_id: int | None = None, + operation_token: str | None = None, +) -> None: + failure = { + "schema_version": 1, + "status": "failed", + "error_code": error.code, + "error_category": _error_category(error), + "message": str(error), + } + if placement_id is not None: + placement = await session.get(StoryArcPlacement, placement_id) + if placement is not None and placement.ownership is StoryArcPlacementOwnership.REFERENCED: + await session.refresh(placement) + if operation_token is not None and placement.operation_token != operation_token: + await session.rollback() + return + previous = dict(placement.last_result or {}) + if isinstance(previous.get("target_fingerprint"), dict): + failure["target_fingerprint"] = dict(previous["target_fingerprint"]) + placement.state = ( + StoryArcPlacementState.MISSING + if error.code == "reference_target_missing" + else StoryArcPlacementState.DRIFTED + ) + placement.last_checked_at = datetime.now(UTC) + placement.operation_token = None + placement.last_result = failure + membership = await session.get(IssueStoryArc, membership_id) + if membership is not None: + membership.last_materialization_result = failure + await session.commit() + + +def _build_plan( + context: _PlacementContext, + policy: StoryArcPlacementPolicy, + *, + adopt_identical_existing: bool, +) -> StoryArcPlacementPlan: + return StoryArcPlacementPlan( + issue_story_arc_id=context.membership_id, + library_file_id=context.library_file_id, + canonical_path=Path(context.canonical_path) if context.canonical_path else None, + destination_root=Path(policy.destination_root) if policy.destination_root else None, + values=context.naming_values(), + mode=_filesystem_mode(policy.mode), + symlink_style=policy.symlink_style, + folder_template=policy.folder_template, + file_template=policy.file_template, + adopt_identical_existing=adopt_identical_existing, + ) + + +def _filesystem_mode(mode: StoryArcPlacementPolicyMode) -> StoryArcPlacementMode: + if mode is StoryArcPlacementPolicyMode.LOGICAL: + return StoryArcPlacementMode.REFERENCE_ONLY + return StoryArcPlacementMode(mode.value) + + +def _require_published_operation_token( + placement: StoryArcPlacement, + operation_token: str, +) -> None: + """Authorize takeover only from the exact durable post-publish checkpoint.""" + last_result = dict(placement.last_result or {}) + target_fingerprint = last_result.get("target_fingerprint") + if ( + not operation_token + or placement.operation_token != operation_token + or last_result.get("schema_version") != 1 + or last_result.get("status") != "published_pending_reconcile" + or last_result.get("operation_token") != operation_token + or not isinstance(target_fingerprint, dict) + or not target_fingerprint + ): + raise StoryArcPlacementIntegrationError( + "placement_published_operation_token_invalid", + "Placement operation is not an exact abandoned published checkpoint", + category="conflict", + ) + + +def _managed_evidence( + placement: StoryArcPlacement, +) -> ManagedStoryArcPlacementEvidence | None: + if placement.ownership is not StoryArcPlacementOwnership.MANAGED: + return None + target_fingerprint = dict(placement.last_result or {}).get("target_fingerprint") + if not placement.source_fingerprint or not isinstance(target_fingerprint, dict): + return None + # Imported placements retain the import action id. For normal sync, the + # committed placement row itself is the durable ownership record, and its + # positive id is used only as the filesystem service's ownership token. + ownership_token = placement.creating_action_id or placement.id + return ManagedStoryArcPlacementEvidence( + issue_story_arc_id=placement.issue_story_arc_id, + placement_path=Path(placement.placement_path), + mode=placement.mode, + ownership=placement.ownership, + symlink_style=placement.symlink_style, + source_fingerprint=dict(placement.source_fingerprint), + target_fingerprint=dict(target_fingerprint), + creating_action_id=ownership_token, + ) + + +def _managed_removal_evidence( + placement: StoryArcPlacement, +) -> ManagedStoryArcPlacementEvidence | None: + """Build removal-only evidence, using an empty fingerprint as absence proof. + + The filesystem removal service checks secure path absence before comparing + fingerprints. Thus ``{}`` can prove only that an absent target is safe to + forget; any existing target necessarily mismatches and is preserved. + """ + if placement.ownership is not StoryArcPlacementOwnership.MANAGED: + return None + raw_target_fingerprint = dict(placement.last_result or {}).get("target_fingerprint") + if raw_target_fingerprint is None: + target_fingerprint: dict[str, object] = {} + elif isinstance(raw_target_fingerprint, dict): + target_fingerprint = dict(raw_target_fingerprint) + else: + return None + if not placement.source_fingerprint: + return None + ownership_token = placement.creating_action_id or placement.id + return ManagedStoryArcPlacementEvidence( + issue_story_arc_id=placement.issue_story_arc_id, + placement_path=Path(placement.placement_path), + mode=placement.mode, + ownership=placement.ownership, + symlink_style=placement.symlink_style, + source_fingerprint=dict(placement.source_fingerprint), + target_fingerprint=target_fingerprint, + creating_action_id=ownership_token, + ) + + +def _preparation_snapshot( + preparation: StoryArcPlacementPreparation, +) -> dict[str, object]: + return { + "schema_version": 1, + "issue_story_arc_id": preparation.issue_story_arc_id, + "placement_path": str(preparation.target_path), + "mode": preparation.mode.value, + "symlink_style": ( + preparation.symlink_style.value if preparation.symlink_style is not None else None + ), + "rendered_reading_order": preparation.rendered_reading_order, + "source_fingerprint": dict(preparation.source_fingerprint), + "destination_root_fingerprint": dict(preparation.destination_root_fingerprint), + } + + +def _prepared_managed_evidence( + placement: StoryArcPlacement, +) -> PreparedManagedStoryArcPlacementEvidence | None: + last_result = dict(placement.last_result or {}) + if last_result.get("status") != "prepared": + return None + operation_token = last_result.get("operation_token") + raw = last_result.get("prepared_evidence") + if ( + not isinstance(operation_token, str) + or not operation_token + or operation_token != placement.operation_token + or not isinstance(raw, dict) + ): + return None + try: + issue_story_arc_id = raw["issue_story_arc_id"] + placement_path = raw["placement_path"] + mode = raw["mode"] + symlink_style = raw["symlink_style"] + source_fingerprint = raw["source_fingerprint"] + root_fingerprint = raw["destination_root_fingerprint"] + if ( + isinstance(issue_story_arc_id, bool) + or not isinstance(issue_story_arc_id, int) + or not isinstance(placement_path, str) + or not isinstance(mode, str) + or (symlink_style is not None and not isinstance(symlink_style, str)) + or not isinstance(source_fingerprint, dict) + or not isinstance(root_fingerprint, dict) + ): + raise ValueError + parsed_mode = StoryArcPlacementMode(mode) + parsed_style = StoryArcSymlinkStyle(symlink_style) if symlink_style is not None else None + except (KeyError, TypeError, ValueError): + return None + if ( + issue_story_arc_id != placement.issue_story_arc_id + or placement_path != placement.placement_path + or parsed_mode is not placement.mode + or parsed_style is not placement.symlink_style + or source_fingerprint != dict(placement.source_fingerprint or {}) + ): + return None + return PreparedManagedStoryArcPlacementEvidence( + issue_story_arc_id=issue_story_arc_id, + placement_path=Path(placement_path), + mode=parsed_mode, + symlink_style=parsed_style, + source_fingerprint=dict(source_fingerprint), + destination_root_fingerprint=dict(root_fingerprint), + operation_token=operation_token, + ) + + +async def _refresh_operation_lease( + session: AsyncSession, + *, + placement_id: int, + operation_token: str, +) -> None: + result = await session.execute( + sa_update(StoryArcPlacement) + .where( + StoryArcPlacement.id == placement_id, + StoryArcPlacement.operation_token == operation_token, + ) + .values(updated_at=datetime.now(UTC)) + ) + if result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_superseded", + "Story-arc placement operation lost its ownership lease", + category="conflict", + ) + await session.commit() + + +async def _persist_removal_failure( + session: AsyncSession, + *, + placement_id: int, + operation_token: str, + error: StoryArcPlacementError | StoryArcPlacementIntegrationError, +) -> None: + """Release one removal lease while retaining evidence needed for a safe retry.""" + session.expire_all() + placement = await session.get(StoryArcPlacement, placement_id) + if placement is None or placement.operation_token != operation_token: + await session.rollback() + return + previous = dict(placement.last_result or {}) + failure = { + **( + {"target_fingerprint": dict(previous["target_fingerprint"])} + if isinstance(previous.get("target_fingerprint"), dict) + else {} + ), + "schema_version": 1, + "status": "failed", + "operation": "remove", + "error_code": error.code, + "error_category": _error_category(error), + "message": str(error), + } + result = await session.execute( + sa_update(StoryArcPlacement) + .where( + StoryArcPlacement.id == placement_id, + StoryArcPlacement.operation_token == operation_token, + ) + .values( + state=( + StoryArcPlacementState.DRIFTED + if _error_category(error) in {"safety", "collision", "ownership"} + else StoryArcPlacementState.FAILED + ), + last_result=failure, + operation_token=None, + ) + ) + if result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + return + await session.execute( + sa_update(IssueStoryArc) + .where(IssueStoryArc.id == placement.issue_story_arc_id) + .values(last_materialization_result=failure) + ) + await session.commit() + + +async def _delete_removed_placement_checkpoint( + session: AsyncSession, + *, + placement_id: int, + membership_id: int, + operation_token: str, + artifact_removed: bool, +) -> None: + """Atomically forget ownership only after the managed artifact is absent.""" + result = await session.execute( + delete(StoryArcPlacement).where( + StoryArcPlacement.id == placement_id, + StoryArcPlacement.operation_token == operation_token, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.MANAGED, + ) + ) + if result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_superseded", + "Story-arc placement removal lost its ownership lease", + category="conflict", + ) + await session.execute( + sa_update(IssueStoryArc) + .where(IssueStoryArc.id == membership_id) + .values( + sync_eligible=False, + last_materialization_result={ + "schema_version": 1, + "status": "placement_removed", + "placement_id": placement_id, + "artifact_removed": artifact_removed, + "canonical_preserved": True, + }, + ) + ) + await session.commit() + + +async def _persist_preflight_failure( + session: AsyncSession, + *, + membership_id: int, + placement_id: int | None, + observed_operation_token: str | None, + error: StoryArcPlacementError | StoryArcPlacementIntegrationError, +) -> None: + category = _error_category(error) + failure = { + "schema_version": 1, + "status": "failed", + "error_code": error.code, + "error_category": category, + "message": str(error), + } + if placement_id is not None: + session.expire_all() + placement = await session.get(StoryArcPlacement, placement_id) + if placement is not None: + previous = dict(placement.last_result or {}) + if isinstance(previous.get("target_fingerprint"), dict): + failure["target_fingerprint"] = dict(previous["target_fingerprint"]) + token_filter = ( + StoryArcPlacement.operation_token.is_(None) + if observed_operation_token is None + else StoryArcPlacement.operation_token == observed_operation_token + ) + result = await session.execute( + sa_update(StoryArcPlacement) + .where(StoryArcPlacement.id == placement_id, token_filter) + .values( + state=StoryArcPlacementState.DRIFTED, + last_result=failure, + operation_token=None, + ) + ) + if result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + return + await session.execute( + sa_update(IssueStoryArc) + .where(IssueStoryArc.id == membership_id) + .values(last_materialization_result=failure) + ) + await session.commit() + + +async def _persist_failure( + session: AsyncSession, + *, + membership_id: int, + placement_id: int, + operation_token: str, + error: StoryArcPlacementError | StoryArcPlacementIntegrationError, +) -> None: + session.expire_all() + placement = await session.get(StoryArcPlacement, placement_id) + if placement is None or placement.operation_token != operation_token: + await session.rollback() + return + category = _error_category(error) + previous = dict(placement.last_result or {}) + failure = { + **( + {"target_fingerprint": dict(previous["target_fingerprint"])} + if isinstance(previous.get("target_fingerprint"), dict) + else {} + ), + "schema_version": 1, + "status": "failed", + "error_code": error.code, + "error_category": category, + "message": str(error), + "operation_token": operation_token, + } + if ( + category == "collision" + and placement.ownership is StoryArcPlacementOwnership.MANAGED + and previous.get("status") == "prepared" + and not isinstance(previous.get("target_fingerprint"), dict) + and placement.creating_action_id is None + ): + result = await session.execute( + delete(StoryArcPlacement).where( + StoryArcPlacement.id == placement_id, + StoryArcPlacement.operation_token == operation_token, + ) + ) + else: + result = await session.execute( + sa_update(StoryArcPlacement) + .where( + StoryArcPlacement.id == placement_id, + StoryArcPlacement.operation_token == operation_token, + ) + .values( + state=( + StoryArcPlacementState.DRIFTED + if category == "collision" + else StoryArcPlacementState.FAILED + ), + last_result=failure, + operation_token=None, + ) + ) + if result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + return + await session.execute( + sa_update(IssueStoryArc) + .where(IssueStoryArc.id == membership_id) + .values(last_materialization_result=failure) + ) + await session.commit() + + +async def _persist_published_checkpoint( + session: AsyncSession, + *, + context: _PlacementContext, + policy: StoryArcPlacementPolicy, + prepared_placement_id: int, + operation_token: str, + result: StoryArcPlacementResult, +) -> None: + """Durably bridge filesystem publish and final user-visible reconciliation.""" + if result.target_path is None: + raise StoryArcPlacementIntegrationError( + "placement_target_missing", + "Placement publish returned no durable target evidence", + category="safety", + ) + checkpoint = { + "schema_version": 1, + "status": "published_pending_reconcile", + "outcome": result.state.value, + "operation_token": operation_token, + "target_fingerprint": dict(result.target_fingerprint), + } + update_result = await session.execute( + sa_update(StoryArcPlacement) + .where( + StoryArcPlacement.id == prepared_placement_id, + StoryArcPlacement.operation_token == operation_token, + ) + .values( + library_file_id=result.library_file_id, + library_root_id=policy.target_library_root_id, + placement_path=str(result.target_path), + mode=result.mode, + ownership=result.ownership, + symlink_style=result.symlink_style, + rendered_reading_order=result.rendered_reading_order, + policy_schema_version=STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + source_fingerprint=dict(result.source_fingerprint), + last_checked_at=datetime.now(UTC), + last_result=checkpoint, + ) + ) + if update_result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_superseded", + "Story-arc placement operation was superseded before checkpoint", + category="conflict", + ) + await session.execute( + sa_update(IssueStoryArc) + .where(IssueStoryArc.id == context.membership_id) + .values( + last_materialization_result={ + "schema_version": 1, + "status": "published_pending_reconcile", + "outcome": result.state.value, + "placement_id": prepared_placement_id, + } + ) + ) + await session.commit() + + +async def _reconcile_result( + session: AsyncSession, + *, + context: _PlacementContext, + policy: StoryArcPlacementPolicy, + prepared_placement_id: int, + operation_token: str, + result: StoryArcPlacementResult, +) -> StoryArcPlacementSyncResult: + target_path = result.target_path + if target_path is None: + raise StoryArcPlacementIntegrationError( + "placement_target_missing", + "Managed placement execution returned no destination", + category="safety", + ) + session.expire_all() + fence_code: str | None = None + current_arc = await session.get(StoryArc, context.story_arc_id) + if ( + current_arc is None + or current_arc.lifecycle is not StoryArcLifecycle.ACTIVE + or _policy_from_arc(current_arc) != policy + ): + fence_code = "placement_policy_changed" + else: + try: + current_context = await _load_one_context( + session, + context.story_arc_id, + context.membership_id, + ) + except StoryArcPlacementIntegrationError: + fence_code = "canonical_context_changed" + else: + if current_context != context: + fence_code = "canonical_context_changed" + if fence_code is None and not await _policy_root_is_available(session, policy): + fence_code = "target_library_root_unavailable" + + final_status = "complete" if fence_code is None else "drifted" + final_result = { + "schema_version": 1, + "status": final_status, + "outcome": result.state.value, + "operation_token": operation_token, + "target_fingerprint": dict(result.target_fingerprint), + } + if fence_code is not None: + final_result["error_code"] = fence_code + update_result = await session.execute( + sa_update(StoryArcPlacement) + .where( + StoryArcPlacement.id == prepared_placement_id, + StoryArcPlacement.operation_token == operation_token, + ) + .values( + library_file_id=result.library_file_id, + library_root_id=policy.target_library_root_id, + placement_path=str(target_path), + mode=result.mode, + ownership=result.ownership, + symlink_style=result.symlink_style, + rendered_reading_order=result.rendered_reading_order, + policy_schema_version=STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + source_fingerprint=dict(result.source_fingerprint), + state=( + StoryArcPlacementState.CURRENT + if fence_code is None + else StoryArcPlacementState.DRIFTED + ), + last_checked_at=datetime.now(UTC), + last_result=final_result, + operation_token=None, + ) + ) + if update_result.rowcount != 1: # type: ignore[attr-defined] + await session.rollback() + raise StoryArcPlacementIntegrationError( + "placement_operation_superseded", + "Story-arc placement operation was superseded before reconciliation", + category="conflict", + ) + await session.execute( + sa_update(IssueStoryArc) + .where(IssueStoryArc.id == context.membership_id) + .values( + sync_eligible=(policy.synchronize if fence_code is None else False), + last_materialization_result={ + "schema_version": 1, + "status": final_status, + "outcome": result.state.value, + "placement_id": prepared_placement_id, + **({"error_code": fence_code} if fence_code is not None else {}), + }, + ) + ) + await session.commit() + session.expire_all() + placement = await session.get(StoryArcPlacement, prepared_placement_id) + if placement is None: + raise StoryArcPlacementIntegrationError( + "prepared_placement_missing", + "Prepared story-arc placement disappeared after reconciliation", + category="conflict", + ) + if fence_code is not None: + raise StoryArcPlacementIntegrationError( + fence_code, + "Story-arc placement was published but its policy or canonical context changed", + category="conflict", + ) + return StoryArcPlacementSyncResult( + membership_id=context.membership_id, + outcome=result.state.value, + placement=_placement_view(placement), + ) + + +async def _require_placement( + session: AsyncSession, + story_arc_id: int, + placement_id: int, +) -> StoryArcPlacement: + placement = await session.scalar( + select(StoryArcPlacement) + .join(IssueStoryArc, StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id) + .where( + StoryArcPlacement.id == placement_id, + IssueStoryArc.story_arc_id == story_arc_id, + ) + ) + if placement is None: + raise _not_found("placement_not_found", "Story-arc placement was not found") + return placement + + +def _placement_view(row: StoryArcPlacement) -> StoryArcPlacementView: + last_result = dict(row.last_result or {}) + target_fingerprint = last_result.get("target_fingerprint") + return StoryArcPlacementView( + id=row.id, + issue_story_arc_id=row.issue_story_arc_id, + library_file_id=row.library_file_id, + library_root_id=row.library_root_id, + placement_path=row.placement_path, + mode=row.mode, + ownership=row.ownership, + symlink_style=row.symlink_style, + rendered_reading_order=row.rendered_reading_order, + policy_schema_version=row.policy_schema_version, + source_fingerprint=dict(row.source_fingerprint or {}), + target_fingerprint=( + dict(target_fingerprint) if isinstance(target_fingerprint, dict) else {} + ), + state=row.state, + last_result=last_result, + last_checked_at=row.last_checked_at, + ) + + +def _translate_filesystem_error(exc: StoryArcPlacementError) -> StoryArcPlacementIntegrationError: + return StoryArcPlacementIntegrationError( + exc.code, + str(exc), + category=_error_category(exc), + ) + + +def _error_category( + exc: StoryArcPlacementError | StoryArcPlacementIntegrationError, +) -> str: + if isinstance(exc, StoryArcPlacementIntegrationError): + return exc.category + if isinstance(exc, StoryArcPlacementCollisionError): + return "collision" + if isinstance(exc, StoryArcPlacementOwnershipError): + return "ownership" + if isinstance(exc, StoryArcPlacementCancellationError): + return "cancelled" + if isinstance(exc, StoryArcPlacementSafetyError): + return "safety" + return "operation" + + +def _bounded_page(limit: int, offset: int) -> tuple[int, int]: + if isinstance(limit, bool) or not 1 <= limit <= MAX_STORY_ARC_PLACEMENT_PAGE_SIZE: + raise StoryArcPlacementIntegrationError( + "invalid_page_limit", + f"Placement page limit must be from 1 to {MAX_STORY_ARC_PLACEMENT_PAGE_SIZE}", + ) + if isinstance(offset, bool) or offset < 0: + raise StoryArcPlacementIntegrationError( + "invalid_page_offset", + "Placement page offset must be non-negative", + ) + return limit, offset + + +def _not_found(code: str, message: str) -> StoryArcPlacementIntegrationError: + return StoryArcPlacementIntegrationError(code, message, category="not_found") diff --git a/src/pullbox/services/story_arc_placement_preview.py b/src/pullbox/services/story_arc_placement_preview.py new file mode 100644 index 00000000..09d650cc --- /dev/null +++ b/src/pullbox/services/story_arc_placement_preview.py @@ -0,0 +1,336 @@ +"""Read-only safety and collision preview for story-arc placements.""" + +from __future__ import annotations + +import enum +import os +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING + +from pullbox.core.story_arc_naming import ( + DEFAULT_STORY_ARC_FILE_TEMPLATE, + DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + StoryArcNamingValues, + StoryArcOriginalFilenameError, + render_story_arc_relative_path, +) +from pullbox.models.story_arc import StoryArcPlacementMode, StoryArcSymlinkStyle + +if TYPE_CHECKING: + from pathlib import Path + + +class StoryArcPlacementPreviewState(enum.StrEnum): + READY = "ready" + LOGICAL_ONLY = "logical_only" + ALREADY_REPRESENTED = "already_represented" + BLOCKED = "blocked" + + +class StoryArcCollisionKind(enum.StrEnum): + NONE = "none" + DIFFERENT_CONTENT = "different_content" + SAME_INODE_REFERENCED = "same_inode_referenced" + CASE_ONLY = "case_only" + PATH_ESCAPE = "path_escape" + CROSS_DEVICE = "cross_device" + ROOT_UNAVAILABLE = "root_unavailable" + SYMLINK_ROOT = "symlink_root" + SYMLINK_PARENT = "symlink_parent" + PARENT_NOT_DIRECTORY = "parent_not_directory" + DIRECTORY_SCAN_LIMIT = "directory_scan_limit" + COLLISION_SCAN_FAILED = "collision_scan_failed" + SOURCE_UNAVAILABLE = "source_unavailable" + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPreview: + """Truthful read-only result for one proposed placement.""" + + state: StoryArcPlacementPreviewState + mode: StoryArcPlacementMode + target_path: Path | None + collision: StoryArcCollisionKind = StoryArcCollisionKind.NONE + reason: str | None = None + required_bytes: int = 0 + proposed_ownership: str = "managed" + overwrite_allowed: bool = False + + +_MAX_CASE_SCAN_ENTRIES = 10_000 + + +class _CollisionScanError(RuntimeError): + def __init__(self, collision: StoryArcCollisionKind, reason: str) -> None: + self.collision = collision + super().__init__(reason) + + +def preview_story_arc_placement( + *, + canonical_path: Path | None, + destination_root: Path | None, + values: StoryArcNamingValues, + mode: StoryArcPlacementMode | str, + symlink_style: StoryArcSymlinkStyle | str | None = None, + folder_template: str | None = None, + file_template: str | None = None, +) -> StoryArcPlacementPreview: + """Plan one arc placement without creating, renaming, or deleting anything.""" + try: + effective_mode = StoryArcPlacementMode(mode) + except ValueError as exc: + msg = f"Unsupported story-arc placement mode: {mode}" + raise ValueError(msg) from exc + + effective_symlink_style: StoryArcSymlinkStyle | None = None + if symlink_style is not None: + try: + effective_symlink_style = StoryArcSymlinkStyle(symlink_style) + except ValueError as exc: + msg = f"Unsupported story-arc symlink style: {symlink_style}" + raise ValueError(msg) from exc + if effective_mode == StoryArcPlacementMode.SYMLINK and effective_symlink_style is None: + msg = "Story-arc symlink mode requires a symlink style" + raise ValueError(msg) + if effective_mode != StoryArcPlacementMode.SYMLINK and effective_symlink_style is not None: + msg = "Story-arc symlink style is only valid for symlink mode" + raise ValueError(msg) + + if effective_mode == StoryArcPlacementMode.REFERENCE_ONLY: + return StoryArcPlacementPreview( + state=StoryArcPlacementPreviewState.LOGICAL_ONLY, + mode=effective_mode, + target_path=None, + proposed_ownership="referenced", + ) + + if canonical_path is None or not canonical_path.exists() or not canonical_path.is_file(): + return _blocked( + effective_mode, + None, + StoryArcCollisionKind.SOURCE_UNAVAILABLE, + "Canonical issue file is unavailable", + ) + if canonical_path.is_symlink(): + return _blocked( + effective_mode, + None, + StoryArcCollisionKind.SOURCE_UNAVAILABLE, + "Canonical issue file is a symbolic link and requires explicit review", + ) + if destination_root is None or not destination_root.is_absolute(): + return _blocked( + effective_mode, + None, + StoryArcCollisionKind.ROOT_UNAVAILABLE, + "Story-arc destination root is unavailable", + ) + if destination_root.is_symlink(): + return _blocked( + effective_mode, + None, + StoryArcCollisionKind.SYMLINK_ROOT, + "Story-arc destination root cannot be a symbolic link", + ) + if not destination_root.exists() or not destination_root.is_dir(): + return _blocked( + effective_mode, + None, + StoryArcCollisionKind.ROOT_UNAVAILABLE, + "Story-arc destination root is unavailable", + ) + if not os.access(destination_root, os.R_OK | os.W_OK | os.X_OK): + return _blocked( + effective_mode, + None, + StoryArcCollisionKind.ROOT_UNAVAILABLE, + "Story-arc destination root is not readable and writable", + ) + + try: + relative_path = render_story_arc_relative_path( + replace(values, original_filename=canonical_path.name), + folder_template=( + DEFAULT_STORY_ARC_FOLDER_TEMPLATE if folder_template is None else folder_template + ), + file_template=( + DEFAULT_STORY_ARC_FILE_TEMPLATE if file_template is None else file_template + ), + ) + except StoryArcOriginalFilenameError as exc: + return _blocked(effective_mode, None, StoryArcCollisionKind.SOURCE_UNAVAILABLE, str(exc)) + target_path = destination_root / relative_path + resolved_root = destination_root.resolve(strict=True) + parent_collision = _existing_parent_collision( + destination_root, + target_path.parent, + resolved_root, + ) + if parent_collision is not None: + collision, reason = parent_collision + return _blocked( + effective_mode, + target_path, + collision, + reason, + ) + # Containment applies to the destination entry and its parents. Do not + # follow an existing final symlink here: symlink mode intentionally points + # at the canonical issue, which commonly lives outside the arc root. + resolved_target = target_path.parent.resolve(strict=False) / target_path.name + if not resolved_target.is_relative_to(resolved_root): + return _blocked( + effective_mode, + target_path, + StoryArcCollisionKind.PATH_ESCAPE, + "Rendered story-arc path resolves outside the selected root", + ) + + try: + case_collision = _find_case_only_collision(target_path) + except _CollisionScanError as exc: + return _blocked( + effective_mode, + target_path, + exc.collision, + str(exc), + ) + if case_collision is not None: + return _blocked( + effective_mode, + target_path, + StoryArcCollisionKind.CASE_ONLY, + "A case-only destination collision already exists", + ) + + if target_path.exists() or target_path.is_symlink(): + try: + same_inode = target_path.exists() and target_path.samefile(canonical_path) + except OSError: + same_inode = False + if same_inode: + return StoryArcPlacementPreview( + state=StoryArcPlacementPreviewState.ALREADY_REPRESENTED, + mode=effective_mode, + target_path=target_path, + collision=StoryArcCollisionKind.SAME_INODE_REFERENCED, + reason="An existing user artifact already references the canonical file", + proposed_ownership="referenced", + ) + return _blocked( + effective_mode, + target_path, + StoryArcCollisionKind.DIFFERENT_CONTENT, + "A different artifact already exists at the rendered destination", + ) + + source_stat = canonical_path.stat() + if effective_mode == StoryArcPlacementMode.HARDLINK: + destination_device = _nearest_existing_ancestor(target_path.parent).stat().st_dev + if source_stat.st_dev != destination_device: + return _blocked( + effective_mode, + target_path, + StoryArcCollisionKind.CROSS_DEVICE, + "Hardlink source and destination are on different filesystems", + ) + + return StoryArcPlacementPreview( + state=StoryArcPlacementPreviewState.READY, + mode=effective_mode, + target_path=target_path, + required_bytes=(source_stat.st_size if effective_mode == StoryArcPlacementMode.COPY else 0), + ) + + +def _existing_parent_collision( + root: Path, + parent: Path, + resolved_root: Path, +) -> tuple[StoryArcCollisionKind, str] | None: + """Mirror execution's fail-closed rule for every existing parent.""" + current = root + for part in parent.relative_to(root).parts: + current = current / part + try: + current.lstat() + except FileNotFoundError: + return None + except OSError: + return ( + StoryArcCollisionKind.COLLISION_SCAN_FAILED, + "Story-arc destination parent could not be inspected safely", + ) + if current.is_symlink(): + try: + resolved = current.resolve(strict=True) + except OSError: + return ( + StoryArcCollisionKind.SYMLINK_PARENT, + "Story-arc destination has an unsafe symbolic-link parent", + ) + collision = ( + StoryArcCollisionKind.PATH_ESCAPE + if not resolved.is_relative_to(resolved_root) + else StoryArcCollisionKind.SYMLINK_PARENT + ) + return ( + collision, + "Story-arc destination has a symbolic-link parent", + ) + if not current.is_dir(): + return ( + StoryArcCollisionKind.PARENT_NOT_DIRECTORY, + "Story-arc destination parent is not a directory", + ) + return None + + +def _find_case_only_collision(target_path: Path) -> Path | None: + parent = target_path.parent + if not parent.exists() or not parent.is_dir(): + return None + target_key = target_path.name.casefold() + try: + for index, child in enumerate(parent.iterdir(), start=1): + if index > _MAX_CASE_SCAN_ENTRIES: + raise _CollisionScanError( + StoryArcCollisionKind.DIRECTORY_SCAN_LIMIT, + "Story-arc collision preview exceeded its bounded directory scan limit", + ) + if child.name != target_path.name and child.name.casefold() == target_key: + return child + except _CollisionScanError: + raise + except OSError as exc: + raise _CollisionScanError( + StoryArcCollisionKind.COLLISION_SCAN_FAILED, + "Story-arc destination collision preview failed", + ) from exc + return None + + +def _nearest_existing_ancestor(path: Path) -> Path: + current = path + while not current.exists(): + parent = current.parent + if parent == current: + return current + current = parent + return current + + +def _blocked( + mode: StoryArcPlacementMode, + target_path: Path | None, + collision: StoryArcCollisionKind, + reason: str, +) -> StoryArcPlacementPreview: + return StoryArcPlacementPreview( + state=StoryArcPlacementPreviewState.BLOCKED, + mode=mode, + target_path=target_path, + collision=collision, + reason=reason, + ) diff --git a/src/pullbox/services/story_arc_placement_service.py b/src/pullbox/services/story_arc_placement_service.py new file mode 100644 index 00000000..d15a8eaa --- /dev/null +++ b/src/pullbox/services/story_arc_placement_service.py @@ -0,0 +1,2569 @@ +"""Bounded filesystem execution for optional story-arc placements. + +This module deliberately owns no database transaction. Callers pass an +immutable plan, persist the returned fingerprint evidence, and may adapt the +small journal callback to the import action journal. Canonical files are read +only; story-arc materialization never has a move operation. +""" + +from __future__ import annotations + +import enum +import errno +import hashlib +import os +import secrets +import stat +from contextlib import contextmanager, suppress +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Literal, Protocol + +from pullbox.models.story_arc import ( + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcSymlinkStyle, +) +from pullbox.services.story_arc_placement_preview import ( + StoryArcCollisionKind, + StoryArcPlacementPreview, + StoryArcPlacementPreviewState, + preview_story_arc_placement, +) + +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + + from pullbox.core.story_arc_naming import StoryArcNamingValues + +MAX_STORY_ARC_PLACEMENTS_PER_BATCH = 250 +_COPY_CHUNK_BYTES = 1024 * 1024 +_MAX_CASE_SCAN_ENTRIES = 10_000 +_TEMP_PREFIX = ".pullbox-story-arc-" +_TEMP_SUFFIX = ".tmp" +_SECURE_DIR_FD_SUPPORTED = ( + os.name == "posix" + and hasattr(os, "O_DIRECTORY") + and hasattr(os, "O_NOFOLLOW") + and hasattr(os, "fchmod") + and os.scandir in os.supports_fd + and os.utime in os.supports_fd + and all( + operation in os.supports_dir_fd + for operation in ( + os.open, + os.mkdir, + os.stat, + os.unlink, + os.link, + os.symlink, + os.readlink, + ) + ) +) + +Fingerprint = dict[str, object] + + +@dataclass(frozen=True, slots=True) +class _PublishedTarget: + """Ephemeral ownership evidence captured from the artifact being published.""" + + stat: os.stat_result + sha256: str | None = None + link_target: str | None = None + + +class StoryArcPlacementError(RuntimeError): + """Base class for a categorized placement failure.""" + + def __init__(self, code: str, message: str) -> None: + self.code = code + super().__init__(message) + + +class StoryArcPlacementSafetyError(StoryArcPlacementError): + """A root, path, source, or mode precondition failed closed.""" + + +class StoryArcPlacementCollisionError(StoryArcPlacementError): + """A destination exists that Pullbox may not overwrite.""" + + +class StoryArcPlacementCancellationError(StoryArcPlacementError): + """Execution observed cancellation before publishing an artifact.""" + + def __init__(self) -> None: + super().__init__("cancelled", "Story-arc placement was cancelled") + + +class StoryArcPlacementOwnershipError(StoryArcPlacementError): + """A destructive operation lacks managed ownership evidence.""" + + +class StoryArcPlacementResultState(enum.StrEnum): + """Stable result values without coupling callers to an ORM state enum.""" + + CREATED = "created" + IDEMPOTENT = "idempotent" + REFERENCED_EXISTING = "referenced_existing" + REFERENCE_ONLY = "reference_only" + + +class StoryArcPlacementInspectionState(enum.StrEnum): + """Stable read-only states for placement lifecycle presentation.""" + + FREE = "free" + MANAGED_CURRENT = "managed_current" + REFERENCED_CURRENT = "referenced_current" + MANAGED_MISSING = "managed_missing" + REFERENCED_MISSING = "referenced_missing" + MANAGED_DRIFTED = "managed_drifted" + REFERENCED_DRIFTED = "referenced_drifted" + UNTRACKED_IDENTICAL = "untracked_identical" + DIFFERENT_CONTENT = "different_content" + BLOCKED = "blocked" + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPlan: + """Complete data-only plan for one resolved story-arc membership.""" + + issue_story_arc_id: int + library_file_id: int | None + canonical_path: Path | None + destination_root: Path | None + values: StoryArcNamingValues + mode: StoryArcPlacementMode | str + symlink_style: StoryArcSymlinkStyle | str | None = None + folder_template: str | None = None + file_template: str | None = None + adopt_identical_existing: bool = False + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPreparation: + """Pre-publish evidence that an async caller can commit durably. + + The target fingerprint cannot exist until after publication. Persisting + the immutable source, destination-root identity, target, mode, and + operation token lets a restart distinguish a prepared managed operation + from an untracked user artifact and validate the result before adopting it. + """ + + issue_story_arc_id: int + target_path: Path + mode: StoryArcPlacementMode + symlink_style: StoryArcSymlinkStyle | None + rendered_reading_order: int + source_fingerprint: Fingerprint + destination_root_fingerprint: Fingerprint + + +@dataclass(frozen=True, slots=True) +class PreparedManagedStoryArcPlacementEvidence: + """Durably committed ownership intent for publish-window recovery.""" + + issue_story_arc_id: int + placement_path: Path + mode: StoryArcPlacementMode | str + symlink_style: StoryArcSymlinkStyle | str | None + source_fingerprint: Fingerprint + destination_root_fingerprint: Fingerprint + operation_token: str + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementResult: + """Persistence-ready evidence for one completed or no-op placement.""" + + issue_story_arc_id: int + library_file_id: int | None + state: StoryArcPlacementResultState + target_path: Path | None + mode: StoryArcPlacementMode + ownership: StoryArcPlacementOwnership + symlink_style: StoryArcSymlinkStyle | None + rendered_reading_order: int + source_fingerprint: Fingerprint + target_fingerprint: Fingerprint + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementInspectionEvidence: + """Persisted target expectation used only for read-only inspection.""" + + placement_path: Path + mode: StoryArcPlacementMode | str + ownership: StoryArcPlacementOwnership | str + symlink_style: StoryArcSymlinkStyle | str | None + source_fingerprint: Fingerprint + target_fingerprint: Fingerprint + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementInspection: + """Pure filesystem observation with a stable state and reason code.""" + + state: StoryArcPlacementInspectionState + mode: StoryArcPlacementMode + target_path: Path | None + collision: StoryArcCollisionKind = StoryArcCollisionKind.NONE + code: str | None = None + reason: str | None = None + required_bytes: int = 0 + proposed_ownership: str = "managed" + source_fingerprint: Fingerprint | None = None + target_fingerprint: Fingerprint | None = None + + +@dataclass(frozen=True, slots=True) +class ManagedStoryArcPlacementEvidence: + """Previously persisted evidence required for retry, repair, or removal.""" + + issue_story_arc_id: int + placement_path: Path + mode: StoryArcPlacementMode | str + ownership: StoryArcPlacementOwnership | str + symlink_style: StoryArcSymlinkStyle | str | None + source_fingerprint: Fingerprint + target_fingerprint: Fingerprint + creating_action_id: int | None + + @classmethod + def from_result( + cls, + result: StoryArcPlacementResult, + *, + creating_action_id: int, + ) -> ManagedStoryArcPlacementEvidence: + """Create durable evidence after the caller records a publish action.""" + if result.ownership is not StoryArcPlacementOwnership.MANAGED or result.target_path is None: + raise StoryArcPlacementOwnershipError( + "not_managed", + "Only a managed placement result can produce managed evidence", + ) + return cls( + issue_story_arc_id=result.issue_story_arc_id, + placement_path=result.target_path, + mode=result.mode, + ownership=result.ownership, + symlink_style=result.symlink_style, + source_fingerprint=dict(result.source_fingerprint), + target_fingerprint=dict(result.target_fingerprint), + creating_action_id=creating_action_id, + ) + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementJournalEvent: + """Narrow event that an import/action-journal adapter can persist.""" + + stage: Literal["prepared", "published", "failed", "remove_prepared", "removed"] + operation: Literal["publish", "remove"] + issue_story_arc_id: int + mode: StoryArcPlacementMode + target_path: Path + source_fingerprint: Fingerprint + target_fingerprint: Fingerprint + failure_code: str | None = None + + +class StoryArcPlacementJournal(Protocol): + """Synchronous observation hook called around filesystem actions. + + This callback is not an async database durability boundary. An async + integration must persist and commit its prepared action before dispatching + this service to a worker thread; these events then provide reconciliation + evidence around the actual publish/remove call. + """ + + def __call__(self, event: StoryArcPlacementJournalEvent) -> None: ... + + +class CancellationRequested(Protocol): + """Cheap bounded cancellation predicate used between copy chunks.""" + + def __call__(self) -> bool: ... + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementRemovalResult: + """Truthful result for an ownership-validated removal.""" + + placement_path: Path + removed: bool + + +@dataclass(frozen=True, slots=True) +class _RootGuard: + """Pinned destination-root identity used across the publish boundary.""" + + path: Path + resolved_path: Path + device: int + inode: int + + +@dataclass(slots=True) +class _SecureParentDirectory: + """Pinned no-follow directory handles for one target parent.""" + + root_guard: _RootGuard + path: Path + root_fd: int + parent_fd: int + device: int + inode: int + + +def prepare_story_arc_placement( + plan: StoryArcPlacementPlan, + *, + existing_managed: ManagedStoryArcPlacementEvidence | None = None, +) -> StoryArcPlacementPreparation: + """Capture immutable source/root/path evidence before a managed publish. + + This performs read-only filesystem validation. Database adapters should + call it only after closing their read transaction, then commit the returned + evidence before dispatching :func:`execute_story_arc_placement`. + """ + preparation, preview, source_path = _inspect_story_arc_placement(plan) + if existing_managed is not None: + existing_result = _resolve_existing_target( + plan=plan, + mode=preparation.mode, + symlink_style=preparation.symlink_style, + source_path=source_path, + source_fingerprint=preparation.source_fingerprint, + target_path=preparation.target_path, + existing_managed=existing_managed, + ) + if existing_result is not None: + return preparation + elif preview.state is StoryArcPlacementPreviewState.BLOCKED: + # Preview is intentionally cheap and does not hash arbitrary existing + # files. Preparation performs the bounded comparison so callers get + # the truthful identical-user versus different-content classification + # before any ownership row is reserved. + _resolve_existing_target( + plan=plan, + mode=preparation.mode, + symlink_style=preparation.symlink_style, + source_path=source_path, + source_fingerprint=preparation.source_fingerprint, + target_path=preparation.target_path, + existing_managed=None, + ) + _raise_from_preview(preview) + if preview.state is not StoryArcPlacementPreviewState.READY: + raise StoryArcPlacementSafetyError( + "preview_not_ready", + preview.reason or "Story-arc placement is not ready for preparation", + ) + return preparation + + +def recover_prepared_story_arc_placement( + plan: StoryArcPlacementPlan, + evidence: PreparedManagedStoryArcPlacementEvidence, +) -> StoryArcPlacementResult | None: + """Recover a target published after prepare but before DB checkpoint. + + No artifact is created or changed here. The target is accepted only when + every precommitted ownership field still matches and its representation and + content exactly match the canonical source. A missing target means the + prepared operation may be retried normally. + """ + preparation, _preview, source_path = _inspect_story_arc_placement(plan) + mode = _coerce_mode(evidence.mode) + symlink_style = _coerce_symlink_style(evidence.symlink_style) + if not evidence.operation_token: + raise StoryArcPlacementOwnershipError( + "operation_token_missing", + "Prepared story-arc placement has no ownership operation token", + ) + if ( + evidence.issue_story_arc_id != preparation.issue_story_arc_id + or Path(evidence.placement_path) != preparation.target_path + or mode is not preparation.mode + or symlink_style is not preparation.symlink_style + or evidence.source_fingerprint != preparation.source_fingerprint + or evidence.destination_root_fingerprint != preparation.destination_root_fingerprint + ): + raise StoryArcPlacementOwnershipError( + "prepared_evidence_mismatch", + "Prepared story-arc ownership evidence no longer matches the requested placement", + ) + target_path = preparation.target_path + if not _path_exists(target_path): + return None + _validate_existing_representation(mode, symlink_style, source_path, target_path) + if not _target_content_matches_source( + target_path, + source_path, + preparation.source_fingerprint, + ): + raise StoryArcPlacementSafetyError( + "prepared_target_mismatch", + "Prepared story-arc target does not match the canonical source", + ) + # The database token is not physically bound to the artifact. A restart + # therefore cannot prove whether Pullbox or another process created an + # identical target after preparation. Preserve the file and track it as + # referenced; only an uninterrupted publish may establish managed ownership. + return StoryArcPlacementResult( + issue_story_arc_id=preparation.issue_story_arc_id, + library_file_id=plan.library_file_id, + state=StoryArcPlacementResultState.REFERENCED_EXISTING, + target_path=target_path, + mode=StoryArcPlacementMode.REFERENCE_ONLY, + ownership=StoryArcPlacementOwnership.REFERENCED, + symlink_style=None, + rendered_reading_order=preparation.rendered_reading_order, + source_fingerprint=dict(preparation.source_fingerprint), + target_fingerprint=_fingerprint_target(target_path), + ) + + +def inspect_story_arc_placement( + plan: StoryArcPlacementPlan, + *, + existing: StoryArcPlacementInspectionEvidence | None = None, +) -> StoryArcPlacementInspection: + """Inspect one target without creating, replacing, repairing, or deleting it. + + Traversal and hashing use the same no-follow, root-anchored boundary as + execution. Expected drift is represented as data so API/UI callers do not + need to duplicate filesystem logic or turn ordinary lifecycle states into + exceptions. + """ + mode = _coerce_mode(plan.mode) + symlink_style = _coerce_symlink_style(plan.symlink_style) + _validate_mode_and_style(mode, symlink_style) + preview = preview_story_arc_placement( + canonical_path=plan.canonical_path, + destination_root=plan.destination_root, + values=plan.values, + mode=mode, + symlink_style=symlink_style, + folder_template=plan.folder_template, + file_template=plan.file_template, + ) + ownership = _inspection_ownership(existing) + if existing is not None and ownership is None: + return _inspection_result( + preview, + state=StoryArcPlacementInspectionState.BLOCKED, + target_path=Path(existing.placement_path), + code="ownership_invalid", + reason="Placement ownership evidence is invalid", + ) + if mode is StoryArcPlacementMode.REFERENCE_ONLY and existing is None: + return _inspection_result( + preview, + state=StoryArcPlacementInspectionState.FREE, + target_path=None, + proposed_ownership="referenced", + ) + + rendered_target = preview.target_path + if mode is not StoryArcPlacementMode.REFERENCE_ONLY and rendered_target is None: + return _inspection_result( + preview, + state=StoryArcPlacementInspectionState.BLOCKED, + target_path=None, + code=preview.collision.value, + reason=preview.reason, + ) + target_path = ( + Path(existing.placement_path) + if mode is StoryArcPlacementMode.REFERENCE_ONLY and existing is not None + else rendered_target + ) + if target_path is None: # pragma: no cover - exhaustiveness above + return _inspection_result( + preview, + state=StoryArcPlacementInspectionState.BLOCKED, + target_path=None, + code="target_unavailable", + reason="Story-arc placement has no inspectable target", + ) + if existing is not None and Path(existing.placement_path) != target_path: + return _tracked_inspection_result( + preview, + ownership=ownership, + target_path=target_path, + code="destination_mismatch", + reason="Recorded placement belongs to another destination", + ) + if preview.state is StoryArcPlacementPreviewState.BLOCKED and preview.collision not in { + StoryArcCollisionKind.DIFFERENT_CONTENT, + StoryArcCollisionKind.CASE_ONLY, + }: + return _inspection_result( + preview, + state=StoryArcPlacementInspectionState.BLOCKED, + target_path=target_path, + code=preview.collision.value, + reason=preview.reason, + ) + if preview.collision is StoryArcCollisionKind.CASE_ONLY: + if existing is not None: + return _tracked_inspection_result( + preview, + ownership=ownership, + target_path=target_path, + code="case_only", + reason=preview.reason, + collision=StoryArcCollisionKind.CASE_ONLY, + ) + return _inspection_result( + preview, + state=StoryArcPlacementInspectionState.BLOCKED, + target_path=target_path, + code="case_only", + reason=preview.reason, + collision=StoryArcCollisionKind.CASE_ONLY, + ) + + try: + source_path = _validated_regular_source(plan.canonical_path) + source_fingerprint = _fingerprint_regular_nofollow(source_path) + root_guard = _validated_root(plan.destination_root) + _validate_target_lexically(root_guard.path, target_path) + _validate_path_limits(target_path) + _reject_canonical_destination(source_path, target_path) + try: + parent_context = _open_secure_parent_directory( + root_guard, + target_path.parent, + create=False, + ) + with parent_context as secure_parent: + _assert_parent_path_stable(secure_parent) + if not _entry_exists_at(secure_parent.parent_fd, target_path.name): + return _missing_inspection_result( + preview, + ownership=ownership, + target_path=target_path, + source_fingerprint=source_fingerprint, + ) + target_fingerprint = _fingerprint_target_at( + secure_parent, + target_path.name, + canonical_path=source_path, + ) + _assert_parent_path_stable(secure_parent) + if existing is None: + state = ( + StoryArcPlacementInspectionState.UNTRACKED_IDENTICAL + if _fingerprint_content_matches( + target_fingerprint, + source_fingerprint, + ) + else StoryArcPlacementInspectionState.DIFFERENT_CONTENT + ) + return _inspection_result( + preview, + state=state, + target_path=target_path, + code=( + "untracked_identical" + if state is StoryArcPlacementInspectionState.UNTRACKED_IDENTICAL + else "different_content" + ), + reason=( + "An identical untracked artifact already exists" + if state is StoryArcPlacementInspectionState.UNTRACKED_IDENTICAL + else "A different artifact exists at the placement destination" + ), + source_fingerprint=source_fingerprint, + target_fingerprint=target_fingerprint, + ) + drift_code = _tracked_drift_code( + plan=plan, + evidence=existing, + ownership=ownership, + source_path=source_path, + source_fingerprint=source_fingerprint, + target_fingerprint=target_fingerprint, + target_name=target_path.name, + parent=secure_parent, + ) + if drift_code is not None: + return _tracked_inspection_result( + preview, + ownership=ownership, + target_path=target_path, + code=drift_code, + reason=_inspection_drift_reason(drift_code), + source_fingerprint=source_fingerprint, + target_fingerprint=target_fingerprint, + ) + return _inspection_result( + preview, + state=( + StoryArcPlacementInspectionState.MANAGED_CURRENT + if ownership is StoryArcPlacementOwnership.MANAGED + else StoryArcPlacementInspectionState.REFERENCED_CURRENT + ), + target_path=target_path, + source_fingerprint=source_fingerprint, + target_fingerprint=target_fingerprint, + proposed_ownership=("managed" if ownership is None else ownership.value), + ) + except FileNotFoundError: + return _missing_inspection_result( + preview, + ownership=ownership, + target_path=target_path, + source_fingerprint=source_fingerprint, + ) + except StoryArcPlacementError as exc: + if existing is not None and ownership is not None: + return _tracked_inspection_result( + preview, + ownership=ownership, + target_path=target_path, + code=exc.code, + reason=str(exc), + ) + if exc.code in { + "dangling_symlink", + "fingerprint_mismatch", + "not_regular_file", + }: + return _inspection_result( + preview, + state=StoryArcPlacementInspectionState.DIFFERENT_CONTENT, + target_path=target_path, + code="different_content", + reason="A different artifact exists at the placement destination", + collision=StoryArcCollisionKind.DIFFERENT_CONTENT, + ) + return _inspection_result( + preview, + state=StoryArcPlacementInspectionState.BLOCKED, + target_path=target_path, + code=exc.code, + reason=str(exc), + collision=_collision_from_code(exc.code), + ) + except OSError: + if existing is not None and ownership is not None: + return _tracked_inspection_result( + preview, + ownership=ownership, + target_path=target_path, + code="inspection_failed", + reason="Placement could not be inspected safely", + ) + return _inspection_result( + preview, + state=StoryArcPlacementInspectionState.BLOCKED, + target_path=target_path, + code="inspection_failed", + reason="Placement could not be inspected safely", + ) + + +def _inspection_result( + preview: StoryArcPlacementPreview, + *, + state: StoryArcPlacementInspectionState, + target_path: Path | None, + code: str | None = None, + reason: str | None = None, + collision: StoryArcCollisionKind | None = None, + proposed_ownership: str | None = None, + source_fingerprint: Fingerprint | None = None, + target_fingerprint: Fingerprint | None = None, +) -> StoryArcPlacementInspection: + effective_collision = ( + collision + if collision is not None + else ( + preview.collision + if state + in { + StoryArcPlacementInspectionState.BLOCKED, + StoryArcPlacementInspectionState.DIFFERENT_CONTENT, + } + else StoryArcCollisionKind.NONE + ) + ) + source_size = source_fingerprint.get("size") if source_fingerprint is not None else None + required_bytes = preview.required_bytes + if ( + required_bytes == 0 + and preview.mode is StoryArcPlacementMode.COPY + and isinstance(source_size, int) + ): + required_bytes = source_size + return StoryArcPlacementInspection( + state=state, + mode=preview.mode, + target_path=target_path, + collision=effective_collision, + code=code, + reason=reason, + required_bytes=required_bytes, + proposed_ownership=( + preview.proposed_ownership if proposed_ownership is None else proposed_ownership + ), + source_fingerprint=source_fingerprint, + target_fingerprint=target_fingerprint, + ) + + +def _inspection_ownership( + evidence: StoryArcPlacementInspectionEvidence | None, +) -> StoryArcPlacementOwnership | None: + if evidence is None: + return None + try: + return StoryArcPlacementOwnership(evidence.ownership) + except ValueError: + return None + + +def _tracked_inspection_result( + preview: StoryArcPlacementPreview, + *, + ownership: StoryArcPlacementOwnership | None, + target_path: Path, + code: str, + reason: str | None, + collision: StoryArcCollisionKind | None = None, + source_fingerprint: Fingerprint | None = None, + target_fingerprint: Fingerprint | None = None, +) -> StoryArcPlacementInspection: + managed = ownership is StoryArcPlacementOwnership.MANAGED + return _inspection_result( + preview, + state=( + StoryArcPlacementInspectionState.MANAGED_DRIFTED + if managed + else StoryArcPlacementInspectionState.REFERENCED_DRIFTED + ), + target_path=target_path, + code=code, + reason=reason, + collision=collision, + proposed_ownership="managed" if managed else "referenced", + source_fingerprint=source_fingerprint, + target_fingerprint=target_fingerprint, + ) + + +def _missing_inspection_result( + preview: StoryArcPlacementPreview, + *, + ownership: StoryArcPlacementOwnership | None, + target_path: Path, + source_fingerprint: Fingerprint, +) -> StoryArcPlacementInspection: + if ownership is StoryArcPlacementOwnership.MANAGED: + state = StoryArcPlacementInspectionState.MANAGED_MISSING + elif ownership is StoryArcPlacementOwnership.REFERENCED: + state = StoryArcPlacementInspectionState.REFERENCED_MISSING + else: + state = StoryArcPlacementInspectionState.FREE + return _inspection_result( + preview, + state=state, + target_path=target_path, + code=None if state is StoryArcPlacementInspectionState.FREE else "target_missing", + reason=None if state is StoryArcPlacementInspectionState.FREE else "Placement is missing", + source_fingerprint=source_fingerprint, + proposed_ownership=( + "managed" + if ownership is None or ownership is StoryArcPlacementOwnership.MANAGED + else "referenced" + ), + ) + + +def _tracked_drift_code( + *, + plan: StoryArcPlacementPlan, + evidence: StoryArcPlacementInspectionEvidence, + ownership: StoryArcPlacementOwnership | None, + source_path: Path, + source_fingerprint: Fingerprint, + target_fingerprint: Fingerprint, + target_name: str, + parent: _SecureParentDirectory, +) -> str | None: + if evidence.source_fingerprint and evidence.source_fingerprint != source_fingerprint: + return "source_fingerprint_mismatch" + if evidence.target_fingerprint and evidence.target_fingerprint != target_fingerprint: + if ownership is StoryArcPlacementOwnership.MANAGED: + try: + expected_mode = _coerce_mode(evidence.mode) + expected_style = _coerce_symlink_style(evidence.symlink_style) + _validate_mode_and_style(expected_mode, expected_style) + _validate_existing_representation_at( + expected_mode, + expected_style, + source_path, + source_fingerprint, + target_name, + parent, + ) + except (ValueError, StoryArcPlacementError): + return "representation_changed" + return "target_fingerprint_mismatch" + if not _fingerprint_content_matches(target_fingerprint, source_fingerprint): + return "content_changed" + if ownership is StoryArcPlacementOwnership.MANAGED: + try: + expected_mode = _coerce_mode(evidence.mode) + expected_style = _coerce_symlink_style(evidence.symlink_style) + requested_style = _coerce_symlink_style(plan.symlink_style) + _validate_mode_and_style(expected_mode, expected_style) + if ( + expected_mode is not _coerce_mode(plan.mode) + or expected_style is not requested_style + ): + return "representation_changed" + _validate_existing_representation_at( + expected_mode, + expected_style, + source_path, + source_fingerprint, + target_name, + parent, + ) + except (ValueError, StoryArcPlacementError): + return "representation_changed" + return None + + +def _fingerprint_content_matches( + target_fingerprint: Fingerprint, + source_fingerprint: Fingerprint, +) -> bool: + content = ( + target_fingerprint.get("content") + if target_fingerprint.get("kind") == "symlink" + else target_fingerprint + ) + return isinstance(content, dict) and content.get("sha256") == source_fingerprint.get("sha256") + + +def _inspection_drift_reason(code: str) -> str: + return { + "source_fingerprint_mismatch": "Canonical source changed after placement was recorded", + "target_fingerprint_mismatch": "Placement changed after it was recorded", + "representation_changed": "Placement representation changed after it was recorded", + "content_changed": "Placement content no longer matches its canonical source", + }.get(code, "Placement no longer matches its recorded evidence") + + +def _collision_from_code(code: str) -> StoryArcCollisionKind: + try: + return StoryArcCollisionKind(code) + except ValueError: + return StoryArcCollisionKind.NONE + + +def _inspect_story_arc_placement( + plan: StoryArcPlacementPlan, +) -> tuple[StoryArcPlacementPreparation, StoryArcPlacementPreview, Path]: + mode = _coerce_mode(plan.mode) + symlink_style = _coerce_symlink_style(plan.symlink_style) + _validate_mode_and_style(mode, symlink_style) + if mode is StoryArcPlacementMode.REFERENCE_ONLY: + raise StoryArcPlacementSafetyError( + "managed_preparation_required", + "Reference-only placement does not use managed publish preparation", + ) + if plan.issue_story_arc_id <= 0: + raise ValueError("Story-arc membership id must be positive") + preview = preview_story_arc_placement( + canonical_path=plan.canonical_path, + destination_root=plan.destination_root, + values=plan.values, + mode=mode, + symlink_style=symlink_style, + folder_template=plan.folder_template, + file_template=plan.file_template, + ) + target_path = preview.target_path + if target_path is None: + _raise_from_preview(preview) + raise StoryArcPlacementSafetyError( + "target_unavailable", + "Story-arc placement preview did not produce a destination", + ) + source_path = _validated_regular_source(plan.canonical_path) + root_guard = _validated_root(plan.destination_root) + _validate_target_lexically(root_guard.path, target_path) + _validate_safe_existing_parents(root_guard, target_path.parent) + _validate_path_limits(target_path) + _reject_canonical_destination(source_path, target_path) + return ( + StoryArcPlacementPreparation( + issue_story_arc_id=plan.issue_story_arc_id, + target_path=target_path, + mode=mode, + symlink_style=symlink_style, + rendered_reading_order=plan.values.reading_order, + source_fingerprint=_fingerprint_regular_nofollow(source_path), + destination_root_fingerprint=_root_fingerprint(root_guard), + ), + preview, + source_path, + ) + + +def execute_story_arc_placement( + plan: StoryArcPlacementPlan, + *, + existing_managed: ManagedStoryArcPlacementEvidence | None = None, + preparation: StoryArcPlacementPreparation | None = None, + cancellation_requested: CancellationRequested | None = None, + journal: StoryArcPlacementJournal | None = None, +) -> StoryArcPlacementResult: + """Execute one resolved membership placement without moving its source.""" + mode = _coerce_mode(plan.mode) + symlink_style = _coerce_symlink_style(plan.symlink_style) + _validate_mode_and_style(mode, symlink_style) + if plan.issue_story_arc_id <= 0: + raise ValueError("Story-arc membership id must be positive") + + if mode is StoryArcPlacementMode.REFERENCE_ONLY: + if existing_managed is not None: + raise StoryArcPlacementOwnershipError( + "mode_changed", + "Managed placement evidence cannot be applied to reference-only mode", + ) + return StoryArcPlacementResult( + issue_story_arc_id=plan.issue_story_arc_id, + library_file_id=plan.library_file_id, + state=StoryArcPlacementResultState.REFERENCE_ONLY, + target_path=None, + mode=mode, + ownership=StoryArcPlacementOwnership.REFERENCED, + symlink_style=None, + rendered_reading_order=plan.values.reading_order, + source_fingerprint={}, + target_fingerprint={}, + ) + + _raise_if_cancelled(cancellation_requested) + preview = preview_story_arc_placement( + canonical_path=plan.canonical_path, + destination_root=plan.destination_root, + values=plan.values, + mode=mode, + symlink_style=symlink_style, + folder_template=plan.folder_template, + file_template=plan.file_template, + ) + target_path = preview.target_path + if target_path is None: + _raise_from_preview(preview) + raise StoryArcPlacementSafetyError( + "target_unavailable", + "Story-arc placement preview did not produce a destination", + ) + source_path = _validated_regular_source(plan.canonical_path) + root_guard = _validated_root(plan.destination_root) + _validate_target_lexically(root_guard.path, target_path) + _validate_safe_existing_parents(root_guard, target_path.parent) + _validate_path_limits(target_path) + _reject_canonical_destination(source_path, target_path) + + source_fingerprint = _fingerprint_regular_nofollow(source_path) + if preparation is not None: + current_preparation = StoryArcPlacementPreparation( + issue_story_arc_id=plan.issue_story_arc_id, + target_path=target_path, + mode=mode, + symlink_style=symlink_style, + rendered_reading_order=plan.values.reading_order, + source_fingerprint=dict(source_fingerprint), + destination_root_fingerprint=_root_fingerprint(root_guard), + ) + if current_preparation != preparation: + raise StoryArcPlacementOwnershipError( + "prepared_evidence_mismatch", + "Story-arc source, root, or target changed after durable preparation", + ) + existing_result = _resolve_existing_target( + plan=plan, + mode=mode, + symlink_style=symlink_style, + source_path=source_path, + source_fingerprint=source_fingerprint, + target_path=target_path, + existing_managed=existing_managed, + ) + if existing_result is not None: + return existing_result + + if preview.state is StoryArcPlacementPreviewState.BLOCKED: + _raise_from_preview(preview) + if preview.state is not StoryArcPlacementPreviewState.READY: + raise StoryArcPlacementSafetyError( + "preview_not_ready", + preview.reason or "Story-arc placement is not ready for execution", + ) + if existing_managed is not None: + _validate_managed_evidence( + existing_managed, + plan=plan, + mode=mode, + symlink_style=symlink_style, + target_path=target_path, + source_fingerprint=source_fingerprint, + ) + + prepared_event = StoryArcPlacementJournalEvent( + stage="prepared", + operation="publish", + issue_story_arc_id=plan.issue_story_arc_id, + mode=mode, + target_path=target_path, + source_fingerprint=dict(source_fingerprint), + target_fingerprint={}, + ) + _record_journal(journal, prepared_event) + + try: + with _open_secure_parent_directory( + root_guard, + target_path.parent, + create=True, + ) as secure_parent: + _raise_if_cancelled(cancellation_requested) + _recheck_free_destination_at(secure_parent, target_path.name) + if mode is StoryArcPlacementMode.COPY: + created_identity = _publish_copy_at( + source_path, + target_path.name, + secure_parent, + source_fingerprint, + cancellation_requested, + ) + elif mode is StoryArcPlacementMode.HARDLINK: + created_identity = _publish_hardlink_at( + source_path, + target_path.name, + secure_parent, + source_fingerprint, + ) + elif mode is StoryArcPlacementMode.SYMLINK: + if symlink_style is None: # pragma: no cover - validated above + raise StoryArcPlacementSafetyError( + "symlink_style_required", + "Symlink placement requires a style", + ) + created_identity = _publish_symlink_at( + source_path, + target_path.name, + target_path.parent, + secure_parent, + symlink_style, + ) + else: # pragma: no cover - enum exhaustiveness + raise StoryArcPlacementSafetyError( + "unsupported_mode", + f"Unsupported story-arc placement mode: {mode.value}", + ) + try: + _assert_parent_path_stable(secure_parent) + target_fingerprint = _validate_published_target_at( + mode, + symlink_style, + source_path, + source_fingerprint, + target_path.name, + secure_parent, + ) + _fsync_directory(secure_parent.parent_fd) + _assert_parent_path_stable(secure_parent) + except BaseException: + _remove_created_target_at( + secure_parent, + target_path.name, + expected_identity=created_identity, + ) + raise + except StoryArcPlacementError as exc: + _record_failure(journal, prepared_event, exc.code) + raise + except OSError as exc: + error = _categorized_os_error(mode, exc) + _record_failure(journal, prepared_event, error.code) + raise error from exc + + result = StoryArcPlacementResult( + issue_story_arc_id=plan.issue_story_arc_id, + library_file_id=plan.library_file_id, + state=StoryArcPlacementResultState.CREATED, + target_path=target_path, + mode=mode, + ownership=StoryArcPlacementOwnership.MANAGED, + symlink_style=symlink_style, + rendered_reading_order=plan.values.reading_order, + source_fingerprint=source_fingerprint, + target_fingerprint=target_fingerprint, + ) + _record_journal( + journal, + StoryArcPlacementJournalEvent( + stage="published", + operation="publish", + issue_story_arc_id=plan.issue_story_arc_id, + mode=mode, + target_path=target_path, + source_fingerprint=dict(source_fingerprint), + target_fingerprint=dict(target_fingerprint), + ), + ) + return result + + +def execute_story_arc_placement_batch( + plans: Sequence[StoryArcPlacementPlan], + *, + cancellation_requested: CancellationRequested | None = None, + journal: StoryArcPlacementJournal | None = None, +) -> tuple[StoryArcPlacementResult, ...]: + """Execute a caller-bounded batch with cancellation between memberships.""" + if len(plans) > MAX_STORY_ARC_PLACEMENTS_PER_BATCH: + raise ValueError( + "Story-arc placement batch exceeds the bounded execution limit of " + f"{MAX_STORY_ARC_PLACEMENTS_PER_BATCH}" + ) + results: list[StoryArcPlacementResult] = [] + for plan in plans: + _raise_if_cancelled(cancellation_requested) + results.append( + execute_story_arc_placement( + plan, + cancellation_requested=cancellation_requested, + journal=journal, + ) + ) + return tuple(results) + + +def repair_managed_story_arc_placement( + plan: StoryArcPlacementPlan, + evidence: ManagedStoryArcPlacementEvidence, + *, + cancellation_requested: CancellationRequested | None = None, + journal: StoryArcPlacementJournal | None = None, +) -> StoryArcPlacementResult: + """Idempotently recreate a missing managed artifact after evidence checks.""" + _require_managed_ownership(evidence) + return execute_story_arc_placement( + plan, + existing_managed=evidence, + cancellation_requested=cancellation_requested, + journal=journal, + ) + + +def remove_managed_story_arc_placement( + evidence: ManagedStoryArcPlacementEvidence, + *, + destination_root: Path, + canonical_path: Path | None, + journal: StoryArcPlacementJournal | None = None, +) -> StoryArcPlacementRemovalResult: + """Remove only an unchanged, action-owned managed placement artifact.""" + mode, _ownership, _symlink_style = _require_managed_ownership(evidence) + root_guard = _validated_root(destination_root) + target_path = Path(evidence.placement_path) + _validate_target_lexically(root_guard.path, target_path) + if canonical_path is not None: + _reject_canonical_destination(canonical_path.resolve(strict=False), target_path) + + prepared: StoryArcPlacementJournalEvent | None = None + try: + with _open_secure_parent_directory( + root_guard, + target_path.parent, + create=False, + ) as secure_parent: + if not _entry_exists_at(secure_parent.parent_fd, target_path.name): + return StoryArcPlacementRemovalResult(target_path, removed=False) + actual_fingerprint = _fingerprint_target_at( + secure_parent, + target_path.name, + canonical_path=canonical_path, + ) + if actual_fingerprint != evidence.target_fingerprint: + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc placement changed after it was recorded", + ) + _validate_removal_representation_at( + mode, + secure_parent, + target_path.name, + ) + + prepared = StoryArcPlacementJournalEvent( + stage="remove_prepared", + operation="remove", + issue_story_arc_id=evidence.issue_story_arc_id, + mode=mode, + target_path=target_path, + source_fingerprint=dict(evidence.source_fingerprint), + target_fingerprint=dict(evidence.target_fingerprint), + ) + _record_journal(journal, prepared) + _assert_parent_path_stable(secure_parent) + if ( + not _entry_exists_at(secure_parent.parent_fd, target_path.name) + or _fingerprint_target_at( + secure_parent, + target_path.name, + canonical_path=canonical_path, + ) + != evidence.target_fingerprint + ): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc placement changed before removal", + ) + os.unlink(target_path.name, dir_fd=secure_parent.parent_fd) + _fsync_directory(secure_parent.parent_fd) + _assert_parent_path_stable(secure_parent) + except StoryArcPlacementError as exc: + if prepared is not None: + _record_failure(journal, prepared, exc.code) + raise + except OSError as exc: + error = _categorized_os_error(mode, exc) + if prepared is not None: + _record_failure(journal, prepared, error.code) + raise error from exc + _record_journal( + journal, + StoryArcPlacementJournalEvent( + stage="removed", + operation="remove", + issue_story_arc_id=evidence.issue_story_arc_id, + mode=mode, + target_path=target_path, + source_fingerprint=dict(evidence.source_fingerprint), + target_fingerprint=dict(evidence.target_fingerprint), + ), + ) + return StoryArcPlacementRemovalResult(target_path, removed=True) + + +def _coerce_mode(value: StoryArcPlacementMode | str) -> StoryArcPlacementMode: + try: + return StoryArcPlacementMode(value) + except ValueError as exc: + raise ValueError(f"Unsupported story-arc placement mode: {value}") from exc + + +def _coerce_symlink_style( + value: StoryArcSymlinkStyle | str | None, +) -> StoryArcSymlinkStyle | None: + if value is None: + return None + try: + return StoryArcSymlinkStyle(value) + except ValueError as exc: + raise ValueError(f"Unsupported story-arc symlink style: {value}") from exc + + +def _validate_mode_and_style( + mode: StoryArcPlacementMode, + symlink_style: StoryArcSymlinkStyle | None, +) -> None: + if mode is StoryArcPlacementMode.SYMLINK and symlink_style is None: + raise ValueError("Story-arc symlink mode requires a symlink style") + if mode is not StoryArcPlacementMode.SYMLINK and symlink_style is not None: + raise ValueError("Story-arc symlink style is only valid for symlink mode") + + +def _validated_regular_source(source_path: Path | None) -> Path: + if source_path is None or not source_path.is_absolute(): + raise StoryArcPlacementSafetyError( + "source_unavailable", + "Canonical story-arc source must be an absolute path", + ) + if source_path.is_symlink(): + raise StoryArcPlacementSafetyError( + "source_symlink", + "Canonical story-arc source cannot be a symbolic link", + ) + try: + resolved = source_path.resolve(strict=True) + source_stat = resolved.stat() + except OSError as exc: + raise StoryArcPlacementSafetyError( + "source_unavailable", + "Canonical story-arc source is unavailable", + ) from exc + if not stat.S_ISREG(source_stat.st_mode): + raise StoryArcPlacementSafetyError( + "source_unavailable", + "Canonical story-arc source is not a regular file", + ) + return resolved + + +def _validated_root(root: Path | None) -> _RootGuard: + if root is None or not root.is_absolute(): + raise StoryArcPlacementSafetyError( + "root_unavailable", + "Story-arc destination root must be an absolute path", + ) + if root.is_symlink(): + raise StoryArcPlacementSafetyError( + "symlink_root", + "Story-arc destination root cannot be a symbolic link", + ) + try: + resolved = root.resolve(strict=True) + root_stat = root.stat() + except OSError as exc: + raise StoryArcPlacementSafetyError( + "root_unavailable", + "Story-arc destination root is unavailable", + ) from exc + if not resolved.is_dir(): + raise StoryArcPlacementSafetyError( + "root_unavailable", + "Story-arc destination root is not a directory", + ) + return _RootGuard( + path=root, + resolved_path=resolved, + device=root_stat.st_dev, + inode=root_stat.st_ino, + ) + + +def _root_fingerprint(root_guard: _RootGuard) -> Fingerprint: + return { + "schema_version": 1, + "kind": "directory", + "path": str(root_guard.path), + "resolved_path": str(root_guard.resolved_path), + "device": root_guard.device, + "inode": root_guard.inode, + } + + +def _secure_dir_fd_supported() -> bool: + """Return whether this runtime can anchor every mutation to a directory fd.""" + return _SECURE_DIR_FD_SUPPORTED + + +def _require_secure_dir_fd_support() -> None: + if not _secure_dir_fd_supported(): + raise StoryArcPlacementSafetyError( + "secure_dir_fd_unavailable", + "This platform cannot safely anchor story-arc filesystem operations", + ) + + +def _directory_open_flags() -> int: + return ( + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + + +def _open_directory_component(name: str, *, parent_fd: int) -> int: + try: + descriptor = os.open(name, _directory_open_flags(), dir_fd=parent_fd) + except OSError as exc: + if exc.errno in {errno.ELOOP, errno.EMLINK}: + raise StoryArcPlacementSafetyError( + "symlink_parent", + "Story-arc destination has a symbolic-link parent", + ) from exc + if exc.errno == errno.ENOTDIR: + raise StoryArcPlacementSafetyError( + "parent_not_directory", + "Story-arc destination parent is not a directory", + ) from exc + raise + opened = os.fstat(descriptor) + if not stat.S_ISDIR(opened.st_mode): + os.close(descriptor) + raise StoryArcPlacementSafetyError( + "parent_not_directory", + "Story-arc destination parent is not a directory", + ) + return descriptor + + +def _fsync_directory(descriptor: int) -> None: + try: + os.fsync(descriptor) + except OSError as exc: + raise StoryArcPlacementSafetyError( + "durability_unavailable", + "Story-arc directory changes could not be durably synchronized", + ) from exc + + +@contextmanager +def _open_secure_parent_directory( + root_guard: _RootGuard, + parent: Path, + *, + create: bool, +) -> Iterator[_SecureParentDirectory]: + """Open a target parent one no-follow component at a time. + + All later entry operations use the returned ``parent_fd``. Renaming or + replacing any pathname component therefore cannot redirect a publish or + removal outside the directory that was actually inspected. + """ + _require_secure_dir_fd_support() + relative = parent.relative_to(root_guard.path) + try: + root_fd = os.open(root_guard.path, _directory_open_flags()) + except OSError as exc: + code = "symlink_root" if exc.errno in {errno.ELOOP, errno.EMLINK} else "root_changed" + raise StoryArcPlacementSafetyError( + code, + "Story-arc destination root changed during execution", + ) from exc + current_fd = root_fd + try: + root_stat = os.fstat(root_fd) + if ( + not stat.S_ISDIR(root_stat.st_mode) + or root_stat.st_dev != root_guard.device + or root_stat.st_ino != root_guard.inode + ): + raise StoryArcPlacementSafetyError( + "root_changed", + "Story-arc destination root changed during execution", + ) + for part in relative.parts: + created = False + try: + child_fd = _open_directory_component(part, parent_fd=current_fd) + except FileNotFoundError: + if not create: + raise + try: + os.mkdir(part, mode=0o755, dir_fd=current_fd) + created = True + except FileExistsError: + pass + child_fd = _open_directory_component(part, parent_fd=current_fd) + try: + if created: + _fsync_directory(child_fd) + _fsync_directory(current_fd) + except BaseException: + os.close(child_fd) + raise + if current_fd != root_fd: + os.close(current_fd) + current_fd = child_fd + parent_stat = os.fstat(current_fd) + yield _SecureParentDirectory( + root_guard=root_guard, + path=parent, + root_fd=root_fd, + parent_fd=current_fd, + device=parent_stat.st_dev, + inode=parent_stat.st_ino, + ) + finally: + if current_fd != root_fd: + os.close(current_fd) + os.close(root_fd) + + +def _assert_parent_path_stable(parent: _SecureParentDirectory) -> None: + """Fail when the configured path no longer identifies the pinned parent.""" + _recheck_root(parent.root_guard) + try: + with _open_secure_parent_directory( + parent.root_guard, + parent.path, + create=False, + ) as reopened: + if reopened.device != parent.device or reopened.inode != parent.inode: + raise StoryArcPlacementSafetyError( + "parent_changed", + "Story-arc destination parent changed during execution", + ) + except StoryArcPlacementSafetyError as exc: + if exc.code in {"root_changed", "symlink_root"}: + raise + raise StoryArcPlacementSafetyError( + "parent_changed", + "Story-arc destination parent changed during execution", + ) from exc + except FileNotFoundError as exc: + raise StoryArcPlacementSafetyError( + "parent_changed", + "Story-arc destination parent changed during execution", + ) from exc + + +def _recheck_root(root_guard: _RootGuard) -> None: + root = root_guard.path + if root.is_symlink(): + raise StoryArcPlacementSafetyError( + "symlink_root", + "Story-arc destination root became a symbolic link", + ) + try: + current_stat = root.stat() + current_resolved = root.resolve(strict=True) + except OSError as exc: + raise StoryArcPlacementSafetyError( + "root_changed", + "Story-arc destination root changed during execution", + ) from exc + if ( + not stat.S_ISDIR(current_stat.st_mode) + or current_stat.st_dev != root_guard.device + or current_stat.st_ino != root_guard.inode + or current_resolved != root_guard.resolved_path + ): + raise StoryArcPlacementSafetyError( + "root_changed", + "Story-arc destination root changed during execution", + ) + + +def _validate_target_lexically(root: Path, target_path: Path) -> None: + try: + relative = target_path.relative_to(root) + except ValueError as exc: + raise StoryArcPlacementSafetyError( + "path_escape", + "Story-arc destination is outside the selected root", + ) from exc + if not relative.parts or any(part in {"", ".", ".."} for part in relative.parts): + raise StoryArcPlacementSafetyError( + "path_escape", + "Story-arc destination contains an unsafe path component", + ) + + +def _validate_safe_existing_parents(root_guard: _RootGuard, parent: Path) -> None: + """Reject every existing symlink parent and every realpath escape.""" + _recheck_root(root_guard) + root = root_guard.path + relative = parent.relative_to(root) + resolved_root = root_guard.resolved_path + current = root + for part in relative.parts: + current = current / part + if current.is_symlink(): + try: + resolved = current.resolve(strict=True) + except OSError as exc: + raise StoryArcPlacementSafetyError( + "symlink_parent", + "Story-arc destination has an unsafe symbolic-link parent", + ) from exc + code = "path_escape" if not resolved.is_relative_to(resolved_root) else "symlink_parent" + raise StoryArcPlacementSafetyError( + code, + "Story-arc destination has a symbolic-link parent", + ) + if current.exists() and not current.is_dir(): + raise StoryArcPlacementSafetyError( + "parent_not_directory", + "Story-arc destination parent is not a directory", + ) + if not current.exists(): + break + _recheck_root(root_guard) + + +def _ensure_safe_destination_parent(root_guard: _RootGuard, parent: Path) -> None: + _recheck_root(root_guard) + root = root_guard.path + relative = parent.relative_to(root) + resolved_root = root_guard.resolved_path + current = root + for part in relative.parts: + current = current / part + if current.is_symlink(): + raise StoryArcPlacementSafetyError( + "symlink_parent", + "Story-arc destination has a symbolic-link parent", + ) + if current.exists(): + if not current.is_dir(): + raise StoryArcPlacementSafetyError( + "parent_not_directory", + "Story-arc destination parent is not a directory", + ) + else: + with suppress(FileExistsError): + current.mkdir() + if current.is_symlink() or not current.is_dir(): + raise StoryArcPlacementSafetyError( + "symlink_parent", + "Story-arc destination parent changed during creation", + ) + resolved_current = current.resolve(strict=True) + if not resolved_current.is_relative_to(resolved_root): + raise StoryArcPlacementSafetyError( + "path_escape", + "Story-arc destination parent resolves outside the selected root", + ) + _recheck_root(root_guard) + + +def _validate_path_limits(target_path: Path) -> None: + if len(os.fsencode(target_path.name)) > 255: + raise StoryArcPlacementSafetyError( + "name_too_long", + "Rendered story-arc filename exceeds the supported length", + ) + if len(os.fsencode(target_path)) > 4096: + raise StoryArcPlacementSafetyError( + "path_too_long", + "Rendered story-arc path exceeds the supported length", + ) + if len(str(target_path)) > 1000: + raise StoryArcPlacementSafetyError( + "path_too_long", + "Rendered story-arc path exceeds the database path limit", + ) + + +def _reject_canonical_destination(source_path: Path, target_path: Path) -> None: + if os.path.normcase(os.path.abspath(source_path)) == os.path.normcase( + os.path.abspath(target_path) + ): + raise StoryArcPlacementSafetyError( + "canonical_destination", + "Story-arc placement path cannot be the canonical source path", + ) + + +def _resolve_existing_target( + *, + plan: StoryArcPlacementPlan, + mode: StoryArcPlacementMode, + symlink_style: StoryArcSymlinkStyle | None, + source_path: Path, + source_fingerprint: Fingerprint, + target_path: Path, + existing_managed: ManagedStoryArcPlacementEvidence | None, +) -> StoryArcPlacementResult | None: + case_collision = _case_only_collision(target_path) + if case_collision is not None: + raise StoryArcPlacementCollisionError( + "case_only", + "A case-only story-arc destination collision exists", + ) + if not _path_exists(target_path): + return None + + target_fingerprint = _fingerprint_target(target_path) + if existing_managed is not None: + _validate_managed_evidence( + existing_managed, + plan=plan, + mode=mode, + symlink_style=symlink_style, + target_path=target_path, + source_fingerprint=source_fingerprint, + ) + if target_fingerprint != existing_managed.target_fingerprint: + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc placement changed after it was recorded", + ) + _validate_existing_representation(mode, symlink_style, source_path, target_path) + return StoryArcPlacementResult( + issue_story_arc_id=plan.issue_story_arc_id, + library_file_id=plan.library_file_id, + state=StoryArcPlacementResultState.IDEMPOTENT, + target_path=target_path, + mode=mode, + ownership=StoryArcPlacementOwnership.MANAGED, + symlink_style=symlink_style, + rendered_reading_order=plan.values.reading_order, + source_fingerprint=source_fingerprint, + target_fingerprint=target_fingerprint, + ) + + same_inode = False + with suppress(OSError): + same_inode = target_path.samefile(source_path) + identical = same_inode or _target_content_matches_source( + target_path, + source_path, + source_fingerprint, + ) + if identical and plan.adopt_identical_existing: + return StoryArcPlacementResult( + issue_story_arc_id=plan.issue_story_arc_id, + library_file_id=plan.library_file_id, + state=StoryArcPlacementResultState.REFERENCED_EXISTING, + target_path=target_path, + mode=StoryArcPlacementMode.REFERENCE_ONLY, + ownership=StoryArcPlacementOwnership.REFERENCED, + symlink_style=None, + rendered_reading_order=plan.values.reading_order, + source_fingerprint=source_fingerprint, + target_fingerprint=target_fingerprint, + ) + if identical: + raise StoryArcPlacementCollisionError( + "identical_unconfirmed", + "An identical user artifact requires explicit referenced-placement confirmation", + ) + raise StoryArcPlacementCollisionError( + "different_content", + "A different artifact already exists at the story-arc destination", + ) + + +def _validate_managed_evidence( + evidence: ManagedStoryArcPlacementEvidence, + *, + plan: StoryArcPlacementPlan, + mode: StoryArcPlacementMode, + symlink_style: StoryArcSymlinkStyle | None, + target_path: Path, + source_fingerprint: Fingerprint, +) -> None: + evidence_mode, _ownership, evidence_style = _require_managed_ownership(evidence) + if evidence.issue_story_arc_id != plan.issue_story_arc_id: + raise StoryArcPlacementOwnershipError( + "membership_mismatch", + "Managed placement evidence belongs to another membership", + ) + if Path(evidence.placement_path) != target_path: + raise StoryArcPlacementOwnershipError( + "destination_mismatch", + "Managed placement evidence belongs to another destination", + ) + if evidence_mode is not mode or evidence_style is not symlink_style: + raise StoryArcPlacementOwnershipError( + "mode_changed", + "Managed placement evidence does not match the requested mode", + ) + if evidence.source_fingerprint != source_fingerprint: + raise StoryArcPlacementSafetyError( + "source_fingerprint_mismatch", + "Canonical source changed after the managed placement was recorded", + ) + if not evidence.target_fingerprint: + raise StoryArcPlacementOwnershipError( + "target_fingerprint_missing", + "Managed placement evidence has no target fingerprint", + ) + + +def _require_managed_ownership( + evidence: ManagedStoryArcPlacementEvidence, +) -> tuple[StoryArcPlacementMode, StoryArcPlacementOwnership, StoryArcSymlinkStyle | None]: + mode = _coerce_mode(evidence.mode) + try: + ownership = StoryArcPlacementOwnership(evidence.ownership) + except ValueError as exc: + raise StoryArcPlacementOwnershipError( + "ownership_invalid", + "Story-arc placement ownership evidence is invalid", + ) from exc + symlink_style = _coerce_symlink_style(evidence.symlink_style) + _validate_mode_and_style(mode, symlink_style) + if ( + ownership is not StoryArcPlacementOwnership.MANAGED + or mode is StoryArcPlacementMode.REFERENCE_ONLY + or evidence.creating_action_id is None + ): + raise StoryArcPlacementOwnershipError( + "not_managed", + "Only an action-owned managed placement may be repaired or removed", + ) + return mode, ownership, symlink_style + + +def _validate_existing_representation( + mode: StoryArcPlacementMode, + symlink_style: StoryArcSymlinkStyle | None, + source_path: Path, + target_path: Path, +) -> None: + if mode is StoryArcPlacementMode.SYMLINK: + if not target_path.is_symlink() or target_path.resolve(strict=True) != source_path: + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc symlink no longer resolves to its canonical source", + ) + link_target = os.readlink(target_path) + if symlink_style is StoryArcSymlinkStyle.ABSOLUTE and not os.path.isabs(link_target): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc symlink style changed", + ) + if symlink_style is StoryArcSymlinkStyle.RELATIVE and os.path.isabs(link_target): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc symlink style changed", + ) + return + if target_path.is_symlink() or not target_path.is_file(): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc artifact type changed", + ) + if mode is StoryArcPlacementMode.HARDLINK and not target_path.samefile(source_path): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed hardlink no longer references its canonical source", + ) + + +def _recheck_free_destination_at( + parent: _SecureParentDirectory, + target_name: str, +) -> None: + if _case_only_collision_at(parent.parent_fd, target_name) is not None: + raise StoryArcPlacementCollisionError( + "case_only", + "A case-only story-arc destination collision appeared during execution", + ) + if _entry_exists_at(parent.parent_fd, target_name): + raise StoryArcPlacementCollisionError( + "destination_exists", + "Story-arc destination appeared during execution; it was not overwritten", + ) + + +def _create_temporary_file_at(parent_fd: int) -> tuple[int, str]: + flags = ( + os.O_WRONLY + | os.O_CREAT + | os.O_EXCL + | getattr(os, "O_CLOEXEC", 0) + | getattr(os, "O_NOFOLLOW", 0) + ) + for _attempt in range(32): + name = f"{_TEMP_PREFIX}{secrets.token_hex(12)}{_TEMP_SUFFIX}" + try: + return os.open(name, flags, 0o600, dir_fd=parent_fd), name + except FileExistsError: + continue + raise StoryArcPlacementSafetyError( + "temporary_name_unavailable", + "A unique story-arc temporary filename could not be allocated", + ) + + +def _publish_copy_at( + source_path: Path, + target_name: str, + parent: _SecureParentDirectory, + source_fingerprint: Fingerprint, + cancellation_requested: CancellationRequested | None, +) -> _PublishedTarget: + source_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + source_fd = os.open(source_path, source_flags) + except OSError as exc: + raise StoryArcPlacementSafetyError( + "source_changed", + "Canonical source changed before story-arc copy", + ) from exc + temporary_fd = -1 + temporary_name = "" + published = False + published_stat: os.stat_result | None = None + try: + source_stat = os.fstat(source_fd) + if not stat.S_ISREG(source_stat.st_mode) or not _stat_matches_fingerprint( + source_stat, + source_fingerprint, + ): + raise StoryArcPlacementSafetyError( + "source_changed", + "Canonical source changed before story-arc copy", + ) + temporary_fd, temporary_name = _create_temporary_file_at(parent.parent_fd) + digest = hashlib.sha256() + with ( + os.fdopen(source_fd, "rb", closefd=False) as source, + os.fdopen(temporary_fd, "wb", closefd=False) as target, + ): + while chunk := source.read(_COPY_CHUNK_BYTES): + _raise_if_cancelled(cancellation_requested) + target.write(chunk) + digest.update(chunk) + target.flush() + os.fchmod(temporary_fd, stat.S_IMODE(source_stat.st_mode)) + os.utime( + temporary_fd, + ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns), + ) + os.fsync(temporary_fd) + source_after = os.fstat(source_fd) + temporary_stat = os.fstat(temporary_fd) + if ( + temporary_stat.st_size != source_fingerprint.get("size") + or digest.hexdigest() != source_fingerprint.get("sha256") + or not _stat_matches_fingerprint(source_after, source_fingerprint) + ): + raise StoryArcPlacementSafetyError( + "source_changed", + "Canonical source changed during story-arc copy", + ) + _raise_if_cancelled(cancellation_requested) + try: + os.link( + temporary_name, + target_name, + src_dir_fd=parent.parent_fd, + dst_dir_fd=parent.parent_fd, + follow_symlinks=False, + ) + except FileExistsError as exc: + raise StoryArcPlacementCollisionError( + "destination_exists", + "Story-arc destination appeared before atomic copy publish", + ) from exc + published = True + finally: + try: + if temporary_name: + try: + os.unlink(temporary_name, dir_fd=parent.parent_fd) + except FileNotFoundError: + pass + except OSError: + if not published: + raise + if published: + # Removing the temporary hardlink changes ctime. Capture from + # the still-open descriptor, never a potentially replaced path. + published_stat = os.fstat(temporary_fd) + finally: + if temporary_fd >= 0: + os.close(temporary_fd) + os.close(source_fd) + if published_stat is None: # pragma: no cover - publish or raise + raise StoryArcPlacementSafetyError( + "publish_validation_failed", + "Atomic story-arc copy publication produced no filesystem identity", + ) + return _PublishedTarget(published_stat, sha256=digest.hexdigest()) + + +def _publish_hardlink_at( + source_path: Path, + target_name: str, + parent: _SecureParentDirectory, + source_fingerprint: Fingerprint, +) -> _PublishedTarget: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(source_path, flags) + try: + source_stat = os.fstat(descriptor) + if not _stat_matches_fingerprint(source_stat, source_fingerprint): + raise StoryArcPlacementSafetyError("source_changed", "Canonical source changed") + if source_stat.st_dev != os.fstat(parent.parent_fd).st_dev: + raise StoryArcPlacementSafetyError( + "cross_device", + "Hardlink source and story-arc destination are on different filesystems", + ) + os.link( + source_path, + target_name, + dst_dir_fd=parent.parent_fd, + follow_symlinks=False, + ) + return _PublishedTarget(os.fstat(descriptor), sha256=str(source_fingerprint["sha256"])) + except OSError as exc: + if exc.errno == errno.EXDEV: + raise StoryArcPlacementSafetyError( + "cross_device", + "Hardlink source and story-arc destination are on different filesystems", + ) from exc + raise + finally: + os.close(descriptor) + + +def _publish_symlink_at( + source_path: Path, + target_name: str, + target_parent: Path, + parent: _SecureParentDirectory, + symlink_style: StoryArcSymlinkStyle, +) -> _PublishedTarget: + link_target = ( + str(source_path) + if symlink_style is StoryArcSymlinkStyle.ABSOLUTE + else os.path.relpath(source_path, start=target_parent.resolve(strict=True)) + ) + prospective = Path(link_target) + if not prospective.is_absolute(): + prospective = target_parent / prospective + if prospective.resolve(strict=True) != source_path: + raise StoryArcPlacementSafetyError( + "symlink_target_mismatch", + "Story-arc symlink would not resolve to the canonical source", + ) + os.symlink(link_target, target_name, dir_fd=parent.parent_fd) + created = _entry_lstat_at(parent.parent_fd, target_name) + if not stat.S_ISLNK(created.st_mode): + raise StoryArcPlacementSafetyError( + "publish_validation_failed", + "Published story-arc symlink changed before it could be validated", + ) + return _PublishedTarget(created, link_target=link_target) + + +def _validate_published_target_at( + mode: StoryArcPlacementMode, + symlink_style: StoryArcSymlinkStyle | None, + source_path: Path, + source_fingerprint: Fingerprint, + target_name: str, + parent: _SecureParentDirectory, +) -> Fingerprint: + _validate_existing_representation_at( + mode, + symlink_style, + source_path, + source_fingerprint, + target_name, + parent, + ) + target_fingerprint = _fingerprint_target_at( + parent, + target_name, + canonical_path=source_path, + ) + content = ( + target_fingerprint.get("content") + if target_fingerprint.get("kind") == "symlink" + else target_fingerprint + ) + if not isinstance(content, dict) or content.get("sha256") != source_fingerprint.get("sha256"): + raise StoryArcPlacementSafetyError( + "publish_validation_failed", + "Published story-arc artifact does not match its canonical source", + ) + if mode is StoryArcPlacementMode.SYMLINK and content != source_fingerprint: + raise StoryArcPlacementSafetyError( + "source_changed", + "Canonical source changed while its story-arc symlink was published", + ) + return target_fingerprint + + +def _remove_created_target_at( + parent: _SecureParentDirectory, + target_name: str, + *, + expected_identity: _PublishedTarget, +) -> None: + try: + current = _entry_lstat_at(parent.parent_fd, target_name) + if _publication_stat_identity(current) != _publication_stat_identity( + expected_identity.stat + ): + return + if expected_identity.link_target is not None: + if os.readlink(target_name, dir_fd=parent.parent_fd) != expected_identity.link_target: + return + elif ( + _fingerprint_regular_at(parent.parent_fd, target_name)["sha256"] + != expected_identity.sha256 + ): + return + # Hashing may take time. Recheck metadata before removal, including + # ctime so a recycled inode or an in-place edit does not prove ownership. + current = _entry_lstat_at(parent.parent_fd, target_name) + if _publication_stat_identity(current) != _publication_stat_identity( + expected_identity.stat + ): + return + os.unlink(target_name, dir_fd=parent.parent_fd) + _fsync_directory(parent.parent_fd) + except (OSError, StoryArcPlacementError): + # If ownership cannot be proven, preserve the entry and original error. + pass + + +def _publication_stat_identity(value: os.stat_result) -> tuple[int, int, int, int, int, int]: + return ( + value.st_dev, + value.st_ino, + value.st_mode, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def _stat_matches_fingerprint(current: os.stat_result, fingerprint: Fingerprint) -> bool: + return ( + current.st_dev == fingerprint.get("device") + and current.st_ino == fingerprint.get("inode") + and current.st_size == fingerprint.get("size") + and current.st_mtime_ns == fingerprint.get("mtime_ns") + ) + + +def _entry_lstat_at(parent_fd: int, name: str) -> os.stat_result: + return os.stat(name, dir_fd=parent_fd, follow_symlinks=False) + + +def _entry_exists_at(parent_fd: int, name: str) -> bool: + try: + _entry_lstat_at(parent_fd, name) + except FileNotFoundError: + return False + return True + + +def _fingerprint_regular_descriptor(descriptor: int) -> Fingerprint: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode): + raise StoryArcPlacementSafetyError( + "not_regular_file", + "Story-arc artifact is not a regular file", + ) + os.lseek(descriptor, 0, os.SEEK_SET) + digest = hashlib.sha256() + while chunk := os.read(descriptor, _COPY_CHUNK_BYTES): + digest.update(chunk) + after = os.fstat(descriptor) + identity_before = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) + identity_after = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) + if identity_before != identity_after: + raise StoryArcPlacementSafetyError( + "source_changed", + "Story-arc artifact changed while it was fingerprinted", + ) + return { + "schema_version": 1, + "kind": "regular", + "size": after.st_size, + "mtime_ns": after.st_mtime_ns, + "device": after.st_dev, + "inode": after.st_ino, + "sha256": digest.hexdigest(), + } + + +def _fingerprint_regular_nofollow(path: Path) -> Fingerprint: + try: + before = path.lstat() + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + except OSError as exc: + raise StoryArcPlacementSafetyError( + "source_changed", + "Canonical source changed while it was opened", + ) from exc + try: + opened = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or opened.st_dev != before.st_dev + or opened.st_ino != before.st_ino + ): + raise StoryArcPlacementSafetyError( + "source_changed", + "Canonical source changed while it was opened", + ) + fingerprint = _fingerprint_regular_descriptor(descriptor) + finally: + os.close(descriptor) + try: + after = path.lstat() + except OSError as exc: + raise StoryArcPlacementSafetyError( + "source_changed", + "Canonical source changed while it was fingerprinted", + ) from exc + if after.st_dev != before.st_dev or after.st_ino != before.st_ino: + raise StoryArcPlacementSafetyError( + "source_changed", + "Canonical source changed while it was fingerprinted", + ) + return fingerprint + + +def _fingerprint_regular_at(parent_fd: int, name: str) -> Fingerprint: + before = _entry_lstat_at(parent_fd, name) + if not stat.S_ISREG(before.st_mode): + raise StoryArcPlacementSafetyError( + "not_regular_file", + "Story-arc artifact is not a regular file", + ) + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(name, flags, dir_fd=parent_fd) + except OSError as exc: + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Story-arc artifact changed while it was opened", + ) from exc + try: + opened = os.fstat(descriptor) + if opened.st_dev != before.st_dev or opened.st_ino != before.st_ino: + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Story-arc artifact changed while it was opened", + ) + fingerprint = _fingerprint_regular_descriptor(descriptor) + finally: + os.close(descriptor) + after = _entry_lstat_at(parent_fd, name) + if after.st_dev != before.st_dev or after.st_ino != before.st_ino: + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Story-arc artifact changed while it was fingerprinted", + ) + return fingerprint + + +def _fingerprint_target_at( + parent: _SecureParentDirectory, + target_name: str, + *, + canonical_path: Path | None, +) -> Fingerprint: + before = _entry_lstat_at(parent.parent_fd, target_name) + if not stat.S_ISLNK(before.st_mode): + return _fingerprint_regular_at(parent.parent_fd, target_name) + if canonical_path is None: + raise StoryArcPlacementSafetyError( + "canonical_source_required", + "Managed story-arc symlink removal requires its canonical source", + ) + link_target = os.readlink(target_name, dir_fd=parent.parent_fd) + after = _entry_lstat_at(parent.parent_fd, target_name) + if ( + before.st_dev != after.st_dev + or before.st_ino != after.st_ino + or before.st_mtime_ns != after.st_mtime_ns + or os.readlink(target_name, dir_fd=parent.parent_fd) != link_target + ): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Story-arc symlink changed while it was fingerprinted", + ) + prospective = Path(link_target) + if not prospective.is_absolute(): + prospective = parent.path / prospective + try: + resolved = prospective.resolve(strict=True) + canonical = canonical_path.resolve(strict=True) + except OSError as exc: + raise StoryArcPlacementSafetyError( + "dangling_symlink", + "Story-arc symlink target is unavailable", + ) from exc + if resolved != canonical: + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc symlink no longer resolves to its canonical source", + ) + return { + "schema_version": 1, + "kind": "symlink", + "link_target": link_target, + "device": after.st_dev, + "inode": after.st_ino, + "mtime_ns": after.st_mtime_ns, + "content": _fingerprint_regular_nofollow(canonical), + } + + +def _validate_existing_representation_at( + mode: StoryArcPlacementMode, + symlink_style: StoryArcSymlinkStyle | None, + source_path: Path, + source_fingerprint: Fingerprint, + target_name: str, + parent: _SecureParentDirectory, +) -> None: + target_stat = _entry_lstat_at(parent.parent_fd, target_name) + if mode is StoryArcPlacementMode.SYMLINK: + if not stat.S_ISLNK(target_stat.st_mode): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc symlink was replaced", + ) + link_target = os.readlink(target_name, dir_fd=parent.parent_fd) + expected = ( + str(source_path) + if symlink_style is StoryArcSymlinkStyle.ABSOLUTE + else os.path.relpath(source_path, start=parent.path.resolve(strict=True)) + ) + if link_target != expected: + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc symlink style or target changed", + ) + return + if not stat.S_ISREG(target_stat.st_mode): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc artifact type changed", + ) + if mode is StoryArcPlacementMode.HARDLINK and ( + target_stat.st_dev != source_fingerprint.get("device") + or target_stat.st_ino != source_fingerprint.get("inode") + ): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed hardlink no longer references its canonical source", + ) + + +def _validate_removal_representation_at( + mode: StoryArcPlacementMode, + parent: _SecureParentDirectory, + target_name: str, +) -> None: + target_stat = _entry_lstat_at(parent.parent_fd, target_name) + if mode is StoryArcPlacementMode.SYMLINK: + if not stat.S_ISLNK(target_stat.st_mode): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc symlink was replaced", + ) + return + if not stat.S_ISREG(target_stat.st_mode): + raise StoryArcPlacementSafetyError( + "fingerprint_mismatch", + "Managed story-arc artifact type changed after it was recorded", + ) + + +def _case_only_collision_at(parent_fd: int, target_name: str) -> str | None: + target_key = target_name.casefold() + try: + with os.scandir(parent_fd) as entries: + for index, child in enumerate(entries, start=1): + if index > _MAX_CASE_SCAN_ENTRIES: + raise StoryArcPlacementSafetyError( + "directory_scan_limit", + "Story-arc collision scan exceeded its bounded entry limit", + ) + if child.name != target_name and child.name.casefold() == target_key: + return child.name + except StoryArcPlacementError: + raise + except OSError as exc: + raise StoryArcPlacementSafetyError( + "collision_scan_failed", + "Story-arc destination collision scan failed", + ) from exc + return None + + +def _fingerprint_regular(path: Path) -> Fingerprint: + before = path.stat() + if not stat.S_ISREG(before.st_mode): + raise StoryArcPlacementSafetyError( + "not_regular_file", + "Story-arc artifact is not a regular file", + ) + digest = hashlib.sha256() + with path.open("rb") as artifact: + while chunk := artifact.read(_COPY_CHUNK_BYTES): + digest.update(chunk) + after = path.stat() + identity_before = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) + identity_after = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) + if identity_before != identity_after: + raise StoryArcPlacementSafetyError( + "source_changed", + "Story-arc artifact changed while it was fingerprinted", + ) + return { + "schema_version": 1, + "kind": "regular", + "size": after.st_size, + "mtime_ns": after.st_mtime_ns, + "device": after.st_dev, + "inode": after.st_ino, + "sha256": digest.hexdigest(), + } + + +def _fingerprint_target(path: Path) -> Fingerprint: + if not path.is_symlink(): + return _fingerprint_regular(path) + link_stat = path.lstat() + link_target = os.readlink(path) + try: + resolved = path.resolve(strict=True) + except OSError as exc: + raise StoryArcPlacementSafetyError( + "dangling_symlink", + "Story-arc symlink target is unavailable", + ) from exc + return { + "schema_version": 1, + "kind": "symlink", + "link_target": link_target, + "device": link_stat.st_dev, + "inode": link_stat.st_ino, + "mtime_ns": link_stat.st_mtime_ns, + "content": _fingerprint_regular(resolved), + } + + +def _target_content_matches_source( + target_path: Path, + source_path: Path, + source_fingerprint: Fingerprint, +) -> bool: + try: + if target_path.is_symlink() and target_path.resolve(strict=True) == source_path: + return True + if target_path.is_symlink() or not target_path.is_file(): + return False + if target_path.stat().st_size != source_fingerprint.get("size"): + return False + return _fingerprint_regular(target_path).get("sha256") == source_fingerprint.get("sha256") + except (OSError, StoryArcPlacementError): + return False + + +def _case_only_collision(target_path: Path) -> Path | None: + parent = target_path.parent + if not parent.exists() or not parent.is_dir() or parent.is_symlink(): + return None + key = target_path.name.casefold() + try: + for index, child in enumerate(parent.iterdir(), start=1): + if index > _MAX_CASE_SCAN_ENTRIES: + raise StoryArcPlacementSafetyError( + "directory_scan_limit", + "Story-arc collision scan exceeded its bounded entry limit", + ) + if child.name != target_path.name and child.name.casefold() == key: + return child + except StoryArcPlacementError: + raise + except OSError as exc: + raise StoryArcPlacementSafetyError( + "collision_scan_failed", + "Story-arc destination collision scan failed", + ) from exc + return None + + +def _path_exists(path: Path) -> bool: + return path.exists() or path.is_symlink() + + +def _raise_from_preview(preview: StoryArcPlacementPreview) -> None: + reason = preview.reason or "Story-arc placement preview blocked execution" + code = preview.collision.value + if preview.collision in { + StoryArcCollisionKind.DIFFERENT_CONTENT, + StoryArcCollisionKind.CASE_ONLY, + }: + raise StoryArcPlacementCollisionError(code, reason) + raise StoryArcPlacementSafetyError(code, reason) + + +def _raise_if_cancelled(callback: CancellationRequested | None) -> None: + if callback is not None and callback(): + raise StoryArcPlacementCancellationError + + +def _record_journal( + journal: StoryArcPlacementJournal | None, + event: StoryArcPlacementJournalEvent, +) -> None: + if journal is not None: + journal(event) + + +def _record_failure( + journal: StoryArcPlacementJournal | None, + prepared: StoryArcPlacementJournalEvent, + failure_code: str, +) -> None: + if journal is None: + return + with suppress(Exception): + journal( + StoryArcPlacementJournalEvent( + stage="failed", + operation=prepared.operation, + issue_story_arc_id=prepared.issue_story_arc_id, + mode=prepared.mode, + target_path=prepared.target_path, + source_fingerprint=dict(prepared.source_fingerprint), + target_fingerprint={}, + failure_code=failure_code, + ) + ) + # Preserve the filesystem failure if this secondary notification fails; + # the durable prepared action still supports reconciliation. + + +def _categorized_os_error( + mode: StoryArcPlacementMode, + error: OSError, +) -> StoryArcPlacementError: + if error.errno == errno.EXDEV and mode is StoryArcPlacementMode.HARDLINK: + return StoryArcPlacementSafetyError( + "cross_device", + "Hardlink source and story-arc destination are on different filesystems", + ) + if error.errno in {errno.EEXIST, errno.ENOTEMPTY}: + return StoryArcPlacementCollisionError( + "destination_exists", + "Story-arc destination exists and was not overwritten", + ) + if error.errno in {errno.EACCES, errno.EPERM, errno.EROFS}: + return StoryArcPlacementSafetyError( + "permission_denied", + "Story-arc destination is not writable", + ) + label = "Hardlink" if mode is StoryArcPlacementMode.HARDLINK else "Story-arc placement" + return StoryArcPlacementSafetyError( + "filesystem_error", + f"{label} filesystem operation failed", + ) diff --git a/src/pullbox/services/story_arc_policy_migration.py b/src/pullbox/services/story_arc_policy_migration.py new file mode 100644 index 00000000..dd6c0a77 --- /dev/null +++ b/src/pullbox/services/story_arc_policy_migration.py @@ -0,0 +1,2095 @@ +"""Read-only preparation for managed Story Arc placement-policy migration. + +The current mutation service deliberately refuses a destination-policy change +while managed placements exist. This module supplies the bounded work that +must precede any future executor: a complete, signed preview and an exact, +actor-bound confirmation check. It never mutates a Story Arc, placement, +canonical file, or referenced artifact. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import shutil +import stat +from dataclasses import dataclass +from datetime import date, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Protocol, cast + +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer +from sqlalchemy import func, or_, select + +from pullbox.core.config_resolver import get_application_secret +from pullbox.models.import_job import ( + ImportJob, + ImportJobAction, + ImportJobActionStatus, + ImportJobStatus, +) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile, LibraryRoot +from pullbox.models.publisher import Publisher +from pullbox.models.series import Series +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcPlacement, + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcPlacementState, +) +from pullbox.models.story_arc_sync import StoryArcSyncWork, StoryArcSyncWorkState +from pullbox.services.story_arc_placement_integration import ( + STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + StoryArcPlacementIntegrationError, + StoryArcPlacementPolicy, + StoryArcPlacementPolicyInput, + StoryArcPlacementPolicyMode, + _placement_policy_shape, + _PlacementContext, + _policy_from_arc, + _rendered_target_path, + validate_story_arc_placement_policy_input, +) +from pullbox.services.story_arc_placement_preview import StoryArcCollisionKind +from pullbox.services.story_arc_placement_service import ( + StoryArcPlacementInspection, + StoryArcPlacementInspectionEvidence, + StoryArcPlacementInspectionState, + StoryArcPlacementPlan, + inspect_story_arc_placement, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + from sqlalchemy.engine import Row + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql import Select + +STORY_ARC_POLICY_MIGRATION_CONFIRMATION = "CHANGE STORY ARC PLACEMENT POLICY" + +_TOKEN_SALT = "pullbox-story-arc-policy-migration-v1" +_TOKEN_SCHEMA_VERSION = 1 +_TOKEN_MAX_AGE_SECONDS = 15 * 60 +_SCAN_PAGE_SIZE = 100 +_MAX_RESPONSE_PAGE_SIZE = 100 +_ACTIVE_PLACEMENT_STATUSES = ( + "prepared", + "published_pending_reconcile", + "reference_validation_prepared", + "remove_prepared", + "rename_prepared", + "rename_recovery_required", +) +_ACTIVE_SYNC_STATES = ( + StoryArcSyncWorkState.QUEUED, + StoryArcSyncWorkState.RUNNING, + StoryArcSyncWorkState.RETRY_WAIT, + StoryArcSyncWorkState.FAILED, +) +_TERMINAL_SYNC_STATES = ( + StoryArcSyncWorkState.COMPLETED, + StoryArcSyncWorkState.CANCELLED, +) +_SETTLED_ROLLBACK_STATUSES = frozenset( + { + "cancelled_before_publish", + "managed_placement_removed", + "referenced_placement_detached", + } +) +_UNCHANGED_MANAGED_STATUSES = frozenset({"complete", "rename_cancelled", "rename_failed"}) +_MANAGED_POLICY_MODES = frozenset( + { + StoryArcPlacementPolicyMode.COPY, + StoryArcPlacementPolicyMode.HARDLINK, + StoryArcPlacementPolicyMode.SYMLINK, + } +) + + +class _DigestWriter(Protocol): + def update(self, value: bytes, /) -> None: ... + + +class StoryArcPolicyMigrationError(StoryArcPlacementIntegrationError): + """Categorized failure at the read-only migration boundary.""" + + +@dataclass(frozen=True, slots=True) +class StoryArcPolicyMigrationPreviewItem: + """One old/new consequence from a complete preview, never an ORM row.""" + + placement_id: int + membership_id: int + ownership: str + action: str + old_mode: str + new_mode: str + old_path: str + new_path: str | None + collision: str + blocked: bool + reason: str | None + required_bytes: int + + +@dataclass(frozen=True, slots=True) +class StoryArcPolicyMigrationPreview: + """Complete counts and digest plus one bounded keyset item page.""" + + story_arc_id: int + expected_revision: int + current_policy: StoryArcPlacementPolicy + proposed_policy: StoryArcPlacementPolicy + scope_digest: str + preview_token: str + required_confirmation: str + total_placement_count: int + managed_migrate_count: int + managed_remove_count: int + managed_unchanged_count: int + referenced_preserved_count: int + collision_count: int + blocked_count: int + required_bytes: int + available_bytes: int | None + global_block_codes: tuple[str, ...] + items: tuple[StoryArcPolicyMigrationPreviewItem, ...] + limit: int + after_placement_id: int + next_cursor: int | None + has_more: bool + requires_confirmation: bool = True + execution_supported: bool = False + filesystem_mutated: bool = False + + +@dataclass(frozen=True, slots=True) +class StoryArcPolicyMigrationConfirmation: + """Exact confirmation result without claiming unavailable execution.""" + + story_arc_id: int + expected_revision: int + scope_digest: str + confirmed: bool = True + ready_for_execution: bool = False + execution_supported: bool = False + mutation_performed: bool = False + policy_update_block_code: str = "managed_policy_change_requires_migration" + + +@dataclass(frozen=True, slots=True) +class _MembershipScope: + context: _PlacementContext + source_ordinal: int + resolution_state: str + sync_eligible: bool + membership_updated_at: datetime + membership_evidence: dict[str, object] + materialization_result: dict[str, object] + issue_updated_at: datetime | None + series_id: int | None + series_updated_at: datetime | None + canonical_id: int | None + canonical_path: str | None + canonical_size: int | None + canonical_format: str | None + canonical_hash: str | None + canonical_modified_at: datetime | None + canonical_updated_at: datetime | None + canonical_library_root_id: int | None + canonical_storage_mode: str | None + canonical_source_signature: dict[str, object] + + def digest_record(self) -> dict[str, object]: + context = self.context + return { + "membership_id": context.membership_id, + "story_arc_id": context.story_arc_id, + "sequence_number": context.sequence_number, + "source_ordinal": self.source_ordinal, + "resolution_state": self.resolution_state, + "sync_eligible": self.sync_eligible, + "membership_updated_at": self.membership_updated_at, + "membership_evidence": self.membership_evidence, + "materialization_result": self.materialization_result, + "issue_id": context.issue_id, + "issue_number_text": context.issue_number_text, + "issue_title": context.issue_title, + "issue_updated_at": self.issue_updated_at, + "story_arc_name": context.story_arc_name, + "series_id": self.series_id, + "series_name": context.series_name, + "series_start_year": context.series_start_year, + "series_end_year": context.series_end_year, + "series_updated_at": self.series_updated_at, + "publisher_name": context.publisher_name, + "year": context.year, + "canonical_id": self.canonical_id, + "canonical_path": self.canonical_path, + "canonical_size": self.canonical_size, + "canonical_format": self.canonical_format, + "canonical_hash": self.canonical_hash, + "canonical_modified_at": self.canonical_modified_at, + "canonical_updated_at": self.canonical_updated_at, + "canonical_library_root_id": self.canonical_library_root_id, + "canonical_storage_mode": self.canonical_storage_mode, + "canonical_source_signature": self.canonical_source_signature, + } + + +@dataclass(frozen=True, slots=True) +class _PlacementScope: + id: int + membership_id: int + library_file_id: int | None + library_root_id: int | None + placement_path: str + mode: StoryArcPlacementMode + ownership: StoryArcPlacementOwnership + symlink_style: str | None + source_kind: str + creating_action_id: int | None + rendered_reading_order: int | None + policy_schema_version: int | None + operation_token: str | None + source_fingerprint: dict[str, object] + target_fingerprint: dict[str, object] + state: StoryArcPlacementState + last_result: dict[str, object] + updated_at: datetime + context: _MembershipScope + + def digest_record(self) -> dict[str, object]: + return { + "placement_id": self.id, + "membership_id": self.membership_id, + "library_file_id": self.library_file_id, + "library_root_id": self.library_root_id, + "placement_path": self.placement_path, + "mode": self.mode.value, + "ownership": self.ownership.value, + "symlink_style": self.symlink_style, + "source_kind": self.source_kind, + "creating_action_id": self.creating_action_id, + "rendered_reading_order": self.rendered_reading_order, + "policy_schema_version": self.policy_schema_version, + "operation_token": self.operation_token, + "source_fingerprint": self.source_fingerprint, + "target_fingerprint": self.target_fingerprint, + "state": self.state.value, + "last_result": self.last_result, + "updated_at": self.updated_at, + } + + def inspection_evidence(self) -> StoryArcPlacementInspectionEvidence: + return StoryArcPlacementInspectionEvidence( + placement_path=Path(self.placement_path), + mode=self.mode, + ownership=self.ownership, + symlink_style=self.symlink_style, + source_fingerprint=dict(self.source_fingerprint), + target_fingerprint=dict(self.target_fingerprint), + ) + + +@dataclass(frozen=True, slots=True) +class _OccupiedPlacement: + id: int + ownership: StoryArcPlacementOwnership + + +@dataclass(frozen=True, slots=True) +class _ClassifiedPlacement: + item: StoryArcPolicyMigrationPreviewItem + digest_record: dict[str, object] + + +@dataclass(frozen=True, slots=True) +class _PreviewCounts: + total: int = 0 + managed_migrate: int = 0 + managed_remove: int = 0 + managed_unchanged: int = 0 + referenced_preserved: int = 0 + collisions: int = 0 + blocked: int = 0 + required_bytes: int = 0 + + def add(self, item: StoryArcPolicyMigrationPreviewItem) -> _PreviewCounts: + return _PreviewCounts( + total=self.total + 1, + managed_migrate=self.managed_migrate + (item.action == "migrate_managed"), + managed_remove=self.managed_remove + (item.action == "remove_managed"), + managed_unchanged=self.managed_unchanged + (item.action == "managed_unchanged"), + referenced_preserved=self.referenced_preserved + (item.action == "preserve_referenced"), + collisions=self.collisions + (item.collision != "none"), + blocked=self.blocked + item.blocked, + required_bytes=self.required_bytes + item.required_bytes, + ) + + def token_record(self) -> dict[str, int]: + return { + "total": self.total, + "managed_migrate": self.managed_migrate, + "managed_remove": self.managed_remove, + "managed_unchanged": self.managed_unchanged, + "referenced_preserved": self.referenced_preserved, + "collisions": self.collisions, + "blocked": self.blocked, + "required_bytes": self.required_bytes, + } + + +@dataclass(frozen=True, slots=True) +class _SignedPreview: + actor_id: int + story_arc_id: int + expected_revision: int + current_policy: dict[str, object] + proposed_policy: dict[str, object] + scope_digest: str + counts: dict[str, int] + + +@dataclass(frozen=True, slots=True) +class _PolicyRootFingerprint: + resolved_path: str | None + device: int | None + inode: int | None + + def digest_record(self) -> dict[str, object]: + return { + "resolved_path": self.resolved_path, + "device": self.device, + "inode": self.inode, + } + + +@dataclass(frozen=True, slots=True) +class _TerminalSyncWorkScope: + id: int + last_result: object + origin_import_job_id: int | None + origin_import_action_id: int | None + issue_story_arc_id: int + desired_generation: str + job_id: int | None + job_status: ImportJobStatus | None + action_id: int | None + action_import_job_id: int | None + action_status: ImportJobActionStatus | None + + +class StoryArcPolicyMigrationService: + """Build and revalidate a complete migration preview without mutation.""" + + async def preview_policy_change( + self, + session: AsyncSession, + story_arc_id: int, + *, + actor_id: int, + expected_revision: int, + proposal: StoryArcPlacementPolicyInput, + limit: int = 50, + after_placement_id: int = 0, + cancellation_requested: Callable[[], bool] | None = None, + ) -> StoryArcPolicyMigrationPreview: + """Return complete counts/digest and one bounded keyset page.""" + limit, after_placement_id = _bounded_page(limit, after_placement_id) + current, proposed, arc_name = await self._validate_boundary( + session, + story_arc_id=story_arc_id, + actor_id=actor_id, + expected_revision=expected_revision, + proposal=proposal, + cancellation_requested=cancellation_requested, + ) + await session.rollback() + current_root_fingerprint = await _validate_policy_root(current, role="current") + proposed_root_fingerprint = await _validate_policy_root(proposed, role="proposed") + + first_membership_digest = await _membership_scope_digest( + session, + story_arc_id=story_arc_id, + arc_name=arc_name, + cancellation_requested=cancellation_requested, + ) + ( + first_placement_digest, + exact_target_counts, + folded_target_counts, + ) = await _placement_target_index( + session, + story_arc_id=story_arc_id, + arc_name=arc_name, + proposed=proposed, + cancellation_requested=cancellation_requested, + ) + ( + second_placement_digest, + plan_digest, + counts, + page_items, + next_cursor, + has_more, + ) = await _classify_placements( + session, + story_arc_id=story_arc_id, + arc_name=arc_name, + current=current, + proposed=proposed, + exact_target_counts=exact_target_counts, + folded_target_counts=folded_target_counts, + limit=limit, + after_placement_id=after_placement_id, + cancellation_requested=cancellation_requested, + ) + second_membership_digest = await _membership_scope_digest( + session, + story_arc_id=story_arc_id, + arc_name=arc_name, + cancellation_requested=cancellation_requested, + ) + if ( + first_membership_digest != second_membership_digest + or first_placement_digest != second_placement_digest + ): + raise StoryArcPolicyMigrationError( + "migration_scope_changed", + "Story Arc placement scope changed while the preview was generated", + category="conflict", + ) + await self._assert_boundary_stable( + session, + story_arc_id=story_arc_id, + expected_revision=expected_revision, + current=current, + cancellation_requested=cancellation_requested, + ) + await session.rollback() + if ( + await _validate_policy_root(current, role="current") != current_root_fingerprint + or await _validate_policy_root(proposed, role="proposed") != proposed_root_fingerprint + ): + raise StoryArcPolicyMigrationError( + "migration_scope_changed", + "Story Arc destination root changed while the preview was generated", + category="conflict", + ) + + global_block_codes: list[str] = [] + available_bytes: int | None = None + if proposed.mode in _MANAGED_POLICY_MODES and proposed.destination_root is not None: + try: + available_bytes = await asyncio.to_thread( + lambda: shutil.disk_usage(proposed.destination_root or "").free + ) + except OSError: + global_block_codes.append("destination_capacity_unavailable") + else: + if counts.required_bytes > available_bytes: + global_block_codes.append("insufficient_space") + + scope_digest = _digest_record( + { + "schema_version": 1, + "story_arc_id": story_arc_id, + "expected_revision": expected_revision, + "current_policy": _policy_token_record(current), + "proposed_policy": _policy_token_record(proposed), + "current_root_fingerprint": current_root_fingerprint.digest_record(), + "proposed_root_fingerprint": proposed_root_fingerprint.digest_record(), + "membership_digest": second_membership_digest, + "placement_digest": second_placement_digest, + "plan_digest": plan_digest, + "global_block_codes": global_block_codes, + } + ) + effective_counts = _PreviewCounts( + total=counts.total, + managed_migrate=counts.managed_migrate, + managed_remove=counts.managed_remove, + managed_unchanged=counts.managed_unchanged, + referenced_preserved=counts.referenced_preserved, + collisions=counts.collisions, + blocked=counts.blocked, + required_bytes=counts.required_bytes, + ) + signed = _SignedPreview( + actor_id=actor_id, + story_arc_id=story_arc_id, + expected_revision=expected_revision, + current_policy=_policy_token_record(current), + proposed_policy=_policy_token_record(proposed), + scope_digest=scope_digest, + counts=effective_counts.token_record(), + ) + token = self._serializer().dumps(_signed_preview_payload(signed)) + await session.rollback() + return StoryArcPolicyMigrationPreview( + story_arc_id=story_arc_id, + expected_revision=expected_revision, + current_policy=current, + proposed_policy=proposed, + scope_digest=scope_digest, + preview_token=token, + required_confirmation=STORY_ARC_POLICY_MIGRATION_CONFIRMATION, + total_placement_count=effective_counts.total, + managed_migrate_count=effective_counts.managed_migrate, + managed_remove_count=effective_counts.managed_remove, + managed_unchanged_count=effective_counts.managed_unchanged, + referenced_preserved_count=effective_counts.referenced_preserved, + collision_count=effective_counts.collisions, + blocked_count=effective_counts.blocked, + required_bytes=effective_counts.required_bytes, + available_bytes=available_bytes, + global_block_codes=tuple(global_block_codes), + items=page_items, + limit=limit, + after_placement_id=after_placement_id, + next_cursor=next_cursor, + has_more=has_more, + ) + + async def prepare_confirmation( + self, + session: AsyncSession, + story_arc_id: int, + *, + actor_id: int, + expected_revision: int, + proposal: StoryArcPlacementPolicyInput, + preview_token: str, + confirmation: str, + cancellation_requested: Callable[[], bool] | None = None, + ) -> StoryArcPolicyMigrationConfirmation: + """Validate exact signed intent and current scope, still without mutation.""" + if confirmation != STORY_ARC_POLICY_MIGRATION_CONFIRMATION: + raise StoryArcPolicyMigrationError( + "confirmation_required", + f'Type exactly "{STORY_ARC_POLICY_MIGRATION_CONFIRMATION}" to continue', + ) + signed = self._decode_preview(preview_token) + if ( + signed.actor_id != actor_id + or signed.story_arc_id != story_arc_id + or signed.expected_revision != expected_revision + ): + raise StoryArcPolicyMigrationError( + "invalid_preview_token", + "The migration preview does not match this actor or Story Arc policy change", + ) + refreshed = await self.preview_policy_change( + session, + story_arc_id, + actor_id=actor_id, + expected_revision=expected_revision, + proposal=proposal, + limit=1, + after_placement_id=0, + cancellation_requested=cancellation_requested, + ) + if ( + signed.current_policy != _policy_token_record(refreshed.current_policy) + or signed.proposed_policy != _policy_token_record(refreshed.proposed_policy) + or signed.scope_digest != refreshed.scope_digest + or signed.counts != _preview_counts_record(refreshed) + ): + raise StoryArcPolicyMigrationError( + "migration_preview_stale", + ( + "Story Arc placements, membership, canonical files, or policy " + "changed; preview again" + ), + category="conflict", + ) + if refreshed.blocked_count or refreshed.global_block_codes: + raise StoryArcPolicyMigrationError( + "policy_migration_blocked", + "Story Arc policy migration has blocked or colliding placements", + category="collision", + ) + await session.rollback() + return StoryArcPolicyMigrationConfirmation( + story_arc_id=story_arc_id, + expected_revision=expected_revision, + scope_digest=refreshed.scope_digest, + ) + + async def _validate_boundary( + self, + session: AsyncSession, + *, + story_arc_id: int, + actor_id: int, + expected_revision: int, + proposal: StoryArcPlacementPolicyInput, + cancellation_requested: Callable[[], bool] | None, + ) -> tuple[StoryArcPlacementPolicy, StoryArcPlacementPolicy, str]: + if not _is_positive_int(actor_id): + raise StoryArcPolicyMigrationError("invalid_actor", "Actor identity is required") + if not _is_positive_int(expected_revision): + raise StoryArcPolicyMigrationError( + "invalid_revision", "Expected Story Arc revision must be a positive integer" + ) + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise StoryArcPolicyMigrationError( + "story_arc_not_found", "Story arc was not found", category="not_found" + ) + if arc.lifecycle is StoryArcLifecycle.ARCHIVED: + raise StoryArcPolicyMigrationError( + "story_arc_archived", "Archived story arcs cannot change placement policy" + ) + if arc.revision != expected_revision: + raise StoryArcPolicyMigrationError( + "revision_conflict", + "Story Arc changed before migration preview", + category="conflict", + ) + current = _policy_from_arc(arc) + if not current.configured: + raise StoryArcPolicyMigrationError( + "placement_policy_not_configured", + "Configure the Story Arc placement policy before migrating it", + ) + if current.target_library_root_id is None or not bool( + await session.scalar( + select(LibraryRoot.enabled).where(LibraryRoot.id == current.target_library_root_id) + ) + ): + raise StoryArcPolicyMigrationError( + "current_destination_root_unavailable", + "The current Story Arc destination root is unavailable", + category="safety", + ) + proposed = await validate_story_arc_placement_policy_input( + session, + proposal, + revision=arc.revision, + ) + if _placement_policy_shape(current) == _placement_policy_shape(proposed): + raise StoryArcPolicyMigrationError( + "managed_policy_migration_not_required", + "This policy change does not alter managed placement destinations", + ) + managed_id = await session.scalar( + select(StoryArcPlacement.id) + .join(IssueStoryArc, StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + StoryArcPlacement.ownership == StoryArcPlacementOwnership.MANAGED, + ) + .order_by(StoryArcPlacement.id) + .limit(1) + ) + if managed_id is None: + raise StoryArcPolicyMigrationError( + "managed_policy_migration_not_required", + "No managed Story Arc placements require migration", + ) + if await session.scalar(_active_operation_statement(story_arc_id)) is not None: + raise StoryArcPolicyMigrationError( + "placement_operation_recovery_pending", + "A Story Arc placement operation or recovery must finish before policy migration", + category="conflict", + ) + if await _active_sync_recovery_exists( + session, + story_arc_id=story_arc_id, + cancellation_requested=cancellation_requested, + ): + raise StoryArcPolicyMigrationError( + "placement_sync_work_pending", + "Story Arc placement synchronization or recovery must finish before migration", + category="conflict", + ) + return current, proposed, arc.name + + async def _assert_boundary_stable( + self, + session: AsyncSession, + *, + story_arc_id: int, + expected_revision: int, + current: StoryArcPlacementPolicy, + cancellation_requested: Callable[[], bool] | None, + ) -> None: + session.expire_all() + arc = await session.get(StoryArc, story_arc_id) + if ( + arc is None + or arc.lifecycle is not StoryArcLifecycle.ACTIVE + or arc.revision != expected_revision + or _policy_from_arc(arc) != current + ): + raise StoryArcPolicyMigrationError( + "migration_scope_changed", + "Story Arc policy changed while the migration preview was generated", + category="conflict", + ) + if await session.scalar(_active_operation_statement(story_arc_id)) is not None: + raise StoryArcPolicyMigrationError( + "placement_operation_recovery_pending", + "A Story Arc placement operation began during migration preview", + category="conflict", + ) + if await _active_sync_recovery_exists( + session, + story_arc_id=story_arc_id, + cancellation_requested=cancellation_requested, + ): + raise StoryArcPolicyMigrationError( + "placement_sync_work_pending", + "Story Arc placement synchronization began during migration preview", + category="conflict", + ) + + def _decode_preview(self, token: str) -> _SignedPreview: + if not token: + raise StoryArcPolicyMigrationError( + "confirmation_required", "A signed migration preview must be confirmed" + ) + try: + raw = self._serializer().loads(token, max_age=_TOKEN_MAX_AGE_SECONDS) + except SignatureExpired as exc: + raise StoryArcPolicyMigrationError( + "migration_preview_expired", + "The migration preview expired; generate a new preview", + ) from exc + except BadSignature as exc: + raise StoryArcPolicyMigrationError( + "invalid_preview_token", "The migration preview token is invalid" + ) from exc + try: + return _signed_preview_from_payload(raw) + except (KeyError, TypeError, ValueError) as exc: + raise StoryArcPolicyMigrationError( + "invalid_preview_token", "The migration preview token is invalid" + ) from exc + + @staticmethod + def _serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(get_application_secret(), salt=_TOKEN_SALT) + + +def _membership_scope_statement( + story_arc_id: int, + *, + after_membership_id: int, + limit: int, +) -> Select[tuple[IssueStoryArc, Issue, Series, Publisher]]: + """Portable, bounded keyset query used by every membership digest pass.""" + return ( + select(IssueStoryArc, Issue, Series, Publisher) + .outerjoin(Issue, IssueStoryArc.issue_id == Issue.id) + .outerjoin(Series, Issue.series_id == Series.id) + .outerjoin(Publisher, Series.publisher_id == Publisher.id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + IssueStoryArc.id > after_membership_id, + ) + .order_by(IssueStoryArc.id) + .limit(limit) + ) + + +def _placement_scope_statement( + story_arc_id: int, + *, + after_placement_id: int, + limit: int, +) -> Select[tuple[StoryArcPlacement]]: + """Portable, bounded keyset query for the exact placement scope.""" + return ( + select(StoryArcPlacement) + .join(IssueStoryArc, StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + StoryArcPlacement.id > after_placement_id, + ) + .order_by(StoryArcPlacement.id) + .limit(limit) + ) + + +def _active_operation_statement(story_arc_id: int) -> Select[tuple[int]]: + """Fail closed on both durable tokens and malformed recovery status rows.""" + return ( + select(StoryArcPlacement.id) + .join(IssueStoryArc, StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + or_( + StoryArcPlacement.operation_token.is_not(None), + StoryArcPlacement.last_result["status"].as_string().in_(_ACTIVE_PLACEMENT_STATUSES), + ), + ) + .order_by(StoryArcPlacement.id) + .limit(1) + ) + + +def _active_sync_work_statement(story_arc_id: int) -> Select[tuple[int]]: + return ( + select(StoryArcSyncWork.id) + .join(IssueStoryArc, StoryArcSyncWork.issue_story_arc_id == IssueStoryArc.id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + or_( + StoryArcSyncWork.state.in_(_ACTIVE_SYNC_STATES), + StoryArcSyncWork.claim_token.is_not(None), + ), + ) + .order_by(StoryArcSyncWork.id) + .limit(1) + ) + + +def _terminal_sync_work_statement( + story_arc_id: int, + *, + after_work_id: int, + limit: int, +) -> Select[tuple[object, ...]]: + """Bound terminal work so Python can validate nested rollback markers exactly.""" + return ( + select( + StoryArcSyncWork.id, + StoryArcSyncWork.last_result, + StoryArcSyncWork.origin_import_job_id, + StoryArcSyncWork.origin_import_action_id, + StoryArcSyncWork.issue_story_arc_id, + StoryArcSyncWork.desired_generation, + ImportJob.id, + ImportJob.status, + ImportJobAction.id, + ImportJobAction.import_job_id, + ImportJobAction.status, + ) + .join(IssueStoryArc, StoryArcSyncWork.issue_story_arc_id == IssueStoryArc.id) + .outerjoin( + ImportJobAction, + StoryArcSyncWork.origin_import_action_id == ImportJobAction.id, + ) + .outerjoin(ImportJob, StoryArcSyncWork.origin_import_job_id == ImportJob.id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + StoryArcSyncWork.state.in_(_TERMINAL_SYNC_STATES), + StoryArcSyncWork.id > after_work_id, + ) + .order_by(StoryArcSyncWork.id) + .limit(limit) + ) + + +async def _active_sync_recovery_exists( + session: AsyncSession, + *, + story_arc_id: int, + cancellation_requested: Callable[[], bool] | None, +) -> bool: + if await session.scalar(_active_sync_work_statement(story_arc_id)) is not None: + return True + after_id = 0 + while True: + _check_cancelled(cancellation_requested) + rows = ( + await session.execute( + _terminal_sync_work_statement( + story_arc_id, + after_work_id=after_id, + limit=_SCAN_PAGE_SIZE, + ) + ) + ).all() + scopes = tuple(_terminal_sync_scope(row) for row in rows) + if any(_terminal_sync_recovery_pending(scope) for scope in scopes): + return True + if len(scopes) < _SCAN_PAGE_SIZE: + return False + after_id = scopes[-1].id + await asyncio.sleep(0) + + +def _terminal_sync_scope(row: Row[tuple[object, ...]]) -> _TerminalSyncWorkScope: + ( + work_id, + last_result, + origin_import_job_id, + origin_import_action_id, + membership_id, + desired_generation, + job_id, + job_status, + action_id, + action_import_job_id, + action_status, + ) = row + return _TerminalSyncWorkScope( + id=cast("int", work_id), + last_result=last_result, + origin_import_job_id=cast("int | None", origin_import_job_id), + origin_import_action_id=cast("int | None", origin_import_action_id), + issue_story_arc_id=cast("int", membership_id), + desired_generation=cast("str", desired_generation), + job_id=cast("int | None", job_id), + job_status=cast("ImportJobStatus | None", job_status), + action_id=cast("int | None", action_id), + action_import_job_id=cast("int | None", action_import_job_id), + action_status=cast("ImportJobActionStatus | None", action_status), + ) + + +def _terminal_sync_recovery_pending(scope: _TerminalSyncWorkScope) -> bool: + result = scope.last_result + if not isinstance(result, dict): + return True + if ( + scope.job_status is ImportJobStatus.ROLLING_BACK + or scope.action_status is ImportJobActionStatus.ROLLBACK_FAILED + or (scope.origin_import_job_id is not None and scope.job_id is None) + or (scope.origin_import_action_id is not None and scope.action_id is None) + or ( + scope.origin_import_action_id is not None + and scope.action_import_job_id != scope.origin_import_job_id + ) + ): + return True + if "rollback" not in result: + return scope.action_status is ImportJobActionStatus.ROLLED_BACK + marker = result["rollback"] + if not isinstance(marker, dict): + return True + status = marker.get("status") + base_keys = { + "schema_version", + "status", + "import_job_id", + "import_action_id", + "sync_work_id", + "membership_id", + "desired_generation", + } + if ( + status not in _SETTLED_ROLLBACK_STATUSES + or scope.action_status is not ImportJobActionStatus.ROLLED_BACK + or scope.action_id is None + or scope.origin_import_action_id != scope.action_id + or scope.origin_import_job_id is None + or marker.get("schema_version") != 1 + or marker.get("import_job_id") != scope.origin_import_job_id + or marker.get("import_action_id") != scope.origin_import_action_id + or marker.get("sync_work_id") != scope.id + or marker.get("membership_id") != scope.issue_story_arc_id + or marker.get("desired_generation") != scope.desired_generation + ): + return True + if status == "cancelled_before_publish": + return set(marker) != base_keys + expected_ownership = ( + StoryArcPlacementOwnership.MANAGED.value + if status == "managed_placement_removed" + else StoryArcPlacementOwnership.REFERENCED.value + ) + placement_id = marker.get("placement_id") + return bool( + set(marker) != base_keys | {"placement_id", "placement_ownership"} + or not _is_positive_int(placement_id) + or marker.get("placement_ownership") != expected_ownership + ) + + +async def _membership_scope_digest( + session: AsyncSession, + *, + story_arc_id: int, + arc_name: str, + cancellation_requested: Callable[[], bool] | None, +) -> str: + digest = hashlib.sha256() + after_id = 0 + while True: + _check_cancelled(cancellation_requested) + page = await _load_membership_scope_page( + session, + story_arc_id=story_arc_id, + arc_name=arc_name, + after_membership_id=after_id, + limit=_SCAN_PAGE_SIZE, + ) + await session.rollback() + for item in page: + _update_digest(digest, item.digest_record()) + if len(page) < _SCAN_PAGE_SIZE: + return digest.hexdigest() + after_id = page[-1].context.membership_id + await asyncio.sleep(0) + + +async def _placement_target_index( + session: AsyncSession, + *, + story_arc_id: int, + arc_name: str, + proposed: StoryArcPlacementPolicy, + cancellation_requested: Callable[[], bool] | None, +) -> tuple[str, dict[str, int], dict[str, int]]: + digest = hashlib.sha256() + exact_counts: dict[str, int] = {} + folded_counts: dict[str, int] = {} + after_id = 0 + while True: + _check_cancelled(cancellation_requested) + page = await _load_placement_scope_page( + session, + story_arc_id=story_arc_id, + arc_name=arc_name, + after_placement_id=after_id, + limit=_SCAN_PAGE_SIZE, + ) + await session.rollback() + for placement in page: + _update_digest(digest, placement.digest_record()) + if ( + placement.ownership is StoryArcPlacementOwnership.MANAGED + and proposed.mode in _MANAGED_POLICY_MODES + ): + target = _rendered_target_path(placement.context.context, proposed) + exact_key, folded_key = _path_keys(target) + exact_counts[exact_key] = exact_counts.get(exact_key, 0) + 1 + folded_counts[folded_key] = folded_counts.get(folded_key, 0) + 1 + if len(page) < _SCAN_PAGE_SIZE: + return digest.hexdigest(), exact_counts, folded_counts + after_id = page[-1].id + await asyncio.sleep(0) + + +async def _classify_placements( + session: AsyncSession, + *, + story_arc_id: int, + arc_name: str, + current: StoryArcPlacementPolicy, + proposed: StoryArcPlacementPolicy, + exact_target_counts: dict[str, int], + folded_target_counts: dict[str, int], + limit: int, + after_placement_id: int, + cancellation_requested: Callable[[], bool] | None, +) -> tuple[ + str, + str, + _PreviewCounts, + tuple[StoryArcPolicyMigrationPreviewItem, ...], + int | None, + bool, +]: + placement_digest = hashlib.sha256() + plan_digest = hashlib.sha256() + counts = _PreviewCounts() + response_items: list[StoryArcPolicyMigrationPreviewItem] = [] + response_has_more = False + cursor = 0 + while True: + _check_cancelled(cancellation_requested) + page = await _load_placement_scope_page( + session, + story_arc_id=story_arc_id, + arc_name=arc_name, + after_placement_id=cursor, + limit=_SCAN_PAGE_SIZE, + ) + candidate_paths = tuple( + dict.fromkeys( + _rendered_target_path(item.context.context, proposed) + for item in page + if item.ownership is StoryArcPlacementOwnership.MANAGED + and proposed.mode in _MANAGED_POLICY_MODES + ) + ) + occupied = await _load_occupied_placements(session, candidate_paths) + canonical_paths = await _load_canonical_destination_paths(session, candidate_paths) + await session.rollback() + classified_page = await asyncio.to_thread( + _classify_page, + page, + current, + proposed, + exact_target_counts, + folded_target_counts, + occupied, + canonical_paths, + cancellation_requested, + ) + for placement, classified in zip(page, classified_page, strict=True): + _update_digest(placement_digest, placement.digest_record()) + _update_digest(plan_digest, classified.digest_record) + counts = counts.add(classified.item) + if placement.id <= after_placement_id: + continue + if len(response_items) < limit: + response_items.append(classified.item) + else: + response_has_more = True + if len(page) < _SCAN_PAGE_SIZE: + break + cursor = page[-1].id + await asyncio.sleep(0) + next_cursor = response_items[-1].placement_id if response_items and response_has_more else None + return ( + placement_digest.hexdigest(), + plan_digest.hexdigest(), + counts, + tuple(response_items), + next_cursor, + response_has_more, + ) + + +async def _load_membership_scope_page( + session: AsyncSession, + *, + story_arc_id: int, + arc_name: str, + after_membership_id: int, + limit: int, +) -> tuple[_MembershipScope, ...]: + rows = ( + await session.execute( + _membership_scope_statement( + story_arc_id, + after_membership_id=after_membership_id, + limit=limit, + ) + ) + ).all() + return await _membership_scopes_from_rows(session, arc_name=arc_name, rows=rows) + + +async def _load_placement_scope_page( + session: AsyncSession, + *, + story_arc_id: int, + arc_name: str, + after_placement_id: int, + limit: int, +) -> tuple[_PlacementScope, ...]: + rows = list( + ( + await session.scalars( + _placement_scope_statement( + story_arc_id, + after_placement_id=after_placement_id, + limit=limit, + ) + ) + ).all() + ) + if not rows: + return () + membership_rows = ( + await session.execute( + select(IssueStoryArc, Issue, Series, Publisher) + .outerjoin(Issue, IssueStoryArc.issue_id == Issue.id) + .outerjoin(Series, Issue.series_id == Series.id) + .outerjoin(Publisher, Series.publisher_id == Publisher.id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + IssueStoryArc.id.in_(tuple(row.issue_story_arc_id for row in rows)), + ) + .order_by(IssueStoryArc.id) + ) + ).all() + contexts = { + item.context.membership_id: item + for item in await _membership_scopes_from_rows( + session, + arc_name=arc_name, + rows=membership_rows, + ) + } + return tuple(_placement_scope(row, contexts[row.issue_story_arc_id]) for row in rows) + + +async def _membership_scopes_from_rows( + session: AsyncSession, + *, + arc_name: str, + rows: Sequence[Row[tuple[IssueStoryArc, Issue, Series, Publisher]]], +) -> tuple[_MembershipScope, ...]: + issue_ids = tuple(issue.id for _membership, issue, _series, _publisher in rows if issue) + files_by_issue: dict[int, LibraryFile] = {} + if issue_ids: + first_ids = ( + select(func.min(LibraryFile.id)) + .where(LibraryFile.issue_id.in_(issue_ids)) + .group_by(LibraryFile.issue_id) + ) + files = list( + ( + await session.scalars( + select(LibraryFile) + .where(LibraryFile.id.in_(first_ids)) + .order_by(LibraryFile.id) + ) + ).all() + ) + for library_file in files: + if library_file.issue_id is not None: + files_by_issue[library_file.issue_id] = library_file + return tuple( + _membership_scope( + arc_name=arc_name, + membership=membership, + issue=issue, + series=series, + publisher=publisher, + library_file=files_by_issue.get(issue.id) if issue is not None else None, + ) + for membership, issue, series, publisher in rows + ) + + +def _membership_scope( + *, + arc_name: str, + membership: IssueStoryArc, + issue: Issue | None, + series: Series | None, + publisher: Publisher | None, + library_file: LibraryFile | None, +) -> _MembershipScope: + issue_number_text = ( + issue.effective_issue_number_text + if issue is not None + else membership.source_issue_number_text or "unknown" + ) + extension = library_file.file_format.value if library_file is not None else "cbz" + context = _PlacementContext( + membership_id=membership.id, + story_arc_id=membership.story_arc_id, + sequence_number=membership.sequence_number, + issue_id=issue.id if issue is not None else None, + issue_number_text=issue_number_text, + story_arc_name=arc_name, + series_name=( + series.title + if series is not None + else membership.source_series_name or "Unknown Series" + ), + publisher_name=publisher.name if publisher is not None else membership.source_publisher, + issue_title=issue.title if issue is not None else membership.source_issue_title, + year=( + issue.release_date.year + if issue is not None and issue.release_date is not None + else series.year_start + if series is not None + else None + ), + series_start_year=series.year_start if series is not None else None, + series_end_year=series.year_end if series is not None else None, + library_file_id=library_file.id if library_file is not None else None, + canonical_path=library_file.file_path if library_file is not None else None, + extension=extension, + ) + return _MembershipScope( + context=context, + source_ordinal=membership.source_ordinal, + resolution_state=membership.resolution_state.value, + sync_eligible=membership.sync_eligible, + membership_updated_at=membership.updated_at, + membership_evidence=dict(membership.evidence or {}), + materialization_result=dict(membership.last_materialization_result or {}), + issue_updated_at=issue.updated_at if issue is not None else None, + series_id=series.id if series is not None else None, + series_updated_at=series.updated_at if series is not None else None, + canonical_id=library_file.id if library_file is not None else None, + canonical_path=library_file.file_path if library_file is not None else None, + canonical_size=library_file.file_size if library_file is not None else None, + canonical_format=library_file.file_format.value if library_file is not None else None, + canonical_hash=library_file.file_hash if library_file is not None else None, + canonical_modified_at=(library_file.file_modified_at if library_file is not None else None), + canonical_updated_at=library_file.updated_at if library_file is not None else None, + canonical_library_root_id=( + library_file.library_root_id if library_file is not None else None + ), + canonical_storage_mode=( + library_file.storage_mode.value if library_file is not None else None + ), + canonical_source_signature=( + dict(library_file.source_signature or {}) if library_file is not None else {} + ), + ) + + +def _placement_scope(row: StoryArcPlacement, context: _MembershipScope) -> _PlacementScope: + last_result = dict(row.last_result or {}) + raw_target = last_result.get("target_fingerprint") + target_fingerprint = dict(raw_target) if isinstance(raw_target, dict) else {} + return _PlacementScope( + id=row.id, + membership_id=row.issue_story_arc_id, + library_file_id=row.library_file_id, + library_root_id=row.library_root_id, + placement_path=row.placement_path, + mode=row.mode, + ownership=row.ownership, + symlink_style=row.symlink_style.value if row.symlink_style is not None else None, + source_kind=row.source_kind.value, + creating_action_id=row.creating_action_id, + rendered_reading_order=row.rendered_reading_order, + policy_schema_version=row.policy_schema_version, + operation_token=row.operation_token, + source_fingerprint=dict(row.source_fingerprint or {}), + target_fingerprint=target_fingerprint, + state=row.state, + last_result=last_result, + updated_at=row.updated_at, + context=context, + ) + + +async def _load_occupied_placements( + session: AsyncSession, + paths: tuple[str, ...], +) -> dict[str, _OccupiedPlacement]: + if not paths: + return {} + rows = ( + await session.execute( + select( + StoryArcPlacement.id, + StoryArcPlacement.placement_path, + StoryArcPlacement.ownership, + ) + .where(StoryArcPlacement.placement_path.in_(paths)) + .order_by(StoryArcPlacement.id) + ) + ).all() + return { + path: _OccupiedPlacement(id=placement_id, ownership=ownership) + for placement_id, path, ownership in rows + } + + +async def _load_canonical_destination_paths( + session: AsyncSession, + paths: tuple[str, ...], +) -> frozenset[str]: + if not paths: + return frozenset() + return frozenset( + ( + await session.scalars( + select(LibraryFile.file_path) + .where(LibraryFile.file_path.in_(paths)) + .order_by(LibraryFile.id) + ) + ).all() + ) + + +def _classify_page( + page: Sequence[_PlacementScope], + current: StoryArcPlacementPolicy, + proposed: StoryArcPlacementPolicy, + exact_target_counts: dict[str, int], + folded_target_counts: dict[str, int], + occupied: dict[str, _OccupiedPlacement], + canonical_paths: frozenset[str], + cancellation_requested: Callable[[], bool] | None, +) -> tuple[_ClassifiedPlacement, ...]: + results: list[_ClassifiedPlacement] = [] + for placement in page: + _check_cancelled(cancellation_requested) + results.append( + _classify_placement( + placement, + current=current, + proposed=proposed, + exact_target_counts=exact_target_counts, + folded_target_counts=folded_target_counts, + occupied=occupied, + canonical_paths=canonical_paths, + ) + ) + return tuple(results) + + +def _classify_placement( + placement: _PlacementScope, + *, + current: StoryArcPlacementPolicy, + proposed: StoryArcPlacementPolicy, + exact_target_counts: dict[str, int], + folded_target_counts: dict[str, int], + occupied: dict[str, _OccupiedPlacement], + canonical_paths: frozenset[str], +) -> _ClassifiedPlacement: + context = placement.context.context + if placement.ownership is StoryArcPlacementOwnership.REFERENCED: + item = StoryArcPolicyMigrationPreviewItem( + placement_id=placement.id, + membership_id=placement.membership_id, + ownership=placement.ownership.value, + action="preserve_referenced", + old_mode=placement.mode.value, + new_mode=placement.mode.value, + old_path=placement.placement_path, + new_path=placement.placement_path, + collision="none", + blocked=False, + reason="Referenced artifact is preserved and is not a migration target", + required_bytes=0, + ) + return _classified(item, placement, old_inspection=None, new_inspection=None) + + desired_path = ( + _rendered_target_path(context, proposed) + if proposed.mode is not StoryArcPlacementPolicyMode.LOGICAL + else None + ) + old_inspection = _inspect_current_managed(placement, current) + if old_inspection.state is not StoryArcPlacementInspectionState.MANAGED_CURRENT: + item = _managed_item( + placement, + proposed=proposed, + desired_path=desired_path, + action=_managed_action(placement, proposed, desired_path), + blocked=True, + collision="none", + reason="Managed placement no longer matches its durable ownership evidence", + ) + return _classified( + item, + placement, + old_inspection=_inspection_digest(old_inspection), + new_inspection=None, + ) + + source_size = _inspection_source_size(old_inspection) + action = _managed_action(placement, proposed, desired_path) + if action == "remove_managed": + item = _managed_item( + placement, + proposed=proposed, + desired_path=desired_path, + action=action, + source_size=source_size, + ) + return _classified( + item, + placement, + old_inspection=_inspection_digest(old_inspection), + new_inspection=None, + ) + if action == "managed_unchanged": + item = _managed_item( + placement, + proposed=proposed, + desired_path=desired_path, + action=action, + source_size=source_size, + ) + return _classified( + item, + placement, + old_inspection=_inspection_digest(old_inspection), + new_inspection=None, + ) + if desired_path is None: # pragma: no cover - action exhaustiveness + raise ValueError("Managed migration target is missing") + + exact_key, folded_key = _path_keys(desired_path) + collision: str | None = None + if exact_target_counts.get(exact_key, 0) > 1: + collision = "duplicate_migration_target" + elif folded_target_counts.get(folded_key, 0) > 1: + collision = "case_only_migration_target" + elif desired_path in canonical_paths: + collision = "canonical_destination" + else: + tracked = occupied.get(desired_path) + if tracked is not None and tracked.id != placement.id: + collision = ( + "referenced_destination_preserved" + if tracked.ownership is StoryArcPlacementOwnership.REFERENCED + else "placement_destination_conflict" + ) + if collision is not None: + item = _managed_item( + placement, + proposed=proposed, + desired_path=desired_path, + action=action, + blocked=True, + collision=collision, + reason="Rendered destination is not exclusively available for this managed placement", + source_size=source_size, + ) + return _classified( + item, + placement, + old_inspection=_inspection_digest(old_inspection), + new_inspection=None, + ) + + if _normal_path(desired_path) == _normal_path(placement.placement_path): + new_inspection = _inspect_same_path_rebuild(placement, proposed, desired_path) + ready = new_inspection.state is StoryArcPlacementInspectionState.FREE + collision_code = ( + "none" + if ready + else new_inspection.collision.value + if new_inspection.collision is not StoryArcCollisionKind.NONE + else new_inspection.code or "inspection_blocked" + ) + item = _managed_item( + placement, + proposed=proposed, + desired_path=desired_path, + action=action, + blocked=not ready, + collision=collision_code, + reason=(None if ready else new_inspection.reason or "Destination is blocked"), + source_size=source_size, + ) + return _classified( + item, + placement, + old_inspection=_inspection_digest(old_inspection), + new_inspection=_inspection_digest(new_inspection), + ) + + new_inspection = inspect_story_arc_placement( + _placement_plan(context, proposed), + ) + ready = new_inspection.state is StoryArcPlacementInspectionState.FREE + collision_code = ( + "none" + if ready + else new_inspection.code or new_inspection.collision.value or "inspection_blocked" + ) + item = _managed_item( + placement, + proposed=proposed, + desired_path=desired_path, + action=action, + blocked=not ready, + collision=collision_code, + reason=(None if ready else new_inspection.reason or "Destination is blocked"), + source_size=source_size, + ) + return _classified( + item, + placement, + old_inspection=_inspection_digest(old_inspection), + new_inspection=_inspection_digest(new_inspection), + ) + + +def _inspect_current_managed( + placement: _PlacementScope, + current: StoryArcPlacementPolicy, +) -> StoryArcPlacementInspection: + rendered = ( + _rendered_target_path(placement.context.context, current) + if current.mode is not StoryArcPlacementPolicyMode.LOGICAL + else None + ) + evidence_valid = ( + current.mode in _MANAGED_POLICY_MODES + and placement.mode.value == current.mode.value + and placement.policy_schema_version == STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION + and placement.rendered_reading_order == placement.context.context.sequence_number + and rendered is not None + and _normal_path(rendered) == _normal_path(placement.placement_path) + and placement.state is StoryArcPlacementState.CURRENT + and placement.operation_token is None + and bool(placement.source_fingerprint) + and bool(placement.target_fingerprint) + and placement.last_result.get("status") in _UNCHANGED_MANAGED_STATUSES + ) + if not evidence_valid: + return _blocked_inspection(placement, "managed_ownership_evidence_changed") + return inspect_story_arc_placement( + _placement_plan(placement.context.context, current), + existing=placement.inspection_evidence(), + ) + + +def _inspect_same_path_rebuild( + placement: _PlacementScope, + proposed: StoryArcPlacementPolicy, + desired_path: str, +) -> StoryArcPlacementInspection: + inspected = inspect_story_arc_placement( + _placement_plan(placement.context.context, proposed), + existing=placement.inspection_evidence(), + ) + if inspected.code != "representation_changed": + return inspected + if proposed.mode is StoryArcPlacementPolicyMode.HARDLINK: + canonical_path = placement.context.context.canonical_path + if canonical_path is None: + return StoryArcPlacementInspection( + state=StoryArcPlacementInspectionState.BLOCKED, + mode=StoryArcPlacementMode.HARDLINK, + target_path=Path(desired_path), + collision=StoryArcCollisionKind.SOURCE_UNAVAILABLE, + code="source_unavailable", + reason="Canonical source is unavailable for hardlink migration", + ) + try: + source_device = _filesystem_device(Path(canonical_path)) + destination_device = _filesystem_device(Path(desired_path).parent) + except OSError: + return StoryArcPlacementInspection( + state=StoryArcPlacementInspectionState.BLOCKED, + mode=StoryArcPlacementMode.HARDLINK, + target_path=Path(desired_path), + collision=StoryArcCollisionKind.NONE, + code="inspection_failed", + reason="Proposed placement mode could not be inspected safely", + ) + if source_device != destination_device: + return StoryArcPlacementInspection( + state=StoryArcPlacementInspectionState.BLOCKED, + mode=StoryArcPlacementMode.HARDLINK, + target_path=Path(desired_path), + collision=StoryArcCollisionKind.CROSS_DEVICE, + code="cross_device", + reason="Hardlink source and destination are on different filesystems", + ) + source_size = _inspection_source_size(inspected) + return StoryArcPlacementInspection( + state=StoryArcPlacementInspectionState.FREE, + mode=_filesystem_mode(proposed.mode), + target_path=Path(desired_path), + collision=StoryArcCollisionKind.NONE, + code="owned_rebuild", + reason="Existing managed artifact is replaceable after ownership revalidation", + required_bytes=( + source_size or 0 if proposed.mode is StoryArcPlacementPolicyMode.COPY else 0 + ), + proposed_ownership=StoryArcPlacementOwnership.MANAGED.value, + source_fingerprint=inspected.source_fingerprint, + target_fingerprint=inspected.target_fingerprint, + ) + + +def _filesystem_device(path: Path) -> int: + return int(os.stat(path, follow_symlinks=False).st_dev) + + +def _inspection_source_size(inspection: StoryArcPlacementInspection) -> int | None: + fingerprint = inspection.source_fingerprint + value = fingerprint.get("size") if fingerprint is not None else None + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value + + +def _blocked_inspection( + placement: _PlacementScope, + code: str, +) -> StoryArcPlacementInspection: + return StoryArcPlacementInspection( + state=StoryArcPlacementInspectionState.BLOCKED, + mode=placement.mode, + target_path=Path(placement.placement_path), + collision=StoryArcCollisionKind.NONE, + code=code, + reason="Managed placement ownership evidence changed", + ) + + +def _managed_action( + placement: _PlacementScope, + proposed: StoryArcPlacementPolicy, + desired_path: str | None, +) -> str: + if proposed.mode not in _MANAGED_POLICY_MODES: + return "remove_managed" + if ( + desired_path is not None + and _normal_path(desired_path) == _normal_path(placement.placement_path) + and placement.mode.value == proposed.mode.value + and placement.symlink_style + == (proposed.symlink_style.value if proposed.symlink_style is not None else None) + ): + return "managed_unchanged" + return "migrate_managed" + + +def _managed_item( + placement: _PlacementScope, + *, + proposed: StoryArcPlacementPolicy, + desired_path: str | None, + action: str, + blocked: bool = False, + collision: str = "none", + reason: str | None = None, + source_size: int | None = None, +) -> StoryArcPolicyMigrationPreviewItem: + required_bytes = ( + source_size or 0 + if action == "migrate_managed" and proposed.mode is StoryArcPlacementPolicyMode.COPY + else 0 + ) + return StoryArcPolicyMigrationPreviewItem( + placement_id=placement.id, + membership_id=placement.membership_id, + ownership=placement.ownership.value, + action=action, + old_mode=placement.mode.value, + new_mode=proposed.mode.value, + old_path=placement.placement_path, + new_path=desired_path, + collision=collision, + blocked=blocked, + reason=reason, + required_bytes=required_bytes, + ) + + +def _classified( + item: StoryArcPolicyMigrationPreviewItem, + placement: _PlacementScope, + *, + old_inspection: dict[str, object] | None, + new_inspection: dict[str, object] | None, +) -> _ClassifiedPlacement: + return _ClassifiedPlacement( + item=item, + digest_record={ + "placement": placement.digest_record(), + "action": item.action, + "old_mode": item.old_mode, + "new_mode": item.new_mode, + "old_path": item.old_path, + "new_path": item.new_path, + "collision": item.collision, + "blocked": item.blocked, + "required_bytes": item.required_bytes, + "old_inspection": old_inspection, + "new_inspection": new_inspection, + }, + ) + + +def _placement_plan( + context: _PlacementContext, + policy: StoryArcPlacementPolicy, +) -> StoryArcPlacementPlan: + return StoryArcPlacementPlan( + issue_story_arc_id=context.membership_id, + library_file_id=context.library_file_id, + canonical_path=Path(context.canonical_path) if context.canonical_path else None, + destination_root=Path(policy.destination_root) if policy.destination_root else None, + values=context.naming_values(), + mode=_filesystem_mode(policy.mode), + symlink_style=policy.symlink_style, + folder_template=policy.folder_template, + file_template=policy.file_template, + ) + + +def _filesystem_mode(mode: StoryArcPlacementPolicyMode) -> StoryArcPlacementMode: + if mode in {StoryArcPlacementPolicyMode.LOGICAL, StoryArcPlacementPolicyMode.REFERENCE_ONLY}: + return StoryArcPlacementMode.REFERENCE_ONLY + return StoryArcPlacementMode(mode.value) + + +def _inspection_digest( + inspection: StoryArcPlacementInspection, +) -> dict[str, object]: + return { + "state": inspection.state.value, + "mode": inspection.mode.value, + "target_path": str(inspection.target_path) if inspection.target_path is not None else None, + "collision": inspection.collision.value, + "code": inspection.code, + "required_bytes": inspection.required_bytes, + "source_fingerprint": dict(inspection.source_fingerprint or {}), + "target_fingerprint": dict(inspection.target_fingerprint or {}), + } + + +async def _validate_policy_root( + policy: StoryArcPlacementPolicy, + *, + role: str, +) -> _PolicyRootFingerprint: + if policy.mode is StoryArcPlacementPolicyMode.LOGICAL: + return _PolicyRootFingerprint(resolved_path=None, device=None, inode=None) + if policy.destination_root is None: + raise StoryArcPolicyMigrationError( + f"{role}_destination_root_unavailable", + f"The {role} Story Arc destination root is unavailable", + category="safety", + ) + + def validate() -> _PolicyRootFingerprint: + root = Path(policy.destination_root or "") + if not root.is_absolute(): + raise OSError + before = os.stat(root, follow_symlinks=False) + if not stat.S_ISDIR(before.st_mode): + raise OSError + resolved = root.resolve(strict=True) + if resolved != root or not resolved.is_dir(): + raise OSError + if not os.access(resolved, os.R_OK | os.W_OK | os.X_OK): + raise OSError + metadata = os.stat(root, follow_symlinks=False) + if ( + not stat.S_ISDIR(metadata.st_mode) + or metadata.st_dev != before.st_dev + or metadata.st_ino != before.st_ino + ): + raise OSError + return _PolicyRootFingerprint( + resolved_path=str(resolved), + device=int(metadata.st_dev), + inode=int(metadata.st_ino), + ) + + try: + return await asyncio.to_thread(validate) + except OSError as exc: + raise StoryArcPolicyMigrationError( + f"{role}_destination_root_unavailable", + f"The {role} Story Arc destination root is unavailable or unsafe", + category="safety", + ) from exc + + +def _policy_token_record(policy: StoryArcPlacementPolicy) -> dict[str, object]: + return { + "configured": policy.configured, + "revision": policy.revision, + **policy.snapshot, + } + + +def _preview_counts_record(preview: StoryArcPolicyMigrationPreview) -> dict[str, int]: + return { + "total": preview.total_placement_count, + "managed_migrate": preview.managed_migrate_count, + "managed_remove": preview.managed_remove_count, + "managed_unchanged": preview.managed_unchanged_count, + "referenced_preserved": preview.referenced_preserved_count, + "collisions": preview.collision_count, + "blocked": preview.blocked_count, + "required_bytes": preview.required_bytes, + } + + +def _signed_preview_payload(preview: _SignedPreview) -> dict[str, object]: + return { + "schema_version": _TOKEN_SCHEMA_VERSION, + "actor_id": preview.actor_id, + "story_arc_id": preview.story_arc_id, + "expected_revision": preview.expected_revision, + "current_policy": preview.current_policy, + "proposed_policy": preview.proposed_policy, + "scope_digest": preview.scope_digest, + "counts": preview.counts, + } + + +def _signed_preview_from_payload(raw: object) -> _SignedPreview: + expected_keys = { + "schema_version", + "actor_id", + "story_arc_id", + "expected_revision", + "current_policy", + "proposed_policy", + "scope_digest", + "counts", + } + if ( + not isinstance(raw, dict) + or set(raw) != expected_keys + or raw.get("schema_version") != _TOKEN_SCHEMA_VERSION + ): + raise ValueError("invalid preview payload") + current = _validated_policy_record(raw["current_policy"]) + proposed = _validated_policy_record(raw["proposed_policy"]) + counts = _validated_counts_record(raw["counts"]) + return _SignedPreview( + actor_id=_positive_int(raw["actor_id"]), + story_arc_id=_positive_int(raw["story_arc_id"]), + expected_revision=_positive_int(raw["expected_revision"]), + current_policy=current, + proposed_policy=proposed, + scope_digest=_fixed_hex(raw["scope_digest"], length=64), + counts=counts, + ) + + +def _validated_policy_record(raw: object) -> dict[str, object]: + expected_keys = { + "configured", + "revision", + "schema_version", + "mode", + "target_library_root_id", + "destination_root", + "folder_template", + "file_template", + "symlink_style", + "synchronize", + } + if not isinstance(raw, dict) or set(raw) != expected_keys: + raise ValueError("invalid policy") + if raw["configured"] is not True or raw["schema_version"] != 1: + raise ValueError("invalid policy") + _positive_int(raw["revision"]) + StoryArcPlacementPolicyMode(_string(raw["mode"])) + root_id = raw["target_library_root_id"] + if root_id is not None: + _positive_int(root_id) + destination = raw["destination_root"] + if destination is not None: + _string(destination) + _string(raw["folder_template"]) + _string(raw["file_template"]) + style = raw["symlink_style"] + if style is not None: + _string(style) + if not isinstance(raw["synchronize"], bool): + raise ValueError("invalid policy") + return dict(raw) + + +def _validated_counts_record(raw: object) -> dict[str, int]: + keys = { + "total", + "managed_migrate", + "managed_remove", + "managed_unchanged", + "referenced_preserved", + "collisions", + "blocked", + "required_bytes", + } + if not isinstance(raw, dict) or set(raw) != keys: + raise ValueError("invalid counts") + return {key: _nonnegative_int(raw[key]) for key in keys} + + +def _bounded_page(limit: int, after_placement_id: int) -> tuple[int, int]: + if ( + isinstance(limit, bool) + or not isinstance(limit, int) + or not 1 <= limit <= _MAX_RESPONSE_PAGE_SIZE + ): + raise StoryArcPolicyMigrationError( + "invalid_page_limit", f"Migration preview limit must be 1-{_MAX_RESPONSE_PAGE_SIZE}" + ) + if ( + isinstance(after_placement_id, bool) + or not isinstance(after_placement_id, int) + or after_placement_id < 0 + ): + raise StoryArcPolicyMigrationError( + "invalid_page_cursor", "Migration preview cursor must be a non-negative integer" + ) + return limit, after_placement_id + + +def _check_cancelled(cancellation_requested: Callable[[], bool] | None) -> None: + if cancellation_requested is not None and cancellation_requested(): + raise StoryArcPolicyMigrationError( + "migration_preview_cancelled", + "Story Arc policy migration preview was cancelled", + category="cancelled", + ) + + +def _path_keys(path: str) -> tuple[str, str]: + exact = os.path.abspath(os.path.normpath(path)) + return hashlib.sha256(exact.encode()).hexdigest(), hashlib.sha256( + exact.casefold().encode() + ).hexdigest() + + +def _normal_path(path: str | Path) -> str: + return os.path.normcase(os.path.abspath(path)) + + +def _update_digest(digest: _DigestWriter, value: object) -> None: + digest.update(_canonical_json(value)) + digest.update(b"\n") + + +def _digest_record(value: object) -> str: + return hashlib.sha256(_canonical_json(value)).hexdigest() + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + default=_json_default, + ).encode() + + +def _json_default(value: object) -> str: + if isinstance(value, (date, datetime)): + return value.isoformat() + return str(value) + + +def _is_positive_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +def _positive_int(value: object) -> int: + if not _is_positive_int(value): + raise ValueError("positive integer required") + return cast("int", value) + + +def _nonnegative_int(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError("non-negative integer required") + return value + + +def _string(value: object) -> str: + if not isinstance(value, str): + raise ValueError("string required") + return value + + +def _fixed_hex(value: object, *, length: int) -> str: + result = _string(value) + if len(result) != length or any(character not in "0123456789abcdef" for character in result): + raise ValueError("fixed hexadecimal string required") + return result + + +__all__ = [ + "STORY_ARC_POLICY_MIGRATION_CONFIRMATION", + "StoryArcPolicyMigrationConfirmation", + "StoryArcPolicyMigrationError", + "StoryArcPolicyMigrationPreview", + "StoryArcPolicyMigrationPreviewItem", + "StoryArcPolicyMigrationService", +] diff --git a/src/pullbox/services/story_arc_search_targets.py b/src/pullbox/services/story_arc_search_targets.py new file mode 100644 index 00000000..1058f763 --- /dev/null +++ b/src/pullbox/services/story_arc_search_targets.py @@ -0,0 +1,122 @@ +"""Bounded canonical targets for an explicitly requested story-arc search.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import exists, func, select + +from pullbox.models.issue import Issue +from pullbox.models.series import Series +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcResolutionState, +) +from pullbox.services.search_targets import ( + IssueSearchTarget, + _target_from_row, + arc_issue_acquisition_filter, +) + +if TYPE_CHECKING: + from collections.abc import Sequence + + from sqlalchemy.ext.asyncio import AsyncSession + + +async def story_arc_search_ceiling(session: AsyncSession, story_arc_id: int) -> int: + """Freeze a finite issue-ID ceiling without materializing the arc catalog.""" + result = await session.scalar( + select(func.max(IssueStoryArc.issue_id)).where(IssueStoryArc.story_arc_id == story_arc_id) + ) + return int(result or 0) + + +async def load_story_arc_search_eligible_counts( + session: AsyncSession, + story_arc_ids: Sequence[int], +) -> dict[int, int]: + """Count currently searchable missing issues for bounded registry rows.""" + if not story_arc_ids: + return {} + if len(story_arc_ids) > 100: + raise ValueError("Story Arc search eligibility is limited to 100 visible arcs") + result = await session.execute( + select(IssueStoryArc.story_arc_id, func.count(func.distinct(Issue.id))) + .join(StoryArc, StoryArc.id == IssueStoryArc.story_arc_id) + .join(Issue, Issue.id == IssueStoryArc.issue_id) + .where( + IssueStoryArc.story_arc_id.in_(story_arc_ids), + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + StoryArc.lifecycle == StoryArcLifecycle.ACTIVE, + arc_issue_acquisition_filter(), + ) + .group_by(IssueStoryArc.story_arc_id) + ) + return {int(story_arc_id): int(count) for story_arc_id, count in result.all()} + + +async def load_story_arc_missing_search_targets( + session: AsyncSession, + story_arc_id: int, + *, + issue_ids: Sequence[int] | None = None, + series_id: int | None = None, + after_issue_id: int = 0, + ceiling_issue_id: int | None = None, + limit: int = 100, +) -> list[IssueSearchTarget]: + """Search only resolved, missing arc members while preserving explicit skips. + + Manual arc searches need not enable whole-series or arc monitoring. They + still respect publication dates, existing files, active downloads, + unresolved memberships, and pending intervention decisions. + """ + if not 1 <= limit <= 100 or after_issue_id < 0: + raise ValueError("Story Arc search pages require a limit of 1-100 and a valid cursor") + if issue_ids is not None and (not issue_ids or len(issue_ids) > 100): + if not issue_ids: + return [] + raise ValueError("Story Arc search ID batches must not exceed 100") + membership = exists().where( + IssueStoryArc.story_arc_id == story_arc_id, + IssueStoryArc.issue_id == Issue.id, + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + StoryArc.id == IssueStoryArc.story_arc_id, + StoryArc.lifecycle == StoryArcLifecycle.ACTIVE, + ) + statement = ( + select( + Issue.id.label("issue_id"), + Issue.series_id.label("series_id"), + Issue.issue_number.label("issue_number"), + Issue.issue_number_text.label("issue_number_text"), + Issue.issue_type.label("issue_type"), + Issue.title.label("issue_title"), + Issue.release_date.label("release_date"), + Issue.store_date.label("store_date"), + Series.title.label("series_title"), + Series.year_start.label("series_year"), + Series.alternate_names.label("alternate_names"), + Series.issue_count.label("series_issue_count"), + Series.status.label("series_status"), + Series.status_override.label("status_override"), + ) + .join(Series, Series.id == Issue.series_id) + .where( + Issue.id > after_issue_id, + arc_issue_acquisition_filter(), + membership, + ) + .order_by(Issue.id) + .limit(limit) + ) + if issue_ids is not None: + statement = statement.where(Issue.id.in_(issue_ids)) + if series_id is not None: + statement = statement.where(Issue.series_id == series_id) + if ceiling_issue_id is not None: + statement = statement.where(Issue.id <= ceiling_issue_id) + return [_target_from_row(row) for row in (await session.execute(statement)).all()] diff --git a/src/pullbox/services/story_arc_service.py b/src/pullbox/services/story_arc_service.py new file mode 100644 index 00000000..7831e260 --- /dev/null +++ b/src/pullbox/services/story_arc_service.py @@ -0,0 +1,640 @@ +"""Provider-free logical story-arc lifecycle and membership management.""" + +from __future__ import annotations + +import enum +from typing import TYPE_CHECKING + +from sqlalchemy import select +from sqlalchemy import update as sa_update +from sqlalchemy.orm import selectinload + +from pullbox.core.issue_numbers import normalize_issue_number_text +from pullbox.core.story_arc_identity import normalize_story_arc_name +from pullbox.models.issue import Issue +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcPlacement, + StoryArcPlacementOwnership, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.services.story_arc_membership_policy import ( + order_review_filter, + provider_issue_identity, + requires_order_review, +) + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +class StoryArcServiceError(Exception): + """Base error for logical story-arc operations.""" + + +class StoryArcNotFoundError(StoryArcServiceError): + """Raised when an arc, membership, or canonical issue does not exist.""" + + +class StoryArcValidationError(StoryArcServiceError): + """Raised when a requested story-arc mutation is invalid.""" + + +class StoryArcProviderIdentityError(StoryArcValidationError): + """A canonical issue cannot replace a different exact provider identity.""" + + +class StoryArcConflictError(StoryArcServiceError): + """Raised for duplicate identities and optimistic revision conflicts.""" + + +class DuplicateStoryArcMembershipError(StoryArcConflictError): + """Raised when one arc would contain the same canonical issue twice.""" + + +class _Unset(enum.Enum): + VALUE = enum.auto() + + +_UNSET = _Unset.VALUE + + +def _display_name_and_key(name: str) -> tuple[str, str]: + display_name = " ".join(name.split()) + if not display_name: + raise StoryArcValidationError("Story-arc name must not be blank") + if len(display_name) > 500: + raise StoryArcValidationError("Story-arc name exceeds 500 characters") + try: + normalized_name = normalize_story_arc_name(display_name) + except ValueError as exc: + raise StoryArcValidationError(str(exc)) from exc + return display_name, normalized_name + + +def _validate_order_value(value: int, *, label: str) -> int: + if isinstance(value, bool) or value < 0: + raise StoryArcValidationError(f"{label} must be a non-negative integer") + return value + + +def _normalize_source_issue_number(value: str | float | int) -> str: + try: + return normalize_issue_number_text(value) + except ValueError as exc: + raise StoryArcValidationError(str(exc)) from exc + + +class StoryArcService: + """Manage logical arcs without providers or filesystem materialization. + + The caller owns the transaction. Methods flush generated identifiers and + constraint checks but never commit. Every membership mutation advances the + parent arc revision so API and UI adapters can use one optimistic token. + """ + + async def create( + self, + session: AsyncSession, + *, + name: str, + description: str | None = None, + monitored: bool = False, + search_missing: bool = False, + include_upcoming: bool = False, + sync_enabled: bool = False, + source_kind: StoryArcSourceKind = StoryArcSourceKind.PULLBOX, + ) -> StoryArc: + """Create an empty logical story arc after exact identity checks.""" + display_name, normalized_name = _display_name_and_key(name) + arc = StoryArc( + name=display_name, + normalized_name=normalized_name, + description=description, + source_kind=source_kind, + lifecycle=StoryArcLifecycle.ACTIVE, + monitored=monitored, + search_missing=search_missing, + include_upcoming=include_upcoming, + sync_enabled=sync_enabled, + revision=1, + ) + session.add(arc) + await session.flush() + return arc + + async def update( + self, + session: AsyncSession, + story_arc_id: int, + *, + expected_revision: int, + name: str | _Unset = _UNSET, + description: str | _Unset | None = _UNSET, + monitored: bool | _Unset = _UNSET, + search_missing: bool | _Unset = _UNSET, + include_upcoming: bool | _Unset = _UNSET, + sync_enabled: bool | _Unset = _UNSET, + ) -> StoryArc: + """Patch arc metadata and monitoring flags with revision protection.""" + arc = await self._get_arc(session, story_arc_id) + self._assert_revision(arc, expected_revision) + + if isinstance(name, str): + display_name, normalized_name = _display_name_and_key(name) + if ( + display_name != arc.name or normalized_name != arc.normalized_name + ) and await self._has_managed_placements(session, story_arc_id=arc.id): + raise StoryArcValidationError( + "Story-arc name cannot change while a managed placement exists" + ) + arc.name = display_name + arc.normalized_name = normalized_name + + if description is None or isinstance(description, str): + arc.description = description + + requested_flags = { + "monitored": monitored, + "search_missing": search_missing, + "include_upcoming": include_upcoming, + "sync_enabled": sync_enabled, + } + if arc.lifecycle == StoryArcLifecycle.ARCHIVED and any( + value is True for value in requested_flags.values() + ): + raise StoryArcValidationError("Archived story arcs cannot enable monitoring or sync") + for attribute, value in requested_flags.items(): + if isinstance(value, bool): + setattr(arc, attribute, value) + + if isinstance(sync_enabled, bool): + await self._set_membership_sync_eligibility( + session, + story_arc_id=arc.id, + enabled=sync_enabled, + ) + + arc.revision += 1 + await session.flush() + return arc + + async def archive( + self, + session: AsyncSession, + story_arc_id: int, + *, + expected_revision: int, + ) -> StoryArc: + """Soft-archive an arc and disable all automated behavior.""" + arc = await self._get_arc(session, story_arc_id) + self._assert_revision(arc, expected_revision) + if arc.lifecycle == StoryArcLifecycle.ARCHIVED: + return arc + arc.lifecycle = StoryArcLifecycle.ARCHIVED + arc.monitored = False + arc.search_missing = False + arc.include_upcoming = False + arc.sync_enabled = False + await self._set_membership_sync_eligibility( + session, + story_arc_id=arc.id, + enabled=False, + ) + arc.revision += 1 + await session.flush() + return arc + + async def list_memberships( + self, + session: AsyncSession, + story_arc_id: int, + ) -> list[IssueStoryArc]: + """Return every membership in stable reading order.""" + await self._get_arc(session, story_arc_id) + result = await session.scalars( + select(IssueStoryArc) + .where(IssueStoryArc.story_arc_id == story_arc_id) + .options(selectinload(IssueStoryArc.issue)) + .order_by( + IssueStoryArc.sequence_number.asc(), + IssueStoryArc.source_ordinal.asc(), + IssueStoryArc.id.asc(), + ) + ) + return list(result.all()) + + async def add_membership( + self, + session: AsyncSession, + story_arc_id: int, + *, + issue_id: int | None, + sequence_number: int, + source_issue_number_text: str | float | int | None = None, + source_ordinal: int = 0, + source_kind: StoryArcSourceKind = StoryArcSourceKind.PULLBOX, + ) -> IssueStoryArc: + """Add a resolved or reviewable missing entry without creating an issue.""" + arc = await self._get_active_arc(session, story_arc_id) + sequence_number = _validate_order_value(sequence_number, label="sequence number") + source_ordinal = _validate_order_value(source_ordinal, label="source ordinal") + + issue: Issue | None = None + if issue_id is not None: + issue = await session.get(Issue, issue_id) + if issue is None: + raise StoryArcNotFoundError(f"Issue {issue_id} was not found") + + exact_number = ( + _normalize_source_issue_number(source_issue_number_text) + if source_issue_number_text is not None + else issue.effective_issue_number_text + if issue is not None + else None + ) + if exact_number is None: + raise StoryArcValidationError( + "An unresolved story-arc entry requires an exact source issue number" + ) + + existing = await self._find_idempotent_or_duplicate_membership( + session, + story_arc_id=story_arc_id, + issue_id=issue_id, + sequence_number=sequence_number, + source_ordinal=source_ordinal, + source_issue_number_text=exact_number, + source_kind=source_kind, + ) + if existing is not None: + return existing + + membership = IssueStoryArc( + story_arc_id=story_arc_id, + issue_id=issue_id, + sequence_number=sequence_number, + source_ordinal=source_ordinal, + resolution_state=( + StoryArcResolutionState.RESOLVED + if issue_id is not None + else StoryArcResolutionState.MISSING + ), + source_kind=source_kind, + source_issue_number_text=exact_number, + sync_eligible=issue_id is not None and bool(arc.sync_enabled), + ) + session.add(membership) + arc.revision += 1 + await session.flush() + return membership + + async def update_membership( + self, + session: AsyncSession, + membership_id: int, + *, + sequence_number: int | None = None, + source_ordinal: int | None = None, + source_issue_number_text: str | float | int | None = None, + intentionally_skipped: bool | None = None, + ) -> IssueStoryArc: + """Update order or review state while preserving canonical ownership.""" + membership = await self._get_membership(session, membership_id) + arc = await self._get_active_arc(session, membership.story_arc_id) + if any( + value is not None + for value in ( + sequence_number, + source_ordinal, + source_issue_number_text, + intentionally_skipped, + ) + ) and await self._has_managed_placements(session, membership_id=membership.id): + raise StoryArcValidationError( + "Story-arc membership cannot change while a managed placement exists" + ) + changed = False + + if sequence_number is not None: + sequence_number = _validate_order_value(sequence_number, label="sequence number") + if membership.sequence_number != sequence_number: + membership.sequence_number = sequence_number + changed = True + if source_ordinal is not None: + source_ordinal = _validate_order_value(source_ordinal, label="source ordinal") + if membership.source_ordinal != source_ordinal: + membership.source_ordinal = source_ordinal + changed = True + if source_issue_number_text is not None: + exact_number = _normalize_source_issue_number(source_issue_number_text) + if membership.source_issue_number_text != exact_number: + membership.source_issue_number_text = exact_number + changed = True + if intentionally_skipped is not None: + next_state = ( + StoryArcResolutionState.SKIPPED + if intentionally_skipped + else StoryArcResolutionState.RESOLVED + if membership.issue_id is not None + else StoryArcResolutionState.MISSING + ) + if membership.resolution_state != next_state: + membership.resolution_state = next_state + membership.sync_eligible = bool( + not intentionally_skipped + and membership.issue_id is not None + and arc.sync_enabled + and not requires_order_review(membership) + ) + changed = True + + if changed: + arc.revision += 1 + await session.flush() + return membership + + async def resolve_membership( + self, + session: AsyncSession, + membership_id: int, + *, + issue_id: int, + ) -> IssueStoryArc: + """Resolve or replace an entry with an existing canonical issue.""" + membership = await self._get_membership(session, membership_id) + arc = await self._get_active_arc(session, membership.story_arc_id) + issue = await session.get(Issue, issue_id) + if issue is None: + raise StoryArcNotFoundError(f"Issue {issue_id} was not found") + + provider_id = provider_issue_identity(membership) + if provider_id is not None and str(issue.comicvine_id) != provider_id: + raise StoryArcProviderIdentityError( + "The selected issue does not match this member's exact provider identity" + ) + + duplicate_id = await session.scalar( + select(IssueStoryArc.id).where( + IssueStoryArc.story_arc_id == membership.story_arc_id, + IssueStoryArc.issue_id == issue_id, + IssueStoryArc.id != membership.id, + ) + ) + if duplicate_id is not None: + raise DuplicateStoryArcMembershipError( + f"Issue {issue_id} already belongs to story arc {membership.story_arc_id}" + ) + if ( + membership.issue_id == issue_id + and membership.resolution_state == StoryArcResolutionState.RESOLVED + and not requires_order_review(membership) + ): + return membership + if await self._has_managed_placements(session, membership_id=membership.id): + raise StoryArcValidationError( + "Story-arc membership cannot resolve differently while a managed placement exists" + ) + + membership.issue_id = issue_id + membership.resolution_state = StoryArcResolutionState.RESOLVED + membership.sync_eligible = bool(arc.sync_enabled) + if requires_order_review(membership): + membership.evidence = {**membership.evidence, "catalog_review_required": False} + if membership.source_issue_number_text is None: + membership.source_issue_number_text = issue.effective_issue_number_text + arc.revision += 1 + await session.flush() + return membership + + async def detach_deleted_issue( + self, + session: AsyncSession, + issue_id: int, + ) -> int: + """Retain memberships as missing entries before deleting an issue.""" + issue = await session.get(Issue, issue_id) + memberships = list( + ( + await session.scalars( + select(IssueStoryArc).where(IssueStoryArc.issue_id == issue_id) + ) + ).all() + ) + if not memberships: + return 0 + if await self._has_managed_placements(session, issue_id=issue_id): + raise StoryArcValidationError( + "Canonical issue cannot detach while a managed story-arc placement exists" + ) + + affected_arc_ids: set[int] = set() + for membership in memberships: + membership.issue_id = None + if membership.resolution_state != StoryArcResolutionState.SKIPPED: + membership.resolution_state = StoryArcResolutionState.MISSING + if membership.source_issue_number_text is None and issue is not None: + membership.source_issue_number_text = issue.effective_issue_number_text + affected_arc_ids.add(membership.story_arc_id) + await self._advance_arc_revisions(session, affected_arc_ids) + await session.flush() + return len(memberships) + + async def reconcile_missing_issue_references( + self, + session: AsyncSession, + *, + story_arc_id: int | None = None, + ) -> int: + """Repair resolved states whose canonical issue was deleted via FK cleanup.""" + statement = select(IssueStoryArc).where( + IssueStoryArc.issue_id.is_(None), + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + ) + if story_arc_id is not None: + statement = statement.where(IssueStoryArc.story_arc_id == story_arc_id) + memberships = list((await session.scalars(statement)).all()) + if not memberships: + return 0 + + affected_arc_ids = {membership.story_arc_id for membership in memberships} + for membership in memberships: + membership.resolution_state = StoryArcResolutionState.MISSING + await self._advance_arc_revisions(session, affected_arc_ids) + await session.flush() + return len(memberships) + + async def reorder_memberships( + self, + session: AsyncSession, + story_arc_id: int, + *, + ordered_membership_ids: list[int], + expected_revision: int, + ) -> list[IssueStoryArc]: + """Apply one complete, duplicate-free order with optimistic locking.""" + arc = await self._get_active_arc(session, story_arc_id) + self._assert_revision(arc, expected_revision) + if await self._has_managed_placements(session, story_arc_id=story_arc_id): + raise StoryArcValidationError( + "Story-arc memberships cannot reorder while a managed placement exists" + ) + memberships = await self.list_memberships(session, story_arc_id) + existing_ids = {membership.id for membership in memberships} + requested_ids = set(ordered_membership_ids) + if len(requested_ids) != len(ordered_membership_ids) or requested_ids != existing_ids: + raise StoryArcValidationError( + "Membership reorder must include every arc membership exactly once" + ) + + by_id = {membership.id: membership for membership in memberships} + reordered = [by_id[membership_id] for membership_id in ordered_membership_ids] + for position, membership in enumerate(reordered, start=1): + membership.sequence_number = position + arc.revision += 1 + await session.flush() + return reordered + + async def remove_membership( + self, + session: AsyncSession, + membership_id: int, + ) -> None: + """Remove one association without touching its canonical issue.""" + membership = await self._get_membership(session, membership_id) + arc = await self._get_active_arc(session, membership.story_arc_id) + if await self._has_managed_placements(session, membership_id=membership.id): + raise StoryArcValidationError( + "Story-arc membership cannot be removed while a managed placement exists" + ) + await session.delete(membership) + arc.revision += 1 + await session.flush() + + async def _get_arc(self, session: AsyncSession, story_arc_id: int) -> StoryArc: + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise StoryArcNotFoundError(f"Story arc {story_arc_id} was not found") + return arc + + async def _get_active_arc(self, session: AsyncSession, story_arc_id: int) -> StoryArc: + arc = await self._get_arc(session, story_arc_id) + if arc.lifecycle == StoryArcLifecycle.ARCHIVED: + raise StoryArcValidationError("Archived story arcs cannot change memberships") + return arc + + async def _get_membership( + self, + session: AsyncSession, + membership_id: int, + ) -> IssueStoryArc: + membership = await session.get(IssueStoryArc, membership_id) + if membership is None: + raise StoryArcNotFoundError(f"Story-arc membership {membership_id} was not found") + return membership + + async def _find_idempotent_or_duplicate_membership( + self, + session: AsyncSession, + *, + story_arc_id: int, + issue_id: int | None, + sequence_number: int, + source_ordinal: int, + source_issue_number_text: str, + source_kind: StoryArcSourceKind, + ) -> IssueStoryArc | None: + if issue_id is not None: + existing = await session.scalar( + select(IssueStoryArc).where( + IssueStoryArc.story_arc_id == story_arc_id, + IssueStoryArc.issue_id == issue_id, + ) + ) + if existing is None: + return None + if ( + existing.sequence_number == sequence_number + and existing.source_ordinal == source_ordinal + and existing.source_issue_number_text == source_issue_number_text + and existing.source_kind == source_kind + and existing.resolution_state == StoryArcResolutionState.RESOLVED + ): + return existing + raise DuplicateStoryArcMembershipError( + f"Issue {issue_id} already belongs to story arc {story_arc_id}" + ) + + unresolved_existing: IssueStoryArc | None = await session.scalar( + select(IssueStoryArc).where( + IssueStoryArc.story_arc_id == story_arc_id, + IssueStoryArc.issue_id.is_(None), + IssueStoryArc.sequence_number == sequence_number, + IssueStoryArc.source_ordinal == source_ordinal, + IssueStoryArc.source_issue_number_text == source_issue_number_text, + IssueStoryArc.source_kind == source_kind, + ) + ) + return unresolved_existing + + async def _advance_arc_revisions( + self, + session: AsyncSession, + story_arc_ids: set[int], + ) -> None: + if not story_arc_ids: + return + arcs = list( + (await session.scalars(select(StoryArc).where(StoryArc.id.in_(story_arc_ids)))).all() + ) + for arc in arcs: + arc.revision += 1 + + @staticmethod + async def _has_managed_placements( + session: AsyncSession, + *, + story_arc_id: int | None = None, + membership_id: int | None = None, + issue_id: int | None = None, + ) -> bool: + statement = ( + select(StoryArcPlacement.id) + .join(IssueStoryArc, StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id) + .where(StoryArcPlacement.ownership == StoryArcPlacementOwnership.MANAGED) + ) + if story_arc_id is not None: + statement = statement.where(IssueStoryArc.story_arc_id == story_arc_id) + if membership_id is not None: + statement = statement.where(IssueStoryArc.id == membership_id) + if issue_id is not None: + statement = statement.where(IssueStoryArc.issue_id == issue_id) + return await session.scalar(statement.limit(1)) is not None + + @staticmethod + async def _set_membership_sync_eligibility( + session: AsyncSession, + *, + story_arc_id: int, + enabled: bool, + ) -> None: + statement = sa_update(IssueStoryArc).where( + IssueStoryArc.story_arc_id == story_arc_id, + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + ) + # Constant values keep already-loaded ORM members coherent on both DBs. + await session.execute(statement.values(sync_eligible=False)) + if enabled: + await session.execute( + statement.where(~order_review_filter()).values(sync_eligible=True) + ) + + @staticmethod + def _assert_revision(arc: StoryArc, expected_revision: int) -> None: + if arc.revision != expected_revision: + raise StoryArcConflictError( + f"Story arc revision changed: expected {expected_revision}, current {arc.revision}" + ) diff --git a/src/pullbox/services/story_arc_sync_queue.py b/src/pullbox/services/story_arc_sync_queue.py new file mode 100644 index 00000000..eeefce69 --- /dev/null +++ b/src/pullbox/services/story_arc_sync_queue.py @@ -0,0 +1,2623 @@ +"""Durable outbox and bounded worker for automatic story-arc synchronization.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import math +import secrets +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any, cast + +import structlog +from sqlalchemy import and_, delete, exists, func, insert, or_, select, tuple_, update + +from pullbox.core.exceptions import NotFoundError, ValidationError +from pullbox.database import get_session_factory +from pullbox.models.import_job import ( + ImportControlRequest, + ImportJob, + ImportJobAction, + ImportJobActionStatus, + ImportJobStatus, +) +from pullbox.models.library import LibraryFile +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcPlacement, + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcPlacementState, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.models.story_arc_sync import ( + StoryArcSyncReason, + StoryArcSyncWork, + StoryArcSyncWorkState, +) +from pullbox.services.import_job_actions import ImportJobActionSpec +from pullbox.services.import_story_arc_placement_completion import ( + ImportStoryArcPlacementCompletionState, + finalize_import_story_arc_placements, +) +from pullbox.services.story_arc_membership_policy import requires_order_review +from pullbox.services.story_arc_placement_integration import ( + STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + StoryArcPlacementImportProvenance, + StoryArcPlacementIntegrationError, + StoryArcPlacementSyncService, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + from sqlalchemy.engine import CursorResult + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from pullbox.services.import_job_execution_types import RecordActionFunc, RecordActionsFunc + +logger = structlog.get_logger(__name__) + +STORY_ARC_SYNC_TASK_ID = "sync_story_arc_placements" +MAX_STORY_ARC_SYNC_BATCH_SIZE = 100 +MAX_STORY_ARC_SYNC_ENQUEUE_MEMBERSHIPS = 200 +MAX_IMPORT_STORY_ARC_SYNC_ENQUEUE_BATCH_SIZE = 200 +DEFAULT_STORY_ARC_SYNC_BATCH_SIZE = 50 +DEFAULT_STORY_ARC_DISCOVERY_LIMIT = 200 +MAX_IMPORT_PLACEMENT_FINALIZE_BATCH_SIZE = 100 +_CLAIM_LEASE = timedelta(minutes=15) +_CLAIM_HEARTBEAT_INTERVAL_SECONDS = 60.0 +_ORIGIN_CANCELLATION_POLL_SECONDS = 0.25 +_MAX_ATTEMPTS = 5 +_RETRY_DELAYS = ( + timedelta(seconds=30), + timedelta(minutes=2), + timedelta(minutes=10), + timedelta(minutes=30), + timedelta(hours=1), +) +_POLICY_KEYS = frozenset( + { + "schema_version", + "mode", + "target_library_root_id", + "destination_root", + "folder_template", + "file_template", + "symlink_style", + "synchronize", + } +) +_IMPORT_PLACEMENT_ACTION_TYPE = "story_arc_managed_placement_requested" +_IMPORT_BUILD_PHASE = "story_arcs" +_IMPORT_PLACEMENT_PHASE = "story_arc_placements" +_STARTUP_RECOVERY_PAUSE_REASON = "startup_recovery" +_IMPORT_ENQUEUE_PHASES = frozenset({_IMPORT_BUILD_PHASE, _IMPORT_PLACEMENT_PHASE}) +_IMPORT_PLACEMENT_PAYLOAD_KEYS = frozenset( + { + "schema_version", + "sync_work_id", + "membership_id", + "desired_generation", + "imported_story_arc_id", + "imported_story_arc_entry_id", + "source_import_job_id", + } +) +_IMPORT_HISTORY_CLEANUP_PAGE_SIZE = 1_000 +_MANAGED_IMPORT_MODES = frozenset({"copy", "hardlink", "symlink"}) +_PENDING_IMPORT_WORK_STATES = frozenset( + { + StoryArcSyncWorkState.QUEUED, + StoryArcSyncWorkState.RUNNING, + StoryArcSyncWorkState.RETRY_WAIT, + } +) +_RETRYABLE_ERROR_CODES = frozenset( + { + "placement_concurrency_conflict", + "placement_execution_failed", + } +) + + +@dataclass(frozen=True, slots=True) +class StoryArcSyncDrainResult: + """Bounded worker outcome used by task logging and continuation scheduling.""" + + discovered: int + claimed: int + completed: int + failed: int + retrying: int + cancelled: int + lost_claims: int + has_more: bool + next_retry_at: datetime | None + import_jobs_evaluated: tuple[int, ...] = () + import_jobs_completed: tuple[int, ...] = () + import_jobs_stalled: tuple[int, ...] = () + import_jobs_rollback_ready: tuple[int, ...] = () + + +@dataclass(frozen=True, slots=True) +class StoryArcImportSyncEnqueueResult: + """Idempotent import enqueue result without retrofitting prior ownership.""" + + work: StoryArcSyncWork | None + action: ImportJobAction | None + classification: str + desired_generation: str + + +@dataclass(frozen=True, slots=True) +class ImportStoryArcSyncProposal: + """One exact staged origin requesting managed placement synchronization.""" + + library_file: LibraryFile + membership: IssueStoryArc + story_arc: StoryArc + imported_story_arc_id: int + imported_story_arc_entry_id: int + + +@dataclass(frozen=True, slots=True) +class _PreparedImportStoryArcSyncProposal: + proposal: ImportStoryArcSyncProposal + desired_generation: str + source_signature_hash: str + + @property + def key(self) -> tuple[int, str]: + return (int(self.proposal.membership.id), self.desired_generation) + + @property + def origin(self) -> tuple[int, int]: + return ( + self.proposal.imported_story_arc_id, + self.proposal.imported_story_arc_entry_id, + ) + + +@dataclass(frozen=True, slots=True) +class _WorkContext: + work_id: int + membership_id: int + story_arc_id: int + library_file_id: int + attempt_count: int + import_provenance: StoryArcPlacementImportProvenance | None + + +def _stable_hash(value: object) -> str: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def _origin_is_not_startup_recovery_paused() -> Any: + """Fence origin work until its startup-recovered import is actively resumed.""" + protected_origin = exists().where( + ImportJob.id == StoryArcSyncWork.origin_import_job_id, + ImportJob.status == ImportJobStatus.PAUSED, + ImportJob.progress_snapshot["pause_reason"].as_string() == _STARTUP_RECOVERY_PAUSE_REASON, + ) + return ~protected_origin + + +def _source_signature_hash(library_file: LibraryFile) -> str: + return _stable_hash( + { + "file_path": library_file.file_path, + "file_size": library_file.file_size, + "file_modified_at": library_file.file_modified_at, + "file_hash": library_file.file_hash, + "source_signature": dict(library_file.source_signature or {}), + } + ) + + +def _source_signature_int(library_file: LibraryFile, key: str) -> int | None: + value = dict(library_file.source_signature or {}).get(key) + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def _source_signature_path(library_file: LibraryFile) -> str | None: + value = dict(library_file.source_signature or {}).get("resolved_path") + return value if isinstance(value, str) else None + + +def _desired_generation( + library_file: LibraryFile, + membership: IssueStoryArc, + story_arc: StoryArc, +) -> tuple[str, str]: + source_hash = _source_signature_hash(library_file) + return ( + _stable_hash( + { + "library_file_id": library_file.id, + "source_signature_hash": source_hash, + "issue_story_arc_id": membership.id, + "sequence_number": membership.sequence_number, + "story_arc_revision": story_arc.revision, + "policy_schema_version": story_arc.policy_schema_version, + } + ), + source_hash, + ) + + +def _automatic_sync_enabled(story_arc: StoryArc) -> bool: + snapshot = dict(story_arc.policy_snapshot or {}) + return bool( + story_arc.lifecycle is StoryArcLifecycle.ACTIVE + and story_arc.sync_enabled + and story_arc.policy_schema_version == STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION + and set(snapshot) == _POLICY_KEYS + and snapshot.get("schema_version") == STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION + and snapshot.get("synchronize") is True + and snapshot.get("mode") != "logical" + and isinstance(snapshot.get("destination_root"), str) + and bool(str(snapshot.get("destination_root")).strip()) + and isinstance(snapshot.get("target_library_root_id"), int) + and not isinstance(snapshot.get("target_library_root_id"), bool) + ) + + +async def _eligible_memberships_for_issue( + session: AsyncSession, + issue_id: int, +) -> list[tuple[IssueStoryArc, StoryArc]]: + rows = list( + ( + await session.execute( + select(IssueStoryArc, StoryArc) + .join(StoryArc, IssueStoryArc.story_arc_id == StoryArc.id) + .where( + IssueStoryArc.issue_id == issue_id, + IssueStoryArc.sync_eligible.is_(True), + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + StoryArc.lifecycle == StoryArcLifecycle.ACTIVE, + StoryArc.sync_enabled.is_(True), + StoryArc.policy_schema_version == STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + ) + .order_by(IssueStoryArc.id.asc()) + .limit(MAX_STORY_ARC_SYNC_ENQUEUE_MEMBERSHIPS) + ) + ).all() + ) + return [ + (membership, story_arc) + for membership, story_arc in rows + if _automatic_sync_enabled(story_arc) + ] + + +async def _enqueue_pairs( + session: AsyncSession, + library_file: LibraryFile, + pairs: list[tuple[IssueStoryArc, StoryArc]], + *, + reason: StoryArcSyncReason, +) -> int: + proposals: list[tuple[IssueStoryArc, StoryArc, str, str]] = [] + for membership, story_arc in pairs: + generation, source_hash = _desired_generation(library_file, membership, story_arc) + proposals.append((membership, story_arc, generation, source_hash)) + if not proposals: + return 0 + + membership_ids = [membership.id for membership, _arc, _generation, _hash in proposals] + desired_generations = [generation for _membership, _arc, generation, _hash in proposals] + existing = set( + ( + await session.execute( + select( + StoryArcSyncWork.issue_story_arc_id, + StoryArcSyncWork.desired_generation, + ) + .where( + StoryArcSyncWork.issue_story_arc_id.in_(membership_ids), + StoryArcSyncWork.desired_generation.in_(desired_generations), + ) + .limit(len(proposals)) + ) + ).all() + ) + queued = 0 + for membership, story_arc, generation, source_hash in proposals: + if (membership.id, generation) in existing: + continue + session.add( + StoryArcSyncWork( + issue_story_arc_id=membership.id, + library_file_id=library_file.id, + desired_generation=generation, + source_signature_hash=source_hash, + source_file_path=library_file.file_path, + source_file_size=library_file.file_size, + source_file_modified_at=library_file.file_modified_at, + source_file_hash=library_file.file_hash, + source_signature_schema_version=_source_signature_int( + library_file, + "schema_version", + ), + source_signature_resolved_path=_source_signature_path(library_file), + source_signature_size=_source_signature_int(library_file, "size"), + source_signature_mtime_ns=_source_signature_int(library_file, "mtime_ns"), + source_signature_device=_source_signature_int(library_file, "device"), + source_signature_inode=_source_signature_int(library_file, "inode"), + story_arc_revision=story_arc.revision, + membership_sequence=membership.sequence_number, + policy_schema_version=story_arc.policy_schema_version + or STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + reason=reason, + state=StoryArcSyncWorkState.QUEUED, + ) + ) + queued += 1 + if queued: + await session.flush() + return queued + + +async def enqueue_story_arc_sync_work( + session: AsyncSession, + library_file: LibraryFile, + *, + reason: StoryArcSyncReason = StoryArcSyncReason.CANONICAL_REGISTERED, +) -> int: + """Add DB-only work for every currently eligible arc in the caller's transaction.""" + if library_file.id is None or library_file.issue_id is None: + return 0 + pairs = await _eligible_memberships_for_issue(session, library_file.issue_id) + return await _enqueue_pairs(session, library_file, pairs, reason=reason) + + +def _positive_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +def _import_managed_policy_configured(story_arc: StoryArc) -> bool: + snapshot = dict(story_arc.policy_snapshot or {}) + mode = snapshot.get("mode") + symlink_style = snapshot.get("symlink_style") + return bool( + story_arc.lifecycle is StoryArcLifecycle.ACTIVE + and story_arc.policy_schema_version == STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION + and set(snapshot) == _POLICY_KEYS + and snapshot.get("schema_version") == STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION + and mode in _MANAGED_IMPORT_MODES + and isinstance(snapshot.get("destination_root"), str) + and bool(str(snapshot.get("destination_root")).strip()) + and isinstance(snapshot.get("target_library_root_id"), int) + and not isinstance(snapshot.get("target_library_root_id"), bool) + and isinstance(snapshot.get("synchronize"), bool) + and ( + (mode == "symlink" and symlink_style in {"absolute", "relative"}) + or (mode != "symlink" and symlink_style is None) + ) + ) + + +def _origin_payload_matches( + action: ImportJobAction, + work: StoryArcSyncWork, + *, + job_id: int, + membership_id: int, +) -> bool: + payload = dict(action.payload or {}) + schema_version = payload.get("schema_version") + sync_work_id = payload.get("sync_work_id") + payload_membership_id = payload.get("membership_id") + desired_generation = payload.get("desired_generation") + imported_story_arc_id = payload.get("imported_story_arc_id") + imported_story_arc_entry_id = payload.get("imported_story_arc_entry_id") + source_import_job_id = payload.get("source_import_job_id") + return bool( + work.origin_import_job_id == job_id + and _positive_int(work.origin_imported_story_arc_id) + and _positive_int(work.origin_imported_story_arc_entry_id) + and set(payload) == _IMPORT_PLACEMENT_PAYLOAD_KEYS + and _positive_int(schema_version) + and schema_version == 1 + and _positive_int(sync_work_id) + and sync_work_id == work.id + and _positive_int(payload_membership_id) + and payload_membership_id == membership_id + and isinstance(desired_generation, str) + and desired_generation == work.desired_generation + and _positive_int(imported_story_arc_id) + and imported_story_arc_id == work.origin_imported_story_arc_id + and _positive_int(imported_story_arc_entry_id) + and imported_story_arc_entry_id == work.origin_imported_story_arc_entry_id + and _positive_int(source_import_job_id) + and source_import_job_id == work.origin_import_job_id + ) + + +def _is_exact_unpublished_import_work( + work: StoryArcSyncWork, + action: ImportJobAction | None, + staged_arc: ImportedStoryArc | None, + staged_entry: ImportedStoryArcEntry | None, + *, + placement_action_ids: frozenset[int], +) -> bool: + """Return whether one held row is safe to discard with import history.""" + if ( + work.claimable + or work.state is not StoryArcSyncWorkState.QUEUED + or work.attempt_count != 0 + or work.next_attempt_at is not None + or work.claim_token is not None + or work.claimed_at is not None + or work.cancel_requested_at is not None + or work.last_error_code is not None + or work.last_error_category is not None + or work.last_error_detail is not None + or bool(dict(work.last_result or {})) + or action is None + or staged_arc is None + or staged_entry is None + or action.id in placement_action_ids + ): + return False + job_id = work.origin_import_job_id + return bool( + isinstance(job_id, int) + and job_id > 0 + and action.id == work.origin_import_action_id + and action.import_job_id == job_id + and action.phase == _IMPORT_PLACEMENT_PHASE + and action.action_type == _IMPORT_PLACEMENT_ACTION_TYPE + and action.status is ImportJobActionStatus.COMPLETED + and _origin_payload_matches( + action, + work, + job_id=job_id, + membership_id=int(work.issue_story_arc_id), + ) + and staged_arc.id == work.origin_imported_story_arc_id + and staged_arc.import_job_id == job_id + and staged_entry.id == work.origin_imported_story_arc_entry_id + and staged_entry.imported_story_arc_id == staged_arc.id + and staged_entry.materialized_membership_id == work.issue_story_arc_id + ) + + +async def discard_unpublished_import_story_arc_sync_work( + session: AsyncSession, + job_ids: Sequence[int], +) -> int: + """Remove only exact, never-published placement reservations for deleted history. + + Validation completes before the bulk delete. Any attempted, malformed, or + artifact-owning row must retain its import provenance and use rollback. + """ + normalized_job_ids = tuple(sorted({int(job_id) for job_id in job_ids if job_id > 0})) + if not normalized_job_ids: + return 0 + + unsafe_work_id = await session.scalar( + select(StoryArcSyncWork.id) + .where( + StoryArcSyncWork.origin_import_job_id.in_(normalized_job_ids), + StoryArcSyncWork.claimable.is_(True), + StoryArcSyncWork.state.in_( + { + StoryArcSyncWorkState.QUEUED, + StoryArcSyncWorkState.RUNNING, + StoryArcSyncWorkState.RETRY_WAIT, + StoryArcSyncWorkState.FAILED, + } + ), + ) + .order_by(StoryArcSyncWork.id.asc()) + .limit(1) + ) + if unsafe_work_id is not None: + raise ValidationError( + "Import history cannot be deleted while Story Arc placement work " + "needs rollback. Roll back this import before deleting its history." + ) + + last_work_id = 0 + while True: + rows = list( + ( + await session.execute( + select( + StoryArcSyncWork, + ImportJobAction, + ImportedStoryArc, + ImportedStoryArcEntry, + ) + .outerjoin( + ImportJobAction, + ImportJobAction.id == StoryArcSyncWork.origin_import_action_id, + ) + .outerjoin( + ImportedStoryArc, + ImportedStoryArc.id == StoryArcSyncWork.origin_imported_story_arc_id, + ) + .outerjoin( + ImportedStoryArcEntry, + ImportedStoryArcEntry.id + == StoryArcSyncWork.origin_imported_story_arc_entry_id, + ) + .where( + StoryArcSyncWork.origin_import_job_id.in_(normalized_job_ids), + StoryArcSyncWork.claimable.is_(False), + StoryArcSyncWork.id > last_work_id, + ) + .order_by(StoryArcSyncWork.id.asc()) + .limit(_IMPORT_HISTORY_CLEANUP_PAGE_SIZE) + ) + ).all() + ) + if not rows: + break + action_ids = tuple( + int(action.id) + for _work, action, _staged_arc, _staged_entry in rows + if action is not None + ) + placement_action_ids = frozenset( + int(action_id) + for action_id in ( + ( + await session.scalars( + select(StoryArcPlacement.creating_action_id).where( + StoryArcPlacement.creating_action_id.in_(action_ids) + ) + ) + ).all() + if action_ids + else () + ) + if action_id is not None + ) + for work, action, staged_arc, staged_entry in rows: + if not _is_exact_unpublished_import_work( + work, + action, + staged_arc, + staged_entry, + placement_action_ids=placement_action_ids, + ): + raise ValidationError( + "Import history cannot be deleted while Story Arc placement work " + "needs rollback. Roll back this import before deleting its history." + ) + last_work_id = int(rows[-1][0].id) + + result = await session.execute( + delete(StoryArcSyncWork).where( + StoryArcSyncWork.origin_import_job_id.in_(normalized_job_ids), + StoryArcSyncWork.claimable.is_(False), + ) + ) + cursor_result = cast("CursorResult[Any]", result) + return max(int(cursor_result.rowcount or 0), 0) + + +def _validate_import_story_arc_sync_proposal( + job: ImportJob, + proposal: ImportStoryArcSyncProposal, +) -> None: + library_file = proposal.library_file + membership = proposal.membership + story_arc = proposal.story_arc + if ( + not _positive_int(job.id) + or not _positive_int(library_file.id) + or not _positive_int(membership.id) + or not _positive_int(story_arc.id) + or not _positive_int(proposal.imported_story_arc_id) + or not _positive_int(proposal.imported_story_arc_entry_id) + or membership.story_arc_id != story_arc.id + or membership.issue_id is None + or membership.issue_id != library_file.issue_id + or membership.resolution_state is not StoryArcResolutionState.RESOLVED + or not _import_managed_policy_configured(story_arc) + ): + raise StoryArcPlacementIntegrationError( + "import_sync_context_invalid", + "Import Story Arc placement context is incomplete or no longer exact", + category="validation", + ) + + +def _import_story_arc_work_insert_statement(*, returning: bool) -> Any: + statement = insert(StoryArcSyncWork) + return statement.returning(StoryArcSyncWork) if returning else statement + + +def _import_story_arc_work_row( + *, + job_id: int, + action_id: int, + prepared: _PreparedImportStoryArcSyncProposal, +) -> dict[str, Any]: + proposal = prepared.proposal + library_file = proposal.library_file + membership = proposal.membership + story_arc = proposal.story_arc + return { + "issue_story_arc_id": membership.id, + "library_file_id": library_file.id, + "origin_import_action_id": action_id, + "origin_import_job_id": job_id, + "origin_imported_story_arc_id": proposal.imported_story_arc_id, + "origin_imported_story_arc_entry_id": proposal.imported_story_arc_entry_id, + "desired_generation": prepared.desired_generation, + "source_signature_hash": prepared.source_signature_hash, + "source_file_path": library_file.file_path, + "source_file_size": library_file.file_size, + "source_file_modified_at": library_file.file_modified_at, + "source_file_hash": library_file.file_hash, + "source_signature_schema_version": _source_signature_int( + library_file, + "schema_version", + ), + "source_signature_resolved_path": _source_signature_path(library_file), + "source_signature_size": _source_signature_int(library_file, "size"), + "source_signature_mtime_ns": _source_signature_int(library_file, "mtime_ns"), + "source_signature_device": _source_signature_int(library_file, "device"), + "source_signature_inode": _source_signature_int(library_file, "inode"), + "story_arc_revision": story_arc.revision, + "membership_sequence": membership.sequence_number, + "policy_schema_version": story_arc.policy_schema_version + or STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + "reason": StoryArcSyncReason.CANONICAL_REGISTERED, + "claimable": False, + "state": StoryArcSyncWorkState.QUEUED, + "attempt_count": 0, + "last_result": {}, + } + + +def _provisional_import_story_arc_action_payload( + *, + job_id: int, + prepared: _PreparedImportStoryArcSyncProposal, +) -> dict[str, Any]: + proposal = prepared.proposal + return { + "schema_version": 1, + "sync_work_id": None, + "membership_id": proposal.membership.id, + "desired_generation": prepared.desired_generation, + "imported_story_arc_id": proposal.imported_story_arc_id, + "imported_story_arc_entry_id": proposal.imported_story_arc_entry_id, + "source_import_job_id": job_id, + } + + +def _invalid_existing_origin() -> StoryArcPlacementIntegrationError: + return StoryArcPlacementIntegrationError( + "import_sync_existing_origin_invalid", + "Existing import placement work has an invalid origin binding", + category="ownership", + ) + + +def _unusable_existing_work() -> StoryArcPlacementIntegrationError: + return StoryArcPlacementIntegrationError( + "import_sync_existing_work_unusable", + "Existing import placement work is terminal without completed placement evidence", + category="conflict", + ) + + +def _invalid_completed_placement() -> StoryArcPlacementIntegrationError: + return StoryArcPlacementIntegrationError( + "import_sync_completed_placement_invalid", + "Completed import placement work lacks one exact owned placement", + category="ownership", + ) + + +def _unverified_non_origin_placement() -> StoryArcPlacementIntegrationError: + return StoryArcPlacementIntegrationError( + "import_sync_non_origin_placement_unverified", + "Existing non-import work lacks one exact current placement", + category="conflict", + ) + + +def _work_matches_prepared_generation( + work: StoryArcSyncWork, + prepared: _PreparedImportStoryArcSyncProposal, +) -> bool: + proposal = prepared.proposal + return bool( + work.issue_story_arc_id == proposal.membership.id + and work.library_file_id == proposal.library_file.id + and work.desired_generation == prepared.desired_generation + and work.source_signature_hash == prepared.source_signature_hash + and work.story_arc_revision == proposal.story_arc.revision + and work.membership_sequence == proposal.membership.sequence_number + and work.policy_schema_version + == (proposal.story_arc.policy_schema_version or STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION) + ) + + +def _placement_matches_work_generation( + placement: StoryArcPlacement, + work: StoryArcSyncWork, +) -> bool: + mode_ownership_valid = bool( + ( + placement.mode is StoryArcPlacementMode.REFERENCE_ONLY + and placement.ownership is StoryArcPlacementOwnership.REFERENCED + ) + or ( + placement.mode + in { + StoryArcPlacementMode.COPY, + StoryArcPlacementMode.HARDLINK, + StoryArcPlacementMode.SYMLINK, + } + and placement.ownership is StoryArcPlacementOwnership.MANAGED + ) + ) + return bool( + placement.issue_story_arc_id == work.issue_story_arc_id + and placement.library_file_id == work.library_file_id + and placement.rendered_reading_order == work.membership_sequence + and placement.policy_schema_version == work.policy_schema_version + and placement.state is StoryArcPlacementState.CURRENT + and placement.operation_token is None + and mode_ownership_valid + ) + + +def _completed_origin_placement_matches( + placement: StoryArcPlacement, + work: StoryArcSyncWork, +) -> bool: + return bool( + _placement_matches_work_generation(placement, work) + and placement.ownership is StoryArcPlacementOwnership.MANAGED + and placement.mode + in { + StoryArcPlacementMode.COPY, + StoryArcPlacementMode.HARDLINK, + StoryArcPlacementMode.SYMLINK, + } + and placement.source_kind is StoryArcSourceKind.PULLBOX + and dict(placement.last_result or {}).get("status") == "complete" + ) + + +def _existing_work_placement_statement( + *, + action_ids: Sequence[int], + membership_library_pairs: Sequence[tuple[int, int]], +) -> Any: + """Load bounded evidence for origin ownership and non-origin generation checks.""" + criteria: list[Any] = [] + if action_ids: + criteria.append(StoryArcPlacement.creating_action_id.in_(action_ids)) + if membership_library_pairs: + criteria.append( + tuple_( + StoryArcPlacement.issue_story_arc_id, + StoryArcPlacement.library_file_id, + ).in_(membership_library_pairs) + ) + if not criteria: + raise ValueError("Existing-work placement lookup requires at least one exact key") + return select(StoryArcPlacement).where(or_(*criteria)).order_by(StoryArcPlacement.id.asc()) + + +def _classify_existing_import_work( + *, + job_id: int, + prepared: _PreparedImportStoryArcSyncProposal, + work: StoryArcSyncWork, + action: ImportJobAction | None, + origin_binding: tuple[int, int | None, int, int | None] | None, + pair_placements: Sequence[StoryArcPlacement], + action_placements: Sequence[StoryArcPlacement], +) -> StoryArcImportSyncEnqueueResult: + proposal = prepared.proposal + if not _work_matches_prepared_generation(work, prepared): + if work.origin_import_action_id is not None: + raise _invalid_existing_origin() + raise _unverified_non_origin_placement() + if work.origin_import_action_id is None: + if any( + value is not None + for value in ( + work.origin_import_job_id, + work.origin_imported_story_arc_id, + work.origin_imported_story_arc_entry_id, + ) + ): + raise _invalid_existing_origin() + exact_placements = [ + placement + for placement in pair_placements + if _placement_matches_work_generation(placement, work) + ] + if len(exact_placements) != 1: + raise _unverified_non_origin_placement() + return StoryArcImportSyncEnqueueResult( + work=work, + action=None, + classification="existing_non_origin_placement", + desired_generation=prepared.desired_generation, + ) + if ( + action is None + or action.id != work.origin_import_action_id + or action.import_job_id != job_id + or action.phase != _IMPORT_PLACEMENT_PHASE + or action.action_type != _IMPORT_PLACEMENT_ACTION_TYPE + or action.status is not ImportJobActionStatus.COMPLETED + or work.origin_import_job_id != job_id + or origin_binding + != ( + work.origin_imported_story_arc_id, + work.issue_story_arc_id, + job_id, + proposal.story_arc.id, + ) + or not _origin_payload_matches( + action, + work, + job_id=job_id, + membership_id=int(proposal.membership.id), + ) + ): + raise _invalid_existing_origin() + same_staged_origin = bool( + work.origin_imported_story_arc_id == proposal.imported_story_arc_id + and work.origin_imported_story_arc_entry_id == proposal.imported_story_arc_entry_id + ) + if not same_staged_origin and ( + work.origin_imported_story_arc_id != proposal.imported_story_arc_id + ): + raise _invalid_existing_origin() + if work.state in _PENDING_IMPORT_WORK_STATES: + classification = ( + "existing_import_work_pending" + if same_staged_origin + else "existing_import_membership_duplicate" + ) + elif work.state is StoryArcSyncWorkState.COMPLETED: + if ( + len(action_placements) != 1 + or action_placements[0].source_import_job_id != job_id + or action_placements[0].creating_action_id != action.id + or not _completed_origin_placement_matches(action_placements[0], work) + ): + raise _invalid_completed_placement() + classification = ( + "existing_import_work_completed" + if same_staged_origin + else "existing_import_membership_duplicate" + ) + else: + raise _unusable_existing_work() + return StoryArcImportSyncEnqueueResult( + work=work, + action=action, + classification=classification, + desired_generation=prepared.desired_generation, + ) + + +async def enqueue_import_story_arc_sync_work_batch( + session: AsyncSession, + *, + job: ImportJob, + proposals: Sequence[ImportStoryArcSyncProposal], + record_actions: RecordActionsFunc, +) -> list[StoryArcImportSyncEnqueueResult]: + """Create a bounded ordered batch of exact import action/work bindings. + + All proposal, staged-origin, duplicate, existing-work, and placement checks + finish before the callback is allowed to write the first journal action. + The caller owns the surrounding transaction; this helper never commits. + """ + ordered_proposals = tuple(proposals) + if not ordered_proposals: + return [] + if len(ordered_proposals) > MAX_IMPORT_STORY_ARC_SYNC_ENQUEUE_BATCH_SIZE: + raise ValueError( + "Import Story Arc placement enqueue accepts at most " + f"{MAX_IMPORT_STORY_ARC_SYNC_ENQUEUE_BATCH_SIZE} proposals" + ) + for proposal in ordered_proposals: + _validate_import_story_arc_sync_proposal(job, proposal) + if ( + job.status is not ImportJobStatus.IMPORTING + or job.control_request is not ImportControlRequest.NONE + or dict(job.progress_snapshot or {}).get("phase") not in _IMPORT_ENQUEUE_PHASES + ): + raise StoryArcPlacementIntegrationError( + "import_sync_job_inactive", + "Import job is not actively publishing Story Arc placements", + category="cancelled", + ) + + prepared = [ + _PreparedImportStoryArcSyncProposal( + proposal=proposal, + desired_generation=generation, + source_signature_hash=source_hash, + ) + for proposal in ordered_proposals + for generation, source_hash in [ + _desired_generation( + proposal.library_file, + proposal.membership, + proposal.story_arc, + ) + ] + ] + entry_ids = sorted({item.proposal.imported_story_arc_entry_id for item in prepared}) + staged_rows = ( + await session.execute( + select( + ImportedStoryArcEntry.id, + ImportedStoryArcEntry.imported_story_arc_id, + ImportedStoryArcEntry.materialized_membership_id, + ImportedStoryArc.import_job_id, + ImportedStoryArc.materialized_story_arc_id, + ) + .join( + ImportedStoryArc, + ImportedStoryArcEntry.imported_story_arc_id == ImportedStoryArc.id, + ) + .where(ImportedStoryArcEntry.id.in_(entry_ids)) + ) + ).all() + staged_bindings = { + int(entry_id): ( + int(imported_story_arc_id), + materialized_membership_id, + int(import_job_id), + materialized_story_arc_id, + ) + for ( + entry_id, + imported_story_arc_id, + materialized_membership_id, + import_job_id, + materialized_story_arc_id, + ) in staged_rows + } + for item in prepared: + proposal = item.proposal + if staged_bindings.get(proposal.imported_story_arc_entry_id) != ( + proposal.imported_story_arc_id, + proposal.membership.id, + job.id, + proposal.story_arc.id, + ): + raise StoryArcPlacementIntegrationError( + "import_sync_origin_binding_invalid", + "Imported Story Arc entry does not own the requested membership", + category="ownership", + ) + + imported_arc_by_key: dict[tuple[int, str], int] = {} + for item in prepared: + prior_imported_arc_id = imported_arc_by_key.setdefault( + item.key, + item.proposal.imported_story_arc_id, + ) + if prior_imported_arc_id != item.proposal.imported_story_arc_id: + raise _invalid_existing_origin() + + keys = sorted(imported_arc_by_key) + existing_work_rows = list( + ( + await session.scalars( + select(StoryArcSyncWork).where( + tuple_( + StoryArcSyncWork.issue_story_arc_id, + StoryArcSyncWork.desired_generation, + ).in_(keys) + ) + ) + ).all() + ) + work_by_key = { + (work.issue_story_arc_id, work.desired_generation): work for work in existing_work_rows + } + missing_origin_entry_ids = sorted( + { + int(work.origin_imported_story_arc_entry_id) + for work in existing_work_rows + if work.origin_imported_story_arc_entry_id is not None + and int(work.origin_imported_story_arc_entry_id) not in staged_bindings + } + ) + if missing_origin_entry_ids: + origin_staged_rows = ( + await session.execute( + select( + ImportedStoryArcEntry.id, + ImportedStoryArcEntry.imported_story_arc_id, + ImportedStoryArcEntry.materialized_membership_id, + ImportedStoryArc.import_job_id, + ImportedStoryArc.materialized_story_arc_id, + ) + .join( + ImportedStoryArc, + ImportedStoryArcEntry.imported_story_arc_id == ImportedStoryArc.id, + ) + .where(ImportedStoryArcEntry.id.in_(missing_origin_entry_ids)) + ) + ).all() + staged_bindings.update( + { + int(entry_id): ( + int(imported_story_arc_id), + materialized_membership_id, + int(import_job_id), + materialized_story_arc_id, + ) + for ( + entry_id, + imported_story_arc_id, + materialized_membership_id, + import_job_id, + materialized_story_arc_id, + ) in origin_staged_rows + } + ) + action_ids = sorted( + { + int(work.origin_import_action_id) + for work in existing_work_rows + if work.origin_import_action_id is not None + } + ) + actions_by_id: dict[int, ImportJobAction] = {} + if action_ids: + actions_by_id = { + int(action.id): action + for action in ( + await session.scalars( + select(ImportJobAction).where(ImportJobAction.id.in_(action_ids)) + ) + ).all() + } + + placement_rows: list[StoryArcPlacement] = [] + if existing_work_rows: + placement_rows = list( + ( + await session.scalars( + _existing_work_placement_statement( + action_ids=action_ids, + membership_library_pairs=sorted( + { + (work.issue_story_arc_id, work.library_file_id) + for work in existing_work_rows + } + ), + ) + ) + ).all() + ) + placements_by_pair: dict[tuple[int, int], list[StoryArcPlacement]] = {} + placements_by_action: dict[int, list[StoryArcPlacement]] = {} + for placement in placement_rows: + if placement.library_file_id is not None: + placements_by_pair.setdefault( + (int(placement.issue_story_arc_id), int(placement.library_file_id)), + [], + ).append(placement) + if placement.creating_action_id is not None: + placements_by_action.setdefault(int(placement.creating_action_id), []).append(placement) + + representative_index_by_key: dict[tuple[int, str], int] = {} + for index, item in enumerate(prepared): + prior_index = representative_index_by_key.get(item.key) + if ( + prior_index is None + or item.proposal.imported_story_arc_entry_id + < prepared[prior_index].proposal.imported_story_arc_entry_id + ): + representative_index_by_key[item.key] = index + representative_by_key = { + key: prepared[index] for key, index in representative_index_by_key.items() + } + existing_results_by_key: dict[tuple[int, str], StoryArcImportSyncEnqueueResult] = {} + for key, work in work_by_key.items(): + item = representative_by_key[key] + existing_results_by_key[key] = _classify_existing_import_work( + job_id=int(job.id), + prepared=item, + work=work, + action=( + actions_by_id.get(int(work.origin_import_action_id)) + if work.origin_import_action_id is not None + else None + ), + origin_binding=( + staged_bindings.get(int(work.origin_imported_story_arc_entry_id)) + if work.origin_imported_story_arc_entry_id is not None + else None + ), + pair_placements=placements_by_pair.get( + (int(work.issue_story_arc_id), int(work.library_file_id)), + (), + ), + action_placements=( + placements_by_action.get(int(work.origin_import_action_id), ()) + if work.origin_import_action_id is not None + else () + ), + ) + + memberships_without_work = sorted( + {item.key[0] for item in prepared if item.key not in existing_results_by_key} + ) + placements_by_membership: dict[int, StoryArcPlacement] = {} + if memberships_without_work: + first_placement_ids = ( + select( + StoryArcPlacement.issue_story_arc_id.label("membership_id"), + func.min(StoryArcPlacement.id).label("placement_id"), + ) + .where(StoryArcPlacement.issue_story_arc_id.in_(memberships_without_work)) + .group_by(StoryArcPlacement.issue_story_arc_id) + .subquery() + ) + placements_by_membership = { + int(placement.issue_story_arc_id): placement + for placement in ( + await session.scalars( + select(StoryArcPlacement).join( + first_placement_ids, + StoryArcPlacement.id == first_placement_ids.c.placement_id, + ) + ) + ).all() + } + + results: list[StoryArcImportSyncEnqueueResult | None] = [None for _item in prepared] + creation_items: list[_PreparedImportStoryArcSyncProposal] = [] + creation_index_by_key: dict[tuple[int, str], int] = {} + duplicate_indexes_by_key: dict[tuple[int, str], list[int]] = {} + for index, item in enumerate(prepared): + representative_index = representative_index_by_key[item.key] + if representative_index != index: + duplicate_indexes_by_key.setdefault(item.key, []).append(index) + continue + existing_result = existing_results_by_key.get(item.key) + if existing_result is not None: + results[index] = existing_result + continue + existing_placement = placements_by_membership.get(item.key[0]) + if existing_placement is not None: + results[index] = StoryArcImportSyncEnqueueResult( + work=None, + action=None, + classification=( + "existing_managed_placement" + if existing_placement.ownership is StoryArcPlacementOwnership.MANAGED + else "existing_referenced_placement" + ), + desired_generation=item.desired_generation, + ) + continue + creation_index_by_key[item.key] = index + creation_items.append(item) + + if creation_items: + provisional_payloads = [ + _provisional_import_story_arc_action_payload( + job_id=int(job.id), + prepared=item, + ) + for item in creation_items + ] + specs = [ + ImportJobActionSpec( + phase=_IMPORT_PLACEMENT_PHASE, + action_type=_IMPORT_PLACEMENT_ACTION_TYPE, + payload=payload, + ) + for payload in provisional_payloads + ] + actions = await record_actions(session, job, specs) + if ( + len(actions) != len(specs) + or len({action.id for action in actions}) != len(actions) + or any( + not _positive_int(action.id) + or action.import_job_id != job.id + or action.phase != spec.phase + or action.action_type != spec.action_type + or dict(action.payload or {}) != spec.payload + for action, spec in zip(actions, specs, strict=True) + ) + ): + raise StoryArcPlacementIntegrationError( + "import_sync_action_invalid", + "Import action recorder returned an invalid ownership action", + category="ownership", + ) + work_rows = [ + _import_story_arc_work_row( + job_id=int(job.id), + action_id=int(action.id), + prepared=item, + ) + for item, action in zip(creation_items, actions, strict=True) + ] + works = list( + ( + await session.scalars( + _import_story_arc_work_insert_statement(returning=True), + work_rows, + ) + ).all() + ) + work_by_action_id = { + int(work.origin_import_action_id): work + for work in works + if work.origin_import_action_id is not None + } + if len(work_by_action_id) != len(actions): + raise StoryArcPlacementIntegrationError( + "import_sync_action_invalid", + "Import action recorder returned an invalid ownership action", + category="ownership", + ) + for item, action, payload in zip( + creation_items, + actions, + provisional_payloads, + strict=True, + ): + created_work = work_by_action_id.get(int(action.id)) + if created_work is None or not _positive_int(created_work.id): + raise StoryArcPlacementIntegrationError( + "import_sync_action_invalid", + "Import action recorder returned an invalid ownership action", + category="ownership", + ) + action.payload = {**payload, "sync_work_id": int(created_work.id)} + creation_result = StoryArcImportSyncEnqueueResult( + work=created_work, + action=action, + classification="created", + desired_generation=item.desired_generation, + ) + results[creation_index_by_key[item.key]] = creation_result + await session.flush() + + for key, duplicate_indexes in duplicate_indexes_by_key.items(): + representative_index = representative_index_by_key[key] + representative_result = results[representative_index] + if representative_result is None: + raise RuntimeError("Import Story Arc placement duplicate lost its representative") + for duplicate_index in duplicate_indexes: + duplicate = prepared[duplicate_index] + representative = prepared[representative_index] + results[duplicate_index] = StoryArcImportSyncEnqueueResult( + work=representative_result.work, + action=representative_result.action, + classification=( + "in_call_duplicate" + if duplicate.origin == representative.origin + else "in_call_membership_duplicate" + ), + desired_generation=representative_result.desired_generation, + ) + + if any(result is None for result in results): + raise RuntimeError("Import Story Arc placement enqueue left an unclassified proposal") + return [cast("StoryArcImportSyncEnqueueResult", result) for result in results] + + +async def enqueue_import_story_arc_sync_work( + session: AsyncSession, + *, + job: ImportJob, + library_file: LibraryFile, + membership: IssueStoryArc, + story_arc: StoryArc, + imported_story_arc_id: int, + imported_story_arc_entry_id: int, + record_action: RecordActionFunc, +) -> StoryArcImportSyncEnqueueResult: + """Compatibility wrapper for one import-owned placement proposal.""" + + async def record_actions_adapter( + callback_session: AsyncSession, + callback_job: ImportJob, + specs: Sequence[ImportJobActionSpec], + ) -> list[ImportJobAction]: + return [ + await record_action( + callback_session, + callback_job, + phase=spec.phase, + action_type=spec.action_type, + payload=spec.payload, + ) + for spec in specs + ] + + results = await enqueue_import_story_arc_sync_work_batch( + session, + job=job, + proposals=[ + ImportStoryArcSyncProposal( + library_file=library_file, + membership=membership, + story_arc=story_arc, + imported_story_arc_id=imported_story_arc_id, + imported_story_arc_entry_id=imported_story_arc_entry_id, + ) + ], + record_actions=cast("RecordActionsFunc", record_actions_adapter), + ) + return results[0] + + +async def discover_story_arc_sync_work( + session: AsyncSession, + *, + limit: int = DEFAULT_STORY_ARC_DISCOVERY_LIMIT, +) -> int: + """Boundedly recover eligible canonical files with no current work generation.""" + if isinstance(limit, bool) or not 1 <= limit <= DEFAULT_STORY_ARC_DISCOVERY_LIMIT: + raise ValueError( + f"Story-arc discrepancy limit must be from 1 to {DEFAULT_STORY_ARC_DISCOVERY_LIMIT}" + ) + + current_work = exists().where( + StoryArcSyncWork.issue_story_arc_id == IssueStoryArc.id, + StoryArcSyncWork.library_file_id == LibraryFile.id, + StoryArcSyncWork.story_arc_revision == StoryArc.revision, + StoryArcSyncWork.membership_sequence == IssueStoryArc.sequence_number, + StoryArcSyncWork.policy_schema_version == StoryArc.policy_schema_version, + StoryArcSyncWork.source_file_path == LibraryFile.file_path, + StoryArcSyncWork.source_file_size == LibraryFile.file_size, + StoryArcSyncWork.source_file_modified_at == LibraryFile.file_modified_at, + or_( + StoryArcSyncWork.source_file_hash == LibraryFile.file_hash, + and_( + StoryArcSyncWork.source_file_hash.is_(None), + LibraryFile.file_hash.is_(None), + ), + ), + or_( + StoryArcSyncWork.source_signature_schema_version + == LibraryFile.source_signature["schema_version"].as_integer(), + and_( + StoryArcSyncWork.source_signature_schema_version.is_(None), + LibraryFile.source_signature["schema_version"].as_integer().is_(None), + ), + ), + or_( + StoryArcSyncWork.source_signature_resolved_path + == LibraryFile.source_signature["resolved_path"].as_string(), + and_( + StoryArcSyncWork.source_signature_resolved_path.is_(None), + LibraryFile.source_signature["resolved_path"].as_string().is_(None), + ), + ), + or_( + StoryArcSyncWork.source_signature_size + == LibraryFile.source_signature["size"].as_integer(), + and_( + StoryArcSyncWork.source_signature_size.is_(None), + LibraryFile.source_signature["size"].as_integer().is_(None), + ), + ), + or_( + StoryArcSyncWork.source_signature_mtime_ns + == LibraryFile.source_signature["mtime_ns"].as_integer(), + and_( + StoryArcSyncWork.source_signature_mtime_ns.is_(None), + LibraryFile.source_signature["mtime_ns"].as_integer().is_(None), + ), + ), + or_( + StoryArcSyncWork.source_signature_device + == LibraryFile.source_signature["device"].as_integer(), + and_( + StoryArcSyncWork.source_signature_device.is_(None), + LibraryFile.source_signature["device"].as_integer().is_(None), + ), + ), + or_( + StoryArcSyncWork.source_signature_inode + == LibraryFile.source_signature["inode"].as_integer(), + and_( + StoryArcSyncWork.source_signature_inode.is_(None), + LibraryFile.source_signature["inode"].as_integer().is_(None), + ), + ), + ) + rows = list( + ( + await session.execute( + select(LibraryFile, IssueStoryArc, StoryArc) + .join(IssueStoryArc, LibraryFile.issue_id == IssueStoryArc.issue_id) + .join(StoryArc, IssueStoryArc.story_arc_id == StoryArc.id) + .where( + IssueStoryArc.sync_eligible.is_(True), + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + StoryArc.lifecycle == StoryArcLifecycle.ACTIVE, + StoryArc.sync_enabled.is_(True), + StoryArc.policy_schema_version == STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + ~current_work, + ) + .order_by(IssueStoryArc.id.asc(), LibraryFile.id.asc()) + .limit(limit) + ) + ).all() + ) + queued = 0 + for library_file, membership, story_arc in rows: + if not _automatic_sync_enabled(story_arc): + continue + queued += await _enqueue_pairs( + session, + library_file, + [(membership, story_arc)], + reason=StoryArcSyncReason.DISCREPANCY_RECOVERY, + ) + return queued + + +async def claim_story_arc_sync_work( + session: AsyncSession, + work_id: int, + *, + now: datetime, + import_only: bool = False, +) -> str | None: + """Atomically lease one ready or stale work row before synchronization I/O.""" + token = secrets.token_urlsafe(24) + stale_before = now - _CLAIM_LEASE + origin_scope = (StoryArcSyncWork.origin_import_job_id.is_not(None),) if import_only else () + result = cast( + "CursorResult[Any]", + await session.execute( + update(StoryArcSyncWork) + .where( + *origin_scope, + _origin_is_not_startup_recovery_paused(), + StoryArcSyncWork.id == work_id, + StoryArcSyncWork.claimable.is_(True), + or_( + StoryArcSyncWork.state == StoryArcSyncWorkState.QUEUED, + and_( + StoryArcSyncWork.state == StoryArcSyncWorkState.RETRY_WAIT, + StoryArcSyncWork.next_attempt_at.is_not(None), + StoryArcSyncWork.next_attempt_at <= now, + ), + and_( + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + or_( + StoryArcSyncWork.claimed_at.is_(None), + StoryArcSyncWork.claimed_at <= stale_before, + ), + ), + ), + ) + .values( + state=StoryArcSyncWorkState.RUNNING, + attempt_count=StoryArcSyncWork.attempt_count + 1, + next_attempt_at=None, + claim_token=token, + claimed_at=now, + ) + ), + ) + await session.commit() + return token if result.rowcount == 1 else None + + +async def _refresh_story_arc_sync_claim( + session: AsyncSession, + work_id: int, + claim_token: str, + *, + now: datetime, +) -> bool: + """Refresh one live claim without reviving work owned by another worker.""" + result = cast( + "CursorResult[Any]", + await session.execute( + update(StoryArcSyncWork) + .where( + StoryArcSyncWork.id == work_id, + StoryArcSyncWork.claim_token == claim_token, + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + ) + .values(claimed_at=now) + ), + ) + await session.commit() + return result.rowcount == 1 + + +async def _maintain_claim_lease( + session_factory: async_sessionmaker[AsyncSession], + work_id: int, + claim_token: str, + stop_requested: asyncio.Event, + *, + interval_seconds: float, + now_fn: Callable[[], datetime], +) -> None: + """Heartbeat a live claim while placement I/O runs in a different session.""" + while not stop_requested.is_set(): + try: + await asyncio.wait_for(stop_requested.wait(), timeout=interval_seconds) + except TimeoutError: + try: + async with session_factory() as heartbeat_session: + refreshed = await _refresh_story_arc_sync_claim( + heartbeat_session, + work_id, + claim_token, + now=now_fn(), + ) + except Exception: + logger.warning( + "story_arc_sync_claim_heartbeat_failed", + work_id=work_id, + exc_info=True, + ) + continue + if not refreshed: + logger.warning( + "story_arc_sync_claim_lost_during_heartbeat", + work_id=work_id, + ) + return + + +async def _origin_claim_may_publish( + session: AsyncSession, + work_id: int, + claim_token: str, +) -> bool: + row = ( + await session.execute( + select(StoryArcSyncWork, ImportJobAction, ImportJob) + .join( + ImportJobAction, + StoryArcSyncWork.origin_import_action_id == ImportJobAction.id, + ) + .join(ImportJob, StoryArcSyncWork.origin_import_job_id == ImportJob.id) + .where( + StoryArcSyncWork.id == work_id, + StoryArcSyncWork.claim_token == claim_token, + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + ) + ) + ).one_or_none() + if row is None: + return False + work, action, job = row + return bool( + work.cancel_requested_at is None + and action.import_job_id == work.origin_import_job_id + and action.status is ImportJobActionStatus.COMPLETED + and action.phase == _IMPORT_PLACEMENT_PHASE + and action.action_type == _IMPORT_PLACEMENT_ACTION_TYPE + and job.status is ImportJobStatus.IMPORTING + and job.control_request is ImportControlRequest.NONE + and dict(job.progress_snapshot or {}).get("phase") == _IMPORT_PLACEMENT_PHASE + and _origin_payload_matches( + action, + work, + job_id=job.id, + membership_id=work.issue_story_arc_id, + ) + ) + + +async def _monitor_origin_cancellation( + session_factory: async_sessionmaker[AsyncSession], + work_id: int, + claim_token: str, + stop_requested: asyncio.Event, + cancellation_requested: asyncio.Event, + *, + interval_seconds: float, +) -> None: + """Poll only claimed import work and signal the filesystem's safe cancellation path.""" + while not stop_requested.is_set(): + try: + await asyncio.wait_for(stop_requested.wait(), timeout=interval_seconds) + except TimeoutError: + try: + async with session_factory() as session: + may_publish = await _origin_claim_may_publish( + session, + work_id, + claim_token, + ) + await session.rollback() + except Exception: + logger.warning( + "story_arc_import_cancellation_check_failed", + work_id=work_id, + exc_info=True, + ) + continue + if not may_publish: + cancellation_requested.set() + return + + +def _ready_work_statements( + *, + now: datetime, + limit: int, + import_only: bool = False, +) -> tuple[Any, ...]: + """Build one bounded, index-orderable query for each readiness lane.""" + stale_before = now - _CLAIM_LEASE + origin_scope = (StoryArcSyncWork.origin_import_job_id.is_not(None),) if import_only else () + return ( + select(StoryArcSyncWork.id, StoryArcSyncWork.claimed_at) + .where( + *origin_scope, + _origin_is_not_startup_recovery_paused(), + StoryArcSyncWork.claimable.is_(True), + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + StoryArcSyncWork.claimed_at.is_(None), + ) + .order_by(StoryArcSyncWork.claimed_at.asc(), StoryArcSyncWork.id.asc()) + .limit(limit), + select(StoryArcSyncWork.id, StoryArcSyncWork.created_at) + .where( + *origin_scope, + _origin_is_not_startup_recovery_paused(), + StoryArcSyncWork.claimable.is_(True), + StoryArcSyncWork.state == StoryArcSyncWorkState.QUEUED, + ) + .order_by(StoryArcSyncWork.created_at.asc(), StoryArcSyncWork.id.asc()) + .limit(limit), + select(StoryArcSyncWork.id, StoryArcSyncWork.next_attempt_at) + .where( + *origin_scope, + _origin_is_not_startup_recovery_paused(), + StoryArcSyncWork.claimable.is_(True), + StoryArcSyncWork.state == StoryArcSyncWorkState.RETRY_WAIT, + StoryArcSyncWork.next_attempt_at.is_not(None), + StoryArcSyncWork.next_attempt_at <= now, + ) + .order_by(StoryArcSyncWork.next_attempt_at.asc(), StoryArcSyncWork.id.asc()) + .limit(limit), + select(StoryArcSyncWork.id, StoryArcSyncWork.claimed_at) + .where( + *origin_scope, + _origin_is_not_startup_recovery_paused(), + StoryArcSyncWork.claimable.is_(True), + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + StoryArcSyncWork.claimed_at.is_not(None), + StoryArcSyncWork.claimed_at <= stale_before, + ) + .order_by(StoryArcSyncWork.claimed_at.asc(), StoryArcSyncWork.id.asc()) + .limit(limit), + ) + + +async def _ready_work_ids( + session: AsyncSession, + *, + now: datetime, + limit: int, + import_only: bool = False, +) -> list[int]: + """Merge bounded ready lanes without an unindexable OR/COALESCE scan.""" + unclaimed_ready_at = datetime.min.replace(tzinfo=UTC) + candidates: dict[int, tuple[datetime, int, int]] = {} + for lane_rank, statement in enumerate( + _ready_work_statements(now=now, limit=limit, import_only=import_only) + ): + rows = (await session.execute(statement)).all() + for work_id, ready_at in rows: + normalized_id = int(work_id) + sort_key = (ready_at or unclaimed_ready_at, lane_rank, normalized_id) + existing = candidates.get(normalized_id) + if existing is None or sort_key < existing: + candidates[normalized_id] = sort_key + ordered = sorted(candidates.items(), key=lambda item: item[1]) + return [work_id for work_id, _sort_key in ordered[:limit]] + + +async def _load_claimed_context( + session: AsyncSession, + work_id: int, + claim_token: str, +) -> _WorkContext | None: + row = ( + await session.execute( + select( + StoryArcSyncWork, + IssueStoryArc, + StoryArc, + LibraryFile, + ImportJobAction, + ImportJob, + ) + .join( + IssueStoryArc, + StoryArcSyncWork.issue_story_arc_id == IssueStoryArc.id, + ) + .join(StoryArc, IssueStoryArc.story_arc_id == StoryArc.id) + .join(LibraryFile, StoryArcSyncWork.library_file_id == LibraryFile.id) + .outerjoin( + ImportJobAction, + StoryArcSyncWork.origin_import_action_id == ImportJobAction.id, + ) + .outerjoin(ImportJob, StoryArcSyncWork.origin_import_job_id == ImportJob.id) + .where( + StoryArcSyncWork.id == work_id, + StoryArcSyncWork.claim_token == claim_token, + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + ) + ) + ).one_or_none() + if row is None: + return None + work, membership, story_arc, library_file, origin_action, origin_job = row + if ( + membership.issue_id is None + or membership.issue_id != library_file.issue_id + or membership.resolution_state is not StoryArcResolutionState.RESOLVED + or work.desired_generation != _desired_generation(library_file, membership, story_arc)[0] + or requires_order_review(membership) + ): + return None + if work.origin_import_action_id is None: + if ( + any( + value is not None + for value in ( + work.origin_import_job_id, + work.origin_imported_story_arc_id, + work.origin_imported_story_arc_entry_id, + ) + ) + or not membership.sync_eligible + or not _automatic_sync_enabled(story_arc) + ): + return None + provenance = None + else: + if ( + work.cancel_requested_at is not None + or origin_action is None + or origin_job is None + or origin_action.id != work.origin_import_action_id + or work.origin_import_job_id != origin_job.id + or origin_action.import_job_id != work.origin_import_job_id + or not _positive_int(work.origin_imported_story_arc_id) + or not _positive_int(work.origin_imported_story_arc_entry_id) + or origin_action.phase != _IMPORT_PLACEMENT_PHASE + or origin_action.action_type != _IMPORT_PLACEMENT_ACTION_TYPE + or origin_action.status is not ImportJobActionStatus.COMPLETED + or origin_job.status is not ImportJobStatus.IMPORTING + or origin_job.control_request is not ImportControlRequest.NONE + or dict(origin_job.progress_snapshot or {}).get("phase") != _IMPORT_PLACEMENT_PHASE + or not _import_managed_policy_configured(story_arc) + or not _origin_payload_matches( + origin_action, + work, + job_id=origin_job.id, + membership_id=membership.id, + ) + ): + return None + imported_story_arc_id = work.origin_imported_story_arc_id + imported_story_arc_entry_id = work.origin_imported_story_arc_entry_id + assert imported_story_arc_id is not None + assert imported_story_arc_entry_id is not None + staged_binding = await session.scalar( + select(ImportedStoryArcEntry.id) + .join( + ImportedStoryArc, + ImportedStoryArcEntry.imported_story_arc_id == ImportedStoryArc.id, + ) + .where( + ImportedStoryArc.id == imported_story_arc_id, + ImportedStoryArc.import_job_id == origin_job.id, + ImportedStoryArc.materialized_story_arc_id == story_arc.id, + ImportedStoryArcEntry.id == imported_story_arc_entry_id, + ImportedStoryArcEntry.materialized_membership_id == membership.id, + ) + ) + if staged_binding is None: + return None + provenance = StoryArcPlacementImportProvenance( + import_job_id=origin_job.id, + import_action_id=origin_action.id, + ) + return _WorkContext( + work_id=work.id, + membership_id=membership.id, + story_arc_id=story_arc.id, + library_file_id=library_file.id, + attempt_count=work.attempt_count, + import_provenance=provenance, + ) + + +async def _finish_work( + session: AsyncSession, + context: _WorkContext, + claim_token: str, + *, + outcome: str, +) -> bool: + result = cast( + "CursorResult[Any]", + await session.execute( + update(StoryArcSyncWork) + .where( + StoryArcSyncWork.id == context.work_id, + StoryArcSyncWork.claim_token == claim_token, + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + ) + .values( + state=StoryArcSyncWorkState.COMPLETED, + claim_token=None, + claimed_at=None, + next_attempt_at=None, + last_error_code=None, + last_error_category=None, + last_error_detail=None, + last_result={"schema_version": 1, "outcome": outcome}, + ) + ), + ) + await session.commit() + return result.rowcount == 1 + + +async def _cancel_work( + session: AsyncSession, + work_id: int, + claim_token: str, + *, + code: str, +) -> bool: + result = cast( + "CursorResult[Any]", + await session.execute( + update(StoryArcSyncWork) + .where( + StoryArcSyncWork.id == work_id, + StoryArcSyncWork.claim_token == claim_token, + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + ) + .values( + state=StoryArcSyncWorkState.CANCELLED, + claim_token=None, + claimed_at=None, + next_attempt_at=None, + last_error_code=code, + last_error_category="superseded", + last_error_detail="Automatic story-arc synchronization is no longer eligible.", + ) + ), + ) + await session.commit() + return result.rowcount == 1 + + +def _is_retryable(exc: StoryArcPlacementIntegrationError) -> bool: + return exc.code in _RETRYABLE_ERROR_CODES or exc.category in {"operation", "cancelled"} + + +async def _fail_or_retry_work( + session: AsyncSession, + context: _WorkContext, + claim_token: str, + *, + now: datetime, + code: str, + category: str, + detail: str, + retryable: bool, +) -> StoryArcSyncWorkState | None: + should_retry = retryable and context.attempt_count < _MAX_ATTEMPTS + state = StoryArcSyncWorkState.RETRY_WAIT if should_retry else StoryArcSyncWorkState.FAILED + next_attempt_at = ( + now + _RETRY_DELAYS[min(context.attempt_count - 1, len(_RETRY_DELAYS) - 1)] + if should_retry + else None + ) + result = cast( + "CursorResult[Any]", + await session.execute( + update(StoryArcSyncWork) + .where( + StoryArcSyncWork.id == context.work_id, + StoryArcSyncWork.claim_token == claim_token, + StoryArcSyncWork.state == StoryArcSyncWorkState.RUNNING, + ) + .values( + state=state, + claim_token=None, + claimed_at=None, + next_attempt_at=next_attempt_at, + last_error_code=code, + last_error_category=category, + last_error_detail=detail, + ) + ), + ) + await session.commit() + return state if result.rowcount == 1 else None + + +async def retry_import_story_arc_sync_work( + session: AsyncSession, + job_id: int, +) -> tuple[ImportJob, int]: + """Safely reopen exact terminal placement work for one stalled import. + + The caller owns the transaction. Every import-origin row is validated + before one conditional UPDATE resets only FAILED/CANCELLED rows; canonical + import execution and any pending or completed placement work are untouched. + """ + from pullbox.services.import_story_arc_placement_completion import ( + inspect_import_story_arc_placement_origin, + ) + + job = await session.scalar(select(ImportJob).where(ImportJob.id == job_id).with_for_update()) + if job is None: + raise NotFoundError("ImportJob", job_id) + + snapshot = dict(job.progress_snapshot or {}) + if ( + job.status is not ImportJobStatus.STALLED + or job.import_started_at is None + or snapshot.get("phase") != _IMPORT_PLACEMENT_PHASE + ): + raise ValidationError("Import job is not stalled on Story Arc placements.") + if job.control_request is not ImportControlRequest.NONE: + raise ValidationError("Import job has an active control request.") + + counts = await inspect_import_story_arc_placement_origin(session, job_id) + expected_total = snapshot.get("story_arc_placements_total") + if ( + not isinstance(expected_total, int) + or isinstance(expected_total, bool) + or expected_total <= 0 + or counts.total != expected_total + ): + raise ValidationError("Import Story Arc placement origin evidence is incomplete.") + + retrying_count = counts.failed + counts.cancelled + if retrying_count == 0: + raise ValidationError("No failed or cancelled Story Arc placement work to retry.") + + terminal_states = ( + StoryArcSyncWorkState.FAILED, + StoryArcSyncWorkState.CANCELLED, + ) + terminal_rows = ( + await session.execute( + select(StoryArcSyncWork, IssueStoryArc, StoryArc, LibraryFile) + .join( + IssueStoryArc, + StoryArcSyncWork.issue_story_arc_id == IssueStoryArc.id, + ) + .join(StoryArc, IssueStoryArc.story_arc_id == StoryArc.id) + .join(LibraryFile, StoryArcSyncWork.library_file_id == LibraryFile.id) + .where( + StoryArcSyncWork.origin_import_job_id == job_id, + StoryArcSyncWork.state.in_(terminal_states), + ) + .order_by(StoryArcSyncWork.id.asc()) + ) + ).all() + if len(terminal_rows) != retrying_count: + raise ValidationError("Import Story Arc placement origin evidence is incomplete.") + + work_ids: list[int] = [] + for work, membership, story_arc, library_file in terminal_rows: + desired_generation, source_signature_hash = _desired_generation( + library_file, + membership, + story_arc, + ) + if ( + not work.claimable + or work.claim_token is not None + or work.claimed_at is not None + or work.next_attempt_at is not None + or work.cancel_requested_at is not None + or not _import_managed_policy_configured(story_arc) + or work.desired_generation != desired_generation + or work.source_signature_hash != source_signature_hash + or work.story_arc_revision != story_arc.revision + or work.membership_sequence != membership.sequence_number + or work.policy_schema_version + != (story_arc.policy_schema_version or STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION) + ): + raise ValidationError( + "Import Story Arc placement work is no longer exact and cannot be retried." + ) + work_ids.append(int(work.id)) + + result = cast( + "CursorResult[Any]", + await session.execute( + update(StoryArcSyncWork) + .where( + StoryArcSyncWork.id.in_(work_ids), + StoryArcSyncWork.origin_import_job_id == job_id, + StoryArcSyncWork.state.in_(terminal_states), + StoryArcSyncWork.claimable.is_(True), + StoryArcSyncWork.claim_token.is_(None), + StoryArcSyncWork.claimed_at.is_(None), + StoryArcSyncWork.cancel_requested_at.is_(None), + ) + .values( + state=StoryArcSyncWorkState.QUEUED, + attempt_count=0, + next_attempt_at=None, + claim_token=None, + claimed_at=None, + cancel_requested_at=None, + last_error_code=None, + last_error_category=None, + last_error_detail=None, + last_result={}, + ) + ), + ) + if result.rowcount != retrying_count: + raise ValidationError("Story Arc placement work changed concurrently; retry refused.") + + updated_snapshot = dict(snapshot) + updated_snapshot.update( + { + "status": ImportJobStatus.IMPORTING.value, + "mode": "import", + "phase": _IMPORT_PLACEMENT_PHASE, + "progress": 99, + "message": "Creating the approved story-arc copies and links...", + "story_arc_placements_total": counts.total, + "story_arc_placements_queued": counts.queued + retrying_count, + "story_arc_placements_running": counts.running, + "story_arc_placements_retry_wait": counts.retry_wait, + "story_arc_placements_failed": 0, + "story_arc_placements_completed": counts.completed, + "story_arc_placements_cancelled": 0, + "story_arc_placement_followup_pending": False, + } + ) + job.status = ImportJobStatus.IMPORTING + job.error_message = None + job.import_completed_at = None + job.story_arc_placement_followup_pending = False + job.progress_snapshot = updated_snapshot + job.progress_revision = int(job.progress_revision or 0) + 1 + await session.flush() + return job, retrying_count + + +async def _next_retry_at( + session: AsyncSession, + *, + import_only: bool = False, +) -> datetime | None: + origin_scope = (StoryArcSyncWork.origin_import_job_id.is_not(None),) if import_only else () + return await session.scalar( + select(func.min(StoryArcSyncWork.next_attempt_at)).where( + *origin_scope, + _origin_is_not_startup_recovery_paused(), + StoryArcSyncWork.claimable.is_(True), + StoryArcSyncWork.state == StoryArcSyncWorkState.RETRY_WAIT, + ) + ) + + +async def _has_ready_work( + session: AsyncSession, + *, + now: datetime, + import_only: bool = False, +) -> bool: + return bool( + await _ready_work_ids( + session, + now=now, + limit=1, + import_only=import_only, + ) + ) + + +async def _origin_import_job_ids_for_work_ids( + session: AsyncSession, + work_ids: list[int], +) -> tuple[int, ...]: + """Return only the import jobs touched by this bounded worker drain.""" + if not work_ids: + return () + return tuple( + int(job_id) + for job_id in ( + await session.scalars( + select(StoryArcSyncWork.origin_import_job_id) + .where( + StoryArcSyncWork.id.in_(work_ids), + StoryArcSyncWork.origin_import_job_id.is_not(None), + ) + .distinct() + .order_by(StoryArcSyncWork.origin_import_job_id.asc()) + ) + ).all() + if job_id is not None + ) + + +def _waiting_import_story_arc_finalizer_predicate() -> Any: + """Build the shared eligibility fence for bounded placement finalization.""" + pending_origin_work = exists().where( + StoryArcSyncWork.origin_import_job_id == ImportJob.id, + StoryArcSyncWork.state.in_(_PENDING_IMPORT_WORK_STATES), + ) + return or_( + and_( + ImportJob.status.in_( + { + ImportJobStatus.IMPORTING, + ImportJobStatus.STALLED, + } + ), + ImportJob.control_request == ImportControlRequest.NONE, + ImportJob.progress_snapshot["phase"].as_string() == _IMPORT_PLACEMENT_PHASE, + ~pending_origin_work, + ), + and_( + ImportJob.status == ImportJobStatus.COMPLETED, + ImportJob.story_arc_placement_followup_pending.is_(True), + ), + ) + + +async def _finalize_waiting_import_story_arc_placements( + session_factory: async_sessionmaker[AsyncSession], + *, + candidate_job_ids: tuple[int, ...] | None = None, +) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + """Finalize terminal touched jobs, or perform one bounded recovery sweep.""" + normalized_candidates = tuple( + dict.fromkeys(job_id for job_id in (candidate_job_ids or ()) if job_id > 0) + )[:MAX_IMPORT_PLACEMENT_FINALIZE_BATCH_SIZE] + + async with session_factory() as list_session: + eligible = _waiting_import_story_arc_finalizer_predicate() + touched_limit = MAX_IMPORT_PLACEMENT_FINALIZE_BATCH_SIZE + if normalized_candidates and touched_limit > 1: + touched_limit -= 1 + touched_rows: tuple[tuple[int, ImportJobStatus], ...] = () + if normalized_candidates and touched_limit: + touched_rows = tuple( + (int(job_id), status) + for job_id, status in ( + await list_session.execute( + select(ImportJob.id, ImportJob.status) + .where( + ImportJob.id.in_(normalized_candidates), + eligible, + ) + .order_by(ImportJob.id.asc()) + .limit(touched_limit) + ) + ).all() + ) + recovery_limit = MAX_IMPORT_PLACEMENT_FINALIZE_BATCH_SIZE - len(touched_rows) + recovery_statement = select(ImportJob.id, ImportJob.status).where(eligible) + if normalized_candidates: + recovery_statement = recovery_statement.where( + ImportJob.id.not_in(normalized_candidates) + ) + recovery_rows = tuple( + (int(job_id), status) + for job_id, status in ( + await list_session.execute( + recovery_statement.order_by(ImportJob.id.asc()).limit(recovery_limit) + ) + ).all() + ) + candidates = (*touched_rows, *recovery_rows) + await list_session.rollback() + + evaluated: list[int] = [] + completed: list[int] = [] + stalled: list[int] = [] + for job_id, status in candidates: + if status is ImportJobStatus.COMPLETED: + evaluated.append(job_id) + completed.append(job_id) + continue + async with session_factory() as finalizer_session: + try: + outcome = await finalize_import_story_arc_placements( + finalizer_session, + job_id, + ) + await finalizer_session.commit() + except (NotFoundError, ValidationError): + await finalizer_session.rollback() + continue + evaluated.append(job_id) + if outcome.state is ImportStoryArcPlacementCompletionState.COMPLETED: + completed.append(job_id) + elif outcome.state is ImportStoryArcPlacementCompletionState.STALLED: + stalled.append(job_id) + return tuple(evaluated), tuple(completed), tuple(stalled) + + +async def _has_remaining_import_story_arc_finalizers( + session_factory: async_sessionmaker[AsyncSession], + *, + exclude_job_ids: tuple[int, ...], +) -> bool: + """Return whether bounded finalization left another eligible job behind.""" + async with session_factory() as session: + eligible = _waiting_import_story_arc_finalizer_predicate() + statement = select(ImportJob.id).where(eligible) + if exclude_job_ids: + statement = statement.where(ImportJob.id.not_in(exclude_job_ids)) + remaining_job_id = await session.scalar(statement.order_by(ImportJob.id.asc()).limit(1)) + await session.rollback() + return remaining_job_id is not None + + +async def _ready_import_story_arc_rollbacks( + session_factory: async_sessionmaker[AsyncSession], +) -> tuple[int, ...]: + """Return bounded deferred rollbacks whose in-flight work is now fenced.""" + async with session_factory() as session: + ready = tuple( + int(job_id) + for job_id in ( + await session.scalars( + select(ImportJob.id) + .outerjoin( + StoryArcSyncWork, + StoryArcSyncWork.id == ImportJob.story_arc_rollback_waiting_work_id, + ) + .where( + ImportJob.status == ImportJobStatus.ROLLING_BACK, + ImportJob.story_arc_rollback_waiting_work_id.is_not(None), + or_( + StoryArcSyncWork.id.is_(None), + StoryArcSyncWork.state != StoryArcSyncWorkState.RUNNING, + ), + ) + .order_by(ImportJob.id.asc()) + .limit(MAX_IMPORT_PLACEMENT_FINALIZE_BATCH_SIZE) + ) + ).all() + ) + await session.rollback() + return ready + + +async def process_story_arc_sync_work( + *, + session_factory: async_sessionmaker[AsyncSession] | None = None, + sync_service: StoryArcPlacementSyncService | Any | None = None, + batch_size: int = DEFAULT_STORY_ARC_SYNC_BATCH_SIZE, + discover: bool = True, + import_only: bool = False, + now_fn: Callable[[], datetime] | None = None, + heartbeat_interval_seconds: float = _CLAIM_HEARTBEAT_INTERVAL_SECONDS, + heartbeat_now_fn: Callable[[], datetime] | None = None, + origin_cancellation_poll_seconds: float = _ORIGIN_CANCELLATION_POLL_SECONDS, +) -> StoryArcSyncDrainResult: + """Discover and process one bounded batch with a fresh session per phase. + + Import-only drains are safe to run under the global import scheduler fence: + they skip discovery and ignore every ordinary synchronization lane. + """ + if isinstance(batch_size, bool) or not 1 <= batch_size <= MAX_STORY_ARC_SYNC_BATCH_SIZE: + raise ValueError( + f"Story-arc sync batch size must be from 1 to {MAX_STORY_ARC_SYNC_BATCH_SIZE}" + ) + if ( + isinstance(heartbeat_interval_seconds, bool) + or heartbeat_interval_seconds <= 0 + or not math.isfinite(heartbeat_interval_seconds) + ): + raise ValueError("Story-arc sync heartbeat interval must be a positive finite number") + if ( + isinstance(origin_cancellation_poll_seconds, bool) + or origin_cancellation_poll_seconds <= 0 + or not math.isfinite(origin_cancellation_poll_seconds) + ): + raise ValueError( + "Story-arc import cancellation poll interval must be a positive finite number" + ) + factory = session_factory or get_session_factory() + service = sync_service or StoryArcPlacementSyncService() + effective_now_fn = now_fn or (lambda: datetime.now(UTC)) + effective_heartbeat_now_fn = heartbeat_now_fn or effective_now_fn + discovered = 0 + if discover and not import_only: + async with factory() as discovery_session: + discovered = await discover_story_arc_sync_work(discovery_session) + await discovery_session.commit() + + async with factory() as list_session: + work_ids = await _ready_work_ids( + list_session, + now=effective_now_fn(), + limit=batch_size, + import_only=import_only, + ) + await list_session.rollback() + + claimed = completed = failed = retrying = cancelled = lost_claims = 0 + for work_id in work_ids: + item_now = effective_now_fn() + async with factory() as claim_session: + claim_token = await claim_story_arc_sync_work( + claim_session, + work_id, + now=item_now, + import_only=import_only, + ) + if claim_token is None: + continue + claimed += 1 + + async with factory() as context_session: + context = await _load_claimed_context(context_session, work_id, claim_token) + await context_session.rollback() + if context is None: + async with factory() as result_session: + cancelled_claim = await _cancel_work( + result_session, + work_id, + claim_token, + code="sync_work_superseded", + ) + if cancelled_claim: + cancelled += 1 + else: + lost_claims += 1 + continue + + heartbeat_stop = asyncio.Event() + origin_cancellation_stop = asyncio.Event() + origin_cancellation_requested = asyncio.Event() + heartbeat = asyncio.create_task( + _maintain_claim_lease( + factory, + work_id, + claim_token, + heartbeat_stop, + interval_seconds=heartbeat_interval_seconds, + now_fn=effective_heartbeat_now_fn, + ) + ) + origin_cancellation_monitor = ( + asyncio.create_task( + _monitor_origin_cancellation( + factory, + work_id, + claim_token, + origin_cancellation_stop, + origin_cancellation_requested, + interval_seconds=origin_cancellation_poll_seconds, + ) + ) + if context.import_provenance is not None + else None + ) + try: + try: + async with factory() as sync_session: + if context.import_provenance is None: + result = await service.sync_membership( + sync_session, + context.story_arc_id, + context.membership_id, + ) + else: + result = await service.sync_membership( + sync_session, + context.story_arc_id, + context.membership_id, + import_provenance=context.import_provenance, + cancellation_requested=origin_cancellation_requested.is_set, + ) + finally: + heartbeat_stop.set() + origin_cancellation_stop.set() + await heartbeat + if origin_cancellation_monitor is not None: + await origin_cancellation_monitor + async with factory() as result_session: + finished_claim = await _finish_work( + result_session, + context, + claim_token, + outcome=result.outcome, + ) + if finished_claim: + completed += 1 + else: + lost_claims += 1 + except asyncio.CancelledError: + heartbeat_stop.set() + if not heartbeat.done(): + await heartbeat + async with factory() as result_session: + cancelled_state = await _fail_or_retry_work( + result_session, + context, + claim_token, + now=effective_now_fn(), + code="sync_worker_cancelled", + category="cancelled", + detail="Automatic story-arc synchronization was interrupted.", + retryable=True, + ) + if cancelled_state is None: + logger.warning( + "story_arc_sync_claim_lost_during_cancellation", + work_id=context.work_id, + issue_story_arc_id=context.membership_id, + ) + raise + except StoryArcPlacementIntegrationError as exc: + if context.import_provenance is not None and ( + origin_cancellation_requested.is_set() or exc.category == "cancelled" + ): + async with factory() as result_session: + cancelled_claim = await _cancel_work( + result_session, + context.work_id, + claim_token, + code=exc.code, + ) + if cancelled_claim: + cancelled += 1 + else: + lost_claims += 1 + continue + async with factory() as result_session: + state = await _fail_or_retry_work( + result_session, + context, + claim_token, + now=effective_now_fn(), + code=exc.code, + category=exc.category, + detail=str(exc), + retryable=_is_retryable(exc), + ) + if state is None: + lost_claims += 1 + elif state is StoryArcSyncWorkState.RETRY_WAIT: + retrying += 1 + else: + failed += 1 + logger.warning( + "story_arc_sync_item_failed", + work_id=context.work_id, + issue_story_arc_id=context.membership_id, + error_code=exc.code, + error_category=exc.category, + claim_lost=state is None, + retrying=state is StoryArcSyncWorkState.RETRY_WAIT, + ) + except Exception: + async with factory() as result_session: + state = await _fail_or_retry_work( + result_session, + context, + claim_token, + now=effective_now_fn(), + code="story_arc_sync_unexpected_failure", + category="operation", + detail="Automatic story-arc synchronization failed unexpectedly.", + retryable=True, + ) + if state is None: + lost_claims += 1 + elif state is StoryArcSyncWorkState.RETRY_WAIT: + retrying += 1 + else: + failed += 1 + logger.exception( + "story_arc_sync_item_failed_unexpectedly", + work_id=context.work_id, + issue_story_arc_id=context.membership_id, + claim_lost=state is None, + retrying=state is StoryArcSyncWorkState.RETRY_WAIT, + ) + + async with factory() as summary_session: + touched_import_job_ids = await _origin_import_job_ids_for_work_ids( + summary_session, + work_ids, + ) + has_more = await _has_ready_work( + summary_session, + now=effective_now_fn(), + import_only=import_only, + ) + next_retry_at = await _next_retry_at( + summary_session, + import_only=import_only, + ) + await summary_session.rollback() + ( + import_jobs_evaluated, + import_jobs_completed, + import_jobs_stalled, + ) = await _finalize_waiting_import_story_arc_placements( + factory, + candidate_job_ids=(touched_import_job_ids if work_ids else None), + ) + finalizer_has_more = await _has_remaining_import_story_arc_finalizers( + factory, + exclude_job_ids=import_jobs_evaluated, + ) + has_more = has_more or finalizer_has_more + import_jobs_rollback_ready = await _ready_import_story_arc_rollbacks(factory) + return StoryArcSyncDrainResult( + discovered=discovered, + claimed=claimed, + completed=completed, + failed=failed, + retrying=retrying, + cancelled=cancelled, + lost_claims=lost_claims, + has_more=has_more, + next_retry_at=next_retry_at, + import_jobs_evaluated=import_jobs_evaluated, + import_jobs_completed=import_jobs_completed, + import_jobs_stalled=import_jobs_stalled, + import_jobs_rollback_ready=import_jobs_rollback_ready, + ) + + +def request_story_arc_sync_now() -> None: + """Best-effort latency nudge; the durable scheduled sweep remains authoritative.""" + from pullbox.core.scheduler import get_scheduler + + try: + status = get_scheduler().run_task_now(STORY_ARC_SYNC_TASK_ID) + if status == "queued": + logger.debug("story_arc_sync_triggered_after_registration") + except Exception: + logger.warning("story_arc_sync_trigger_failed", exc_info=True) diff --git a/src/pullbox/services/wanted_search_sweep.py b/src/pullbox/services/wanted_search_sweep.py index 999a532b..97a8da41 100644 --- a/src/pullbox/services/wanted_search_sweep.py +++ b/src/pullbox/services/wanted_search_sweep.py @@ -10,13 +10,14 @@ from sqlalchemy import and_, case, exists, func, select from pullbox.models.config import SystemConfig -from pullbox.models.issue import Issue, IssueStatus +from pullbox.models.issue import Issue from pullbox.models.pending_match import PendingMatch, PendingMatchStatus from pullbox.models.search_log import SearchLog from pullbox.models.series import Series from pullbox.services.search_targets import ( IssueSearchTarget, load_wanted_issue_search_targets_by_ids, + wanted_issue_eligibility_filter, ) if TYPE_CHECKING: @@ -280,8 +281,7 @@ async def _load_fair_wanted_issue_ids(session: AsyncSession) -> list[int]: select(Issue.id) .join(Series, Series.id == Issue.series_id) .where( - Issue.status == IssueStatus.WANTED, - Series.monitored.is_(True), + wanted_issue_eligibility_filter(), ~exists().where( and_( PendingMatch.issue_id == Issue.id, diff --git a/src/pullbox/tasks/__init__.py b/src/pullbox/tasks/__init__.py index 246748a0..86cc5373 100644 --- a/src/pullbox/tasks/__init__.py +++ b/src/pullbox/tasks/__init__.py @@ -6,6 +6,7 @@ from pullbox.tasks import backup_task as backup_task from pullbox.tasks import blocklist_task as blocklist_task +from pullbox.tasks import catalog_task as catalog_task from pullbox.tasks import cover_backfill_task as cover_backfill_task from pullbox.tasks import dashboard_task as dashboard_task from pullbox.tasks import database_maintenance_task as database_maintenance_task @@ -36,6 +37,8 @@ from pullbox.tasks import scan_task as scan_task from pullbox.tasks import search_scheduler_task as search_scheduler_task from pullbox.tasks import search_task as search_task +from pullbox.tasks import story_arc_metadata_task as story_arc_metadata_task +from pullbox.tasks import story_arc_sync_task as story_arc_sync_task from pullbox.tasks import update_check_task as update_check_task from pullbox.tasks import usage_stats_task as usage_stats_task from pullbox.tasks import whats_new_task as whats_new_task diff --git a/src/pullbox/tasks/catalog_task.py b/src/pullbox/tasks/catalog_task.py new file mode 100644 index 00000000..74dfafde --- /dev/null +++ b/src/pullbox/tasks/catalog_task.py @@ -0,0 +1,19 @@ +"""Catalog update scheduler integration.""" + +from pullbox.core.scheduler import get_current_task_trigger_type, scheduled_task + + +@scheduled_task( + task_id="catalog_update", + display_name="Local Catalog Update", + trigger="cron", + hour=6, + minute=30, + jitter=1800, + misfire_grace_time=3600, +) +async def update_catalog() -> None: + """Check daily after opt-in; manual runs also allow the first download.""" + from pullbox.services.catalog.service import get_catalog_service + + await get_catalog_service().sync(manual=get_current_task_trigger_type() == "manual") diff --git a/src/pullbox/tasks/download_post_processing_destination.py b/src/pullbox/tasks/download_post_processing_destination.py index 6539d34f..6fea388b 100644 --- a/src/pullbox/tasks/download_post_processing_destination.py +++ b/src/pullbox/tasks/download_post_processing_destination.py @@ -8,7 +8,8 @@ from pathlib import Path from typing import Any -from pullbox.models.library import MatchConfidence +from pullbox.core.library_root_resolution import preferred_managed_root_id +from pullbox.models.library import LibraryFileStorageMode, MatchConfidence from pullbox.tasks.post_processing_progress import PostProcessingPhase @@ -50,11 +51,12 @@ async def build_destination_plan( # on the later recovery path to find an alternate existing extension. extension = comic_file.suffix.lstrip(".").lower() if comic_file else "cbz" destination_probe = comic_file or Path(f"{series.title} #probe.{extension}") + destination_root_id = preferred_managed_root_id(series) dest_path, _resolved_root = await resolve_library_destination( session, destination_probe, issue, - library_root_id=series.library_root_id, + library_root_id=destination_root_id, ) dest_dir = dest_path.parent issue_filename = dest_path.name @@ -75,7 +77,7 @@ async def build_destination_plan( "post_processing_transfer_plan", source=str(comic_file), destination=str(dest_path), - library_root_id=series.library_root_id, + library_root_id=destination_root_id, ) return DestinationPlan( @@ -159,8 +161,10 @@ async def register_existing_destination_file( issue, MatchConfidence.HIGH, move_to_library=False, + storage_mode=LibraryFileStorageMode.MANAGED, + recover_existing_managed_artifact=True, rename=False, - library_root_id=series.library_root_id, + library_root_id=preferred_managed_root_id(series), ) trace.finalize_current_phase() log.info( diff --git a/src/pullbox/tasks/download_post_processing_queue.py b/src/pullbox/tasks/download_post_processing_queue.py index 9b242803..3ce8cede 100644 --- a/src/pullbox/tasks/download_post_processing_queue.py +++ b/src/pullbox/tasks/download_post_processing_queue.py @@ -19,6 +19,7 @@ from pullbox.services.post_processing_operation_progress import ( project_post_processing_operation_progress, ) +from pullbox.services.story_arc_sync_queue import request_story_arc_sync_now from pullbox.tasks.post_processing_progress import ( PostProcessingPhase, _clear_post_processing, @@ -193,6 +194,16 @@ async def process_completed( ) await session.commit() + try: + request_story_arc_sync_now() + except Exception: + # Canonical completion is already durable. This + # latency-only nudge must never enter failure repair. + log.warning( + "story_arc_sync_trigger_failed_after_completion", + download_id=dl_id, + exc_info=True, + ) except Exception as exc: failed_duration_ms = ( round((_time.monotonic() - handoff_start) * 1000, 1) diff --git a/src/pullbox/tasks/download_post_processing_transfer.py b/src/pullbox/tasks/download_post_processing_transfer.py index f7dfe1fc..7c2e8685 100644 --- a/src/pullbox/tasks/download_post_processing_transfer.py +++ b/src/pullbox/tasks/download_post_processing_transfer.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any +from pullbox.core.library_root_resolution import preferred_managed_root_id from pullbox.models.library import MatchConfidence from pullbox.tasks.post_processing_progress import PostProcessingPhase @@ -59,7 +60,7 @@ def _on_transfer_progress(done: int, total: int) -> None: issue, MatchConfidence.HIGH, move_to_library=True, - library_root_id=series.library_root_id, + library_root_id=preferred_managed_root_id(series), transfer_progress_callback=_on_transfer_progress, download_client=download.download_client, replace_existing_library_file=bool(getattr(download, "replace_existing_file", False)), diff --git a/src/pullbox/tasks/download_task.py b/src/pullbox/tasks/download_task.py index 1474d44e..1ec3e495 100644 --- a/src/pullbox/tasks/download_task.py +++ b/src/pullbox/tasks/download_task.py @@ -21,6 +21,7 @@ from pullbox.composition.events import build_domain_event_bus from pullbox.composition.providers import register_download_clients +from pullbox.core.sqlite_lock import run_sqlite_transaction_with_retry from pullbox.database import get_session_factory from pullbox.models.download import DownloadClientType from pullbox.models.issue import Issue, IssueStatus @@ -336,11 +337,12 @@ async def monitor_downloads() -> None: event_logger=logger, ) - # ── Phase 3: Write — apply updates and run throttled recovery checks ── - async with factory() as session: - try: - if updates: - apply_result = await _apply_monitor_updates( + # ── Phase 3: Write — short, independently retryable DB transactions ── + if updates: + async with factory() as session: + + async def apply_updates() -> _download_monitor_apply.MonitorApplyResult: + return await _apply_monitor_updates( session, updates, first_active_observed_at=_first_active_observed_at, @@ -350,19 +352,32 @@ async def monitor_downloads() -> None: event_logger=logger, publish_progress=project_download_operation_progress, ) - completed = apply_result.completed - failed = apply_result.failed - # Throttle expensive recovery checks to ~every 30s - if recovery_due: - retried += await _process_retry_pending(factory, download_svc) - recovered = await _recover_orphaned_downloads(session) - _last_recovery_check = recovery_checked_at + apply_result = await run_sqlite_transaction_with_retry( + session, + apply_updates, + event_name="download_monitor_write", + logger=logger, + ) + completed = apply_result.completed + failed = apply_result.failed - await session.commit() - except Exception: - await session.rollback() - raise + # Throttle expensive recovery checks to ~every 30s. Network retries run + # outside this transaction; only the local orphan-recovery write is retried. + if recovery_due: + retried += await _process_retry_pending(factory, download_svc) + async with factory() as session: + + async def recover_orphans() -> int: + return await _recover_orphaned_downloads(session) + + recovered = await run_sqlite_transaction_with_retry( + session, + recover_orphans, + event_name="download_orphan_recovery", + logger=logger, + ) + _last_recovery_check = recovery_checked_at duration_ms = (time.monotonic() - start) * 1000 log_kwargs = { diff --git a/src/pullbox/tasks/import_orphan_recovery_task.py b/src/pullbox/tasks/import_orphan_recovery_task.py index aa52b44b..44e5491f 100644 --- a/src/pullbox/tasks/import_orphan_recovery_task.py +++ b/src/pullbox/tasks/import_orphan_recovery_task.py @@ -130,7 +130,7 @@ async def progress_callback( current=current, total=total, ) - _set_orphan_recovery_state( + progress = _set_orphan_recovery_state( imported_series_id, state="running", message=( @@ -147,12 +147,14 @@ async def progress_callback( file_index=file_index, total_files=total_files, ) + await _queue_orphan_recovery_progress(progress) - completed = _set_orphan_recovery_state( + preparing = _set_orphan_recovery_state( imported_series_id, state="running", message=f"Preparing recovery for {item.cv_title or item.raw_series_name}...", ) + await _queue_orphan_recovery_progress(preparing) payload = await service.recover_orphan( session, imported_series_id, @@ -160,7 +162,7 @@ async def progress_callback( progress_callback=progress_callback, ) await session.commit() - _set_orphan_recovery_state( + completed = _set_orphan_recovery_state( imported_series_id, state="completed", message=( diff --git a/src/pullbox/tasks/import_task.py b/src/pullbox/tasks/import_task.py index a15e160c..cbdcade6 100644 --- a/src/pullbox/tasks/import_task.py +++ b/src/pullbox/tasks/import_task.py @@ -26,6 +26,15 @@ ImportSeriesStatus, ) from pullbox.schemas.import_job import ImportProgressEvent +from pullbox.services.import_counters import job_stats +from pullbox.services.import_deferred_recovery_execution import ( + cancel_deferred_preparation, + prepare_deferred_recovery, +) +from pullbox.services.import_job_execution_progress import ( + reconcile_durable_import_execution_counters, +) +from pullbox.services.import_library_adoption import prepare_clean_library_import from pullbox.services.import_workflow_state import ( emit_progress, import_control_state_for_job, @@ -45,6 +54,7 @@ logger = structlog.get_logger(__name__) _COMPAT_QUEUE_MAXSIZE = 200 +_BULK_SAFETY_REMATCH_PAGE_SIZE = 25 _import_progress_queues: dict[int, asyncio.Queue[ImportProgressEvent]] = {} _latest_import_progress_events: dict[int, ImportProgressEvent] = {} @@ -84,6 +94,7 @@ } ) _STARTUP_RECOVERY_PAUSE_REASON = "startup_recovery" +_STARTUP_RECOVERY_DISPATCH_PENDING = "startup_recovery_dispatch_pending" _RECOVERED_SCAN_PHASE_BY_STATUS = { ImportJobStatus.SCANNING: "scanning", ImportJobStatus.ANALYZING: "analyzing", @@ -196,6 +207,8 @@ async def _emit_terminal_event_for_job( job = await session.get(ImportJob, job_id) if job is None: return None + if job.status not in _TERMINAL_STATES: + return await _publish_current_snapshot_event_for_job(session, job_id) terminal_mode = snapshot_mode_for_job(job) terminal_phase = "done" @@ -245,25 +258,16 @@ async def _emit_terminal_event_for_job( "error_message": job.error_message, "control_state": import_control_state_for_job(job), } + payload.update(job_stats(job)) snapshot = dict(job.progress_snapshot or {}) for field_name in ( - "scan_total_files", - "scan_total_dirs", - "series_found", - "series_duplicate", - "series_matched", - "series_no_match", - "series_new", - "series_imported", - "series_failed", - "total_files_found", - "total_files_matched", - "total_files_duplicate", - "total_files_already_owned", - "total_files_conflict", - "total_files_no_match", - "total_files_imported", - "total_files_failed", + "story_arc_placements_total", + "story_arc_placements_queued", + "story_arc_placements_running", + "story_arc_placements_retry_wait", + "story_arc_placements_failed", + "story_arc_placements_completed", + "story_arc_placements_cancelled", "review_summary", "scan_started_at", "import_started_at", @@ -329,6 +333,13 @@ async def _publish_current_snapshot_event_for_job( "current_file_progress_unit": snapshot.get("current_file_progress_unit"), "current_series": snapshot.get("current_series") or snapshot.get("current_series_name"), "estimated_seconds_remaining": snapshot.get("estimated_seconds_remaining"), + "story_arc_placements_total": snapshot.get("story_arc_placements_total"), + "story_arc_placements_queued": snapshot.get("story_arc_placements_queued"), + "story_arc_placements_running": snapshot.get("story_arc_placements_running"), + "story_arc_placements_retry_wait": snapshot.get("story_arc_placements_retry_wait"), + "story_arc_placements_failed": snapshot.get("story_arc_placements_failed"), + "story_arc_placements_completed": snapshot.get("story_arc_placements_completed"), + "story_arc_placements_cancelled": snapshot.get("story_arc_placements_cancelled"), "error_message": job.error_message, "control_state": import_control_state_for_job(job), } @@ -354,6 +365,60 @@ def _should_schedule_comicinfo_enrichment(result: Any) -> bool: return getattr(result, "schedule_comicinfo_enrichment", False) is True +def _should_schedule_story_arc_sync(result: Any) -> bool: + """Return True only for a committed import-owned placement batch.""" + return getattr(result, "schedule_story_arc_sync", False) is True + + +def _rollback_waits_for_story_arc(result: Any) -> bool: + """Return True only for the explicit cooperative rollback deferral signal.""" + return result is False + + +async def publish_story_arc_import_updates( + job_ids: tuple[int, ...], + *, + completed_job_ids: tuple[int, ...] = (), + session_factory: async_sessionmaker[AsyncSession] | None = None, +) -> None: + """Publish reconciled import state and start post-commit completion work.""" + factory = session_factory or get_session_factory() + completed = frozenset(completed_job_ids) + completed_for_dispatch = False + for job_id in dict.fromkeys(job_ids): + async with factory() as session: + job = await session.get(ImportJob, job_id) + if job is None: + continue + service: Any | None = None + followup_pending = bool( + job_id in completed + and job.status is ImportJobStatus.COMPLETED + and job.story_arc_placement_followup_pending + ) + if followup_pending: + service = await _build_import_service(session) + await session.commit() + service.schedule_comicinfo_enrichment(factory, job_id=job_id) + + terminal_status = await _emit_terminal_event_for_job(session, job_id) + if followup_pending and terminal_status is ImportJobStatus.COMPLETED: + refreshed = await session.get(ImportJob, job_id) + if refreshed is not None: + snapshot = dict(refreshed.progress_snapshot or {}) + snapshot["story_arc_placement_followup_pending"] = False + refreshed.progress_snapshot = snapshot + refreshed.story_arc_placement_followup_pending = False + await session.commit() + completed_for_dispatch = completed_for_dispatch or ( + terminal_status is ImportJobStatus.COMPLETED + ) + if terminal_status in _TERMINAL_STATES: + purge_import_runtime_state(job_id) + if completed_for_dispatch and _import_runner is not None: + await _import_runner.request_recovered_dispatch() + + class ImportRunner: """Single-import durable runner with startup recovery and broadcast progress.""" @@ -362,6 +427,7 @@ def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: self._lock = asyncio.Lock() self._worker_task: asyncio.Task[None] | None = None self._active_job_id: int | None = None + self._dispatch_recovered_requested = False async def recover_and_dispatch(self) -> int: """Recover interrupted imports and resume any runnable job.""" @@ -385,40 +451,66 @@ async def request_rollback(self, job_id: int) -> None: """Run a rollback for the given import job.""" await self._start_if_idle(job_id) + async def request_recovered_dispatch(self) -> None: + """Resume the next startup-recovered job once the runner is idle.""" + await self._dispatch_recovered_job() + async def _dispatch_recovered_job(self) -> None: + async with self._lock: + if self._worker_task is not None and not self._worker_task.done(): + self._dispatch_recovered_requested = True + return + self._dispatch_recovered_requested = False + job_id = await self._resume_next_recovered_job() + if job_id is not None: + self._start_worker_locked(job_id) + + async def _resume_next_recovered_job(self) -> int | None: + """Restore and return the oldest runnable startup-recovered job.""" async with self._session_factory() as session: + stalled_job_id = await session.scalar( + sa_select(ImportJob.id) + .where(ImportJob.status == ImportJobStatus.STALLED) + .order_by(ImportJob.created_at.asc(), ImportJob.id.asc()) + .limit(1) + ) + if stalled_job_id is not None: + logger.info( + "import_recovered_dispatch_blocked_by_stalled_job", + job_id=int(stalled_job_id), + ) + return None result = await session.execute( sa_select(ImportJob) .where( ImportJob.status.in_( { - ImportJobStatus.SCANNING, - ImportJobStatus.ANALYZING, - ImportJobStatus.MATCHING, - ImportJobStatus.FILE_MATCHING, - ImportJobStatus.IMPORTING, ImportJobStatus.ROLLING_BACK, ImportJobStatus.PAUSED, } ) ) - .order_by(ImportJob.created_at.asc()) + .order_by(ImportJob.created_at.asc(), ImportJob.id.asc()) ) - job_id: int | None = None for job in result.scalars().all(): if job.status == ImportJobStatus.PAUSED: if not _is_startup_recovered_pause(job): continue _resume_startup_recovered_job(job) - job_id = job.id await session.commit() - logger.info("import_recovered_job_auto_resumed", job_id=job_id) - break - - job_id = job.id - break - if job_id is not None: - await self._start_if_idle(job_id) + logger.info("import_recovered_job_auto_resumed", job_id=job.id) + return int(job.id) + if not _is_startup_recovered_rollback(job): + continue + snapshot = dict(job.progress_snapshot or {}) + snapshot.pop(_STARTUP_RECOVERY_DISPATCH_PENDING, None) + snapshot.pop("recovered_status", None) + snapshot.pop("recovered_at", None) + job.progress_snapshot = snapshot + await session.commit() + logger.info("import_recovered_rollback_auto_resumed", job_id=job.id) + return int(job.id) + return None async def _start_if_idle(self, job_id: int) -> None: async with self._lock: @@ -431,9 +523,28 @@ async def _start_if_idle(self, job_id: int) -> None: ) return - self._active_job_id = job_id - self._worker_task = asyncio.create_task(self._run_job(job_id)) - self._worker_task.add_done_callback(self._on_worker_done) + self._start_worker_locked(job_id) + + def _start_worker_locked(self, job_id: int) -> None: + """Start one serial worker while the runner lock is held.""" + self._active_job_id = job_id + self._worker_task = asyncio.create_task(self._run_job_sequence(job_id)) + self._worker_task.add_done_callback(self._on_worker_done) + + async def _run_job_sequence(self, job_id: int) -> None: + """Drain recovered jobs serially while each predecessor reaches an idle state.""" + current_job_id: int | None = job_id + while current_job_id is not None: + self._active_job_id = current_job_id + await self._run_job(current_job_id) + if not await self._job_is_terminal_or_deleted(current_job_id): + return + current_job_id = await self._resume_next_recovered_job() + + async def _job_is_terminal_or_deleted(self, job_id: int) -> bool: + async with self._session_factory() as session: + job = await session.get(ImportJob, job_id) + return job is None or job.status in _TERMINAL_STATES def _on_worker_done(self, task: asyncio.Task[None]) -> None: try: @@ -441,8 +552,12 @@ def _on_worker_done(self, task: asyncio.Task[None]) -> None: except Exception: logger.exception("import_runner_worker_failed") finally: - self._worker_task = None - self._active_job_id = None + if self._worker_task is task: + self._worker_task = None + self._active_job_id = None + if self._dispatch_recovered_requested: + self._dispatch_recovered_requested = False + _fire_and_forget(self._dispatch_recovered_job()) async def _mark_paused(self, session: AsyncSession, job_id: int) -> None: job = await session.get(ImportJob, job_id) @@ -462,6 +577,9 @@ async def _finalize_cancel(self, session: AsyncSession, job_id: int) -> None: job = await session.get(ImportJob, job_id) if job is None: return + if await cancel_deferred_preparation(session, job): + purge_import_runtime_state(job_id) + return if job.import_started_at is None: await session.delete(job) await session.commit() @@ -585,24 +703,42 @@ async def progress_callback(event: ImportProgressEvent) -> None: ) await session.commit() elif job.status == ImportJobStatus.IMPORTING: + if dict(job.progress_snapshot or {}).get("deferred_recovery"): + await prepare_deferred_recovery( + session, + job_id, + metadata_service=service._metadata_service, + progress_callback=progress_callback, + ) + if job.status != ImportJobStatus.IMPORTING: + return + await prepare_clean_library_import( + session, + job_id, + progress_callback=progress_callback, + ) result = await service.run_import( session, job_id, progress_callback=progress_callback, ) await session.commit() + if _should_schedule_story_arc_sync(result): + service.schedule_story_arc_sync() if _should_schedule_comicinfo_enrichment(result): service.schedule_comicinfo_enrichment( self._session_factory, job_id=job_id, ) elif job.status == ImportJobStatus.ROLLING_BACK: - await service.rollback_import( + rollback_result = await service.rollback_import( session, job_id, progress_callback=progress_callback, ) await session.commit() + if _rollback_waits_for_story_arc(rollback_result): + service.schedule_story_arc_sync() else: logger.info("import_runner_noop", job_id=job_id, status=job.status.value) return @@ -614,12 +750,14 @@ async def progress_callback(event: ImportProgressEvent) -> None: await self._finalize_cancel(session, job_id) job = await session.get(ImportJob, job_id) if job is not None and job.status == ImportJobStatus.ROLLING_BACK: - await service.rollback_import( + rollback_result = await service.rollback_import( session, job_id, progress_callback=progress_callback, ) await session.commit() + if _rollback_waits_for_story_arc(rollback_result): + service.schedule_story_arc_sync() except Exception as exc: logger.exception("import_runner_job_failed", job_id=job_id) await session.rollback() @@ -663,6 +801,7 @@ async def _run_single_job_once( service = await _build_import_service(session) terminal_event_override: ImportProgressEvent | None = None run_import_result: Any = None + rollback_result: Any = None async def progress_callback(event: ImportProgressEvent) -> None: await _publish_progress_event(event) @@ -686,12 +825,27 @@ async def progress_callback(event: ImportProgressEvent) -> None: await service.start_scan(session, job_id, progress_callback=progress_callback) elif job.status in {ImportJobStatus.IMPORTING, ImportJobStatus.ROLLING_BACK}: if job.status == ImportJobStatus.ROLLING_BACK: - await service.rollback_import( + rollback_result = await service.rollback_import( session, job_id, progress_callback=progress_callback, ) else: + if dict(job.progress_snapshot or {}).get("deferred_recovery"): + await prepare_deferred_recovery( + session, + job_id, + metadata_service=service._metadata_service, + progress_callback=progress_callback, + ) + if job.status != ImportJobStatus.IMPORTING: + await session.commit() + return + await prepare_clean_library_import( + session, + job_id, + progress_callback=progress_callback, + ) run_import_result = await service.run_import( session, job_id, @@ -702,6 +856,10 @@ async def progress_callback(event: ImportProgressEvent) -> None: return await session.commit() + if _should_schedule_story_arc_sync(run_import_result): + service.schedule_story_arc_sync() + if _rollback_waits_for_story_arc(rollback_result): + service.schedule_story_arc_sync() if _should_schedule_comicinfo_enrichment(run_import_result): service.schedule_comicinfo_enrichment(session_factory, job_id=job_id) except JobPausedError: @@ -714,7 +872,9 @@ async def progress_callback(event: ImportProgressEvent) -> None: await session.rollback() job = await session.get(ImportJob, job_id) if job is not None: - if job.import_started_at is None: + if await cancel_deferred_preparation(session, job): + purge_import_runtime_state(job_id) + elif job.import_started_at is None: terminal_event_override = ImportProgressEvent( job_id=job_id, status=ImportJobStatus.CANCELLED, @@ -737,12 +897,14 @@ async def progress_callback(event: ImportProgressEvent) -> None: status=ImportJobStatus.ROLLING_BACK, ) await session.commit() - await service.rollback_import( + rollback_result = await service.rollback_import( session, job_id, progress_callback=progress_callback, ) await session.commit() + if _rollback_waits_for_story_arc(rollback_result): + service.schedule_story_arc_sync() except Exception: logger.exception("import_task_failed", job_id=job_id) await session.rollback() @@ -796,6 +958,53 @@ async def run_import_series_rematch_task(job_id: int, imported_series_id: int) - await _run_import_series_rematch_task(job_id, imported_series_id) +async def run_import_safety_bulk_rematch_task(job_id: int) -> None: + """Rematch safety-approved series from one job in bounded keyset pages.""" + lock = _review_rematch_locks.setdefault(job_id, asyncio.Lock()) + async with lock: + session_factory = get_session_factory() + cursor = 0 + while True: + async with session_factory() as session: + job = await session.get(ImportJob, job_id) + if ( + job is None + or job.status != ImportJobStatus.REVIEW + or job.control_request != ImportControlRequest.NONE + ): + return + series_ids = list( + ( + await session.execute( + sa_select(ImportedSeries.id) + .join( + ImportedFile, + ImportedFile.import_series_id == ImportedSeries.id, + ) + .where( + ImportedSeries.import_job_id == job_id, + ImportedSeries.id > cursor, + ImportedSeries.diagnostics["rematch_pending"] + .as_boolean() + .is_(True), + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.SAFETY_APPROVED, + ) + .distinct() + .order_by(ImportedSeries.id) + .limit(_BULK_SAFETY_REMATCH_PAGE_SIZE) + ) + ) + .scalars() + .all() + ) + if not series_ids: + return + for imported_series_id in series_ids: + await _run_import_series_rematch_task(job_id, imported_series_id) + cursor = imported_series_id + + async def _run_import_series_rematch_task(job_id: int, imported_series_id: int) -> None: """Perform one rematch after the job-level review rematch lock is held.""" session_factory = get_session_factory() @@ -881,6 +1090,12 @@ def trigger_import_series_rematch(job_id: int, imported_series_id: int) -> None: ) +def trigger_import_safety_bulk_rematch(job_id: int) -> None: + """Schedule one bounded worker for all pending safety rematches in a job.""" + _fire_and_forget(run_import_safety_bulk_rematch_task(job_id)) + logger.info("import_safety_bulk_rematch_triggered", job_id=job_id) + + async def recover_stuck_import_jobs( session_factory: async_sessionmaker[AsyncSession], ) -> int: @@ -908,12 +1123,53 @@ async def recover_stuck_import_jobs( ) for item in series_result.scalars().all(): item.status = ImportSeriesStatus.CONFIRMED + await reconcile_durable_import_execution_counters(session, job) + + if ( + job.control_request is ImportControlRequest.CANCEL + or previous_status is ImportJobStatus.CANCELLING + ): + job.status = ImportJobStatus.ROLLING_BACK + job.control_request = ImportControlRequest.CANCEL + job.error_message = job.error_message or "Import cancelled by user." + job.story_arc_placement_followup_pending = False + job.progress_snapshot = initialize_progress_snapshot( + job, + mode="rollback", + phase="queued", + progress=0, + message="Cancelling import and rolling back changes...", + status=ImportJobStatus.ROLLING_BACK, + ) + recovered_snapshot = dict(job.progress_snapshot or {}) + recovered_snapshot[_STARTUP_RECOVERY_DISPATCH_PENDING] = True + recovered_snapshot["recovered_status"] = previous_status.value + recovered_snapshot["recovered_at"] = datetime.now(UTC).isoformat() + job.progress_snapshot = recovered_snapshot + logger.warning( + "import_cancel_recovered_on_startup", + job_id=job.id, + previous_status=previous_status.value, + ) + continue + + if ( + job.control_request is ImportControlRequest.PAUSE + or previous_status is ImportJobStatus.PAUSING + ): + sync_paused_job_state(job) + logger.warning( + "import_pause_recovered_on_startup", + job_id=job.id, + previous_status=previous_status.value, + ) + continue snapshot = dict(job.progress_snapshot or {}) if previous_status == ImportJobStatus.ROLLING_BACK: snapshot["phase"] = "rollback" elif previous_status == ImportJobStatus.IMPORTING: - snapshot["phase"] = "importing" + snapshot.setdefault("phase", "importing") snapshot["mode"] = "import" else: snapshot["mode"] = "scan" @@ -948,6 +1204,12 @@ def _is_startup_recovered_pause(job: ImportJob) -> bool: return snapshot.get("pause_reason") == _STARTUP_RECOVERY_PAUSE_REASON +def _is_startup_recovered_rollback(job: ImportJob) -> bool: + """Return True only for rollback work durably queued during startup recovery.""" + snapshot = dict(job.progress_snapshot or {}) + return snapshot.get(_STARTUP_RECOVERY_DISPATCH_PENDING) is True + + def _resume_startup_recovered_job(job: ImportJob) -> None: """Restore the runnable status for a job paused by startup recovery.""" snapshot = dict(job.progress_snapshot or {}) diff --git a/src/pullbox/tasks/search_task.py b/src/pullbox/tasks/search_task.py index 78f3b4de..80f5f23c 100644 --- a/src/pullbox/tasks/search_task.py +++ b/src/pullbox/tasks/search_task.py @@ -20,6 +20,7 @@ import structlog from sqlalchemy.exc import OperationalError +from pullbox.composition.airdcpp import get_airdcpp_supervisor_registry, load_airdcpp_search_clients from pullbox.composition.events import build_domain_event_bus from pullbox.config import get_settings from pullbox.core.config_resolver import get_int_setting, load_system_config_values, parse_bool @@ -70,6 +71,7 @@ load_wanted_issue_search_targets, ) from pullbox.services.search_source_selection import select_search_source +from pullbox.services.story_arc_search_targets import load_story_arc_missing_search_targets from pullbox.services.wanted_search_sweep import ( WantedSearchSweepState, checkpoint_wanted_search_items, @@ -230,6 +232,7 @@ async def _ensure_pending_series_search_logs( *, series_id: int, existing_log_ids_by_issue: dict[int, int], + story_arc_id: int | None = None, ) -> dict[int, int]: """Expose missing bulk-search rows before a series search starts.""" @@ -250,8 +253,17 @@ async def _ensure_pending_series_search_logs( details={ "run_state": "running", "action_status": "searching", - "task_id": f"search_series_{series_id}", + "task_id": ( + f"search_story_arc_{story_arc_id}" + if story_arc_id is not None + else f"search_series_{series_id}" + ), "trigger_type": "automated", + **( + {"story_arc_id": story_arc_id, "search_scope": "story_arc"} + if story_arc_id is not None + else {} + ), }, ) session.add(search_log) @@ -286,13 +298,7 @@ async def _persist_wanted_search_outcome( issue_grabbed = 0 issue_queued = 0 best_confidence: str | None = None - direct_outcome = outcome.direct_outcome - direct_results = ( - len(direct_outcome.matched) + len(direct_outcome.rejected) if direct_outcome else 0 - ) - dc_outcome = outcome.dc_outcome - dc_results = len(dc_outcome.matched) + len(dc_outcome.rejected) if dc_outcome else 0 - total_results = len(outcome.raw_results) + direct_results + dc_results + total_results = outcome.results_found_count action_status = "no_results" if total_results == 0 else "no_match" try: search_log = await session.get(SearchLog, pending_log_id) if pending_log_id else None @@ -338,10 +344,7 @@ async def _persist_wanted_search_outcome( search_log.results_found = total_results search_log.results_grabbed = issue_grabbed search_log.results_queued = issue_queued - search_log.results_rejected = max( - 0, - total_results - issue_grabbed - issue_queued, - ) + search_log.results_rejected = outcome.results_rejected_count search_log.details = _merge_search_log_details( existing_details=search_log.details or {}, next_details=next_details, @@ -377,7 +380,7 @@ async def _persist_wanted_search_outcome( results_found=total_results, results_grabbed=0, results_queued=0, - results_rejected=total_results, + results_rejected=outcome.results_rejected_count, details=_merge_search_log_details( existing_details=None, next_details=outcome.search_details, @@ -402,6 +405,7 @@ async def _persist_series_search_outcome( runtime: SearchRuntime, download_svc: DownloadService, intervention_svc: InterventionService, + story_arc_id: int | None = None, ) -> tuple[int, int, int]: """Route and persist one completed series-search outcome.""" @@ -411,6 +415,21 @@ async def _persist_series_search_outcome( validator_kwargs=runtime.validator_kwargs, ) target = primary_outcome.target + + async def arc_eligible() -> bool: + assert story_arc_id is not None + return bool( + await load_story_arc_missing_search_targets( + session, story_arc_id, series_id=target.series_id, issue_ids=[target.issue_id] + ) + ) + + if story_arc_id is not None and not await arc_eligible(): + if pending_log_id is not None: + await _complete_pending_bulk_search_logs( + session, {target.issue_id: pending_log_id}, action_status="no_longer_eligible" + ) + return 0, 0, 0 issue_log = log.bind(issue_id=target.issue_id, issue_number=target.issue_number) issue_log.info( "search_series_issue_results", @@ -479,6 +498,7 @@ async def _persist_series_search_outcome( runner=(get_direct_acquisition_runner() if runtime.direct_providers else None), source_priority=runtime.source_priority, planner=plan_direct_acquisition, + eligibility_check=arc_eligible if story_arc_id is not None else None, ) issue_grabbed = routed.grabbed issue_queued = routed.queued @@ -493,7 +513,7 @@ async def _persist_series_search_outcome( ) elif routed.source_kind == "dc": issue_log.info( - "search_series_issue_dc_evaluated", + "search_series_issue_dc_routed", action_status=routed.action_status, confidence=routed.best_confidence, search_pass=selected_pass, @@ -560,14 +580,18 @@ async def _persist_series_search_outcome( if routed.source_kind is not None: details["acquisition_method"] = routed.source_kind + rejected_count = selected_outcome.results_rejected_count + if fallback_outcome is not None: + other_pass = primary_outcome if selected_pass == 2 else fallback_outcome + rejected_count += len(other_pass.rejected) await _persist_bulk_search_log( session, target=target, - pending_log_id=pending_log_id, + pending_log_id=search_log.id, results_found=total_found, results_grabbed=issue_grabbed, results_queued=issue_queued, - results_rejected=max(0, total_found - issue_grabbed - issue_queued), + results_rejected=rejected_count, details=details, best_confidence=routed.best_confidence, action_status=( @@ -654,6 +678,11 @@ async def _build_task_search_runtime( include_download_clients: bool = True, ) -> SearchRuntime | None: """Build task runtime state using the task module's registry patch point.""" + registry = get_airdcpp_supervisor_registry() + has_automatic_dc = bool( + registry is not None + and await load_airdcpp_search_clients(session, registry, automatic=True) + ) return await _search_runtime.build_search_runtime( session, include_download_clients=include_download_clients, @@ -661,6 +690,7 @@ async def _build_task_search_runtime( default_type_thresholds=DEFAULT_TYPE_THRESHOLDS, eval_kwargs_builder=build_eval_kwargs, include_direct_providers=True, + allow_empty_registry=has_automatic_dc, ) @@ -916,16 +946,22 @@ async def search_series_issues( series_id: int, *, pending_log_ids_by_issue: dict[int, int] | None = None, + story_arc_id: int | None = None, + issue_ids: list[int] | None = None, ) -> dict[str, int]: - """Search indexers for all wanted issues of a single series. + """Search wanted series issues, or an explicit bounded arc-member batch. Obtains its own DB session so it can be called from event subscribers and background tasks without sharing caller state. + An arc scope never widens to the rest of its parent series, and eligibility + is checked again after provider work before routing a download. Returns: Dict with ``wanted``, ``sent``, and ``queued`` counts. """ - log = logger.bind(series_id=series_id) + if (story_arc_id is None) != (issue_ids is None): + raise ValueError("Arc-scoped searches require both the arc and its issue-ID batch") + log = logger.bind(series_id=series_id, story_arc_id=story_arc_id) log.info("search_series_issues_start") remaining_pending_log_ids = dict(pending_log_ids_by_issue or {}) @@ -958,7 +994,13 @@ async def search_series_issues( log.warning("search_series_issues_not_found") return {"wanted": 0, "sent": 0, "queued": 0} - targets = await load_series_wanted_search_targets(session, series_id) + targets = ( + await load_series_wanted_search_targets(session, series_id) + if story_arc_id is None + else await load_story_arc_missing_search_targets( + session, story_arc_id, series_id=series_id, issue_ids=issue_ids + ) + ) if not targets: if remaining_pending_log_ids: await _complete_pending_bulk_search_logs( @@ -974,6 +1016,7 @@ async def search_series_issues( targets, series_id=series_id, existing_log_ids_by_issue=remaining_pending_log_ids, + story_arc_id=story_arc_id, ) preload_ms = int((time.monotonic() - preload_started_at) * 1000) log.info( @@ -1013,6 +1056,20 @@ async def _process_outcome_pair( routing_started_at = time.monotonic() # Preserve provider health even if downstream routing rolls back. await session.commit() + if story_arc_id is not None: + current = await load_story_arc_missing_search_targets( + session, story_arc_id, series_id=series_id, issue_ids=[issue_id] + ) + if not current: + pending_id = remaining_pending_log_ids.pop(issue_id, None) + if pending_id is not None: + await _complete_pending_bulk_search_logs( + session, + {issue_id: pending_id}, + action_status="no_longer_eligible", + ) + processed_issue_ids.add(issue_id) + return issue_sent, issue_queued, issue_failed = await _persist_series_search_outcome( session, log=log, @@ -1022,6 +1079,7 @@ async def _process_outcome_pair( runtime=runtime, download_svc=download_svc, intervention_svc=intervention_svc, + story_arc_id=story_arc_id, ) sent += issue_sent queued += issue_queued diff --git a/src/pullbox/tasks/story_arc_metadata_task.py b/src/pullbox/tasks/story_arc_metadata_task.py new file mode 100644 index 00000000..f08e4af2 --- /dev/null +++ b/src/pullbox/tasks/story_arc_metadata_task.py @@ -0,0 +1,157 @@ +"""Daily, bounded provider membership discovery for monitored story arcs.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +import structlog +from sqlalchemy import func, select +from sqlalchemy.exc import SQLAlchemyError + +from pullbox.core.comicvine_key import get_comicvine_api_key +from pullbox.core.scheduler import scheduled_task +from pullbox.database import get_session_factory +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryRoot +from pullbox.models.series import Series +from pullbox.models.story_arc import IssueStoryArc, StoryArc, StoryArcLifecycle +from pullbox.providers.metadata.comicvine import ComicVineError, ComicVineProvider +from pullbox.services.import_activity import has_active_import_scheduler_protection +from pullbox.services.story_arc_catalog import MAX_CATALOG_PARENTS, StoryArcCatalogService +from pullbox.services.story_arc_service import StoryArcServiceError + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + from sqlalchemy.sql.elements import ColumnElement + +logger = structlog.get_logger(__name__) +_PAGE_SIZE = 25 + + +def _active_monitored() -> tuple[ColumnElement[bool], ...]: + return ( + StoryArc.monitored.is_(True), + StoryArc.lifecycle == StoryArcLifecycle.ACTIVE, + StoryArc.comicvine_id.is_not(None), + ) + + +async def _refresh_arc( + factory: async_sessionmaker[AsyncSession], service: StoryArcCatalogService, arc_id: int +) -> int: + """Snapshot, fetch outside the session, recheck consent, then persist.""" + async with factory() as session: + arc = await session.scalar( + select(StoryArc).where(StoryArc.id == arc_id, *_active_monitored()) + ) + if arc is None: + return 0 + provider_id, revision = str(arc.comicvine_id), arc.revision + known_parents = tuple( + str(value) + for value in await session.scalars( + select(Series.comicvine_id) + .join(Issue, Issue.series_id == Series.id) + .join(IssueStoryArc, IssueStoryArc.issue_id == Issue.id) + .where(IssueStoryArc.story_arc_id == arc_id, Series.comicvine_id.is_not(None)) + .distinct() + .limit(MAX_CATALOG_PARENTS) + ) + ) + preview = await service.preview(provider_id, known_series_provider_ids=known_parents) + if await has_active_import_scheduler_protection(factory): + return 0 + async with factory() as session: + arc = await session.scalar( + select(StoryArc).where(StoryArc.id == arc_id, *_active_monitored()) + ) + if arc is None or arc.revision != revision: + return 0 + # Older imports may lack an arc-specific future destination. Only the + # explicitly configured default managed root is a safe fallback. Never + # infer it from an import source or the separate arc-copy directory. + default_roots = list( + await session.scalars( + select(LibraryRoot.id) + .where(LibraryRoot.is_default_managed_destination.is_(True)) + .limit(2) + ) + ) + result = await service.refresh( + session, + arc_id, + preview, + expected_revision=revision, + library_root_id=default_roots[0] if len(default_roots) == 1 else None, + ) + await session.commit() + # The shared wanted sweep observes new members after this commit and + # rechecks monitoring, explicit skips, dates, and duplicate downloads. + return len(result.added_membership_ids) + + +async def sync_story_arc_metadata() -> None: + """Refresh each eligible arc once, isolating provider failures per arc.""" + factory = get_session_factory() + if await has_active_import_scheduler_protection(factory): + return + async with factory() as session: + api_key = await get_comicvine_api_key(session) + ceiling = await session.scalar(select(func.max(StoryArc.id)).where(*_active_monitored())) + if not api_key or ceiling is None: + return + provider = ComicVineProvider(api_key=api_key) + service = StoryArcCatalogService(provider) + cursor = refreshed = added = failed = 0 + try: + while cursor < ceiling: + async with factory() as session: + ids = list( + await session.scalars( + select(StoryArc.id) + .where(StoryArc.id > cursor, StoryArc.id <= ceiling, *_active_monitored()) + .order_by(StoryArc.id) + .limit(_PAGE_SIZE) + ) + ) + if not ids: + break + for arc_id in ids: + if await has_active_import_scheduler_protection(factory): + return + cursor = arc_id + try: + added += await _refresh_arc(factory, service, arc_id) + refreshed += 1 + except (ComicVineError, StoryArcServiceError, SQLAlchemyError) as exc: + failed += 1 + code = getattr(exc, "code", "provider_unavailable") + logger.warning( + "story_arc_metadata_refresh_failed", story_arc_id=arc_id, category=code + ) + async with factory() as session: + arc = await session.get(StoryArc, arc_id) + if arc is not None: + arc.diagnostics = { + **arc.diagnostics, + "provider_refresh_error": { + "code": str(code), + "checked_at": datetime.now(UTC).isoformat(), + }, + } + await session.commit() + finally: + await provider.close() + logger.info("story_arc_metadata_refresh_done", refreshed=refreshed, added=added, failed=failed) + + +@scheduled_task( + task_id="sync_story_arc_metadata", + trigger="cron", + display_name="Sync Story Arc Members", + hour=1, + minute=30, +) +async def scheduled_sync_story_arc_metadata() -> None: + await sync_story_arc_metadata() diff --git a/src/pullbox/tasks/story_arc_search_task.py b/src/pullbox/tasks/story_arc_search_task.py new file mode 100644 index 00000000..c7aefeb0 --- /dev/null +++ b/src/pullbox/tasks/story_arc_search_task.py @@ -0,0 +1,76 @@ +"""Explicit arc-member searches reuse the shared acquisition runner in small batches.""" + +from __future__ import annotations + +import asyncio +from collections import defaultdict + +import structlog + +from pullbox.database import get_session_factory +from pullbox.services.story_arc_search_targets import ( + load_story_arc_missing_search_targets, + story_arc_search_ceiling, +) +from pullbox.tasks.search_task import search_series_issues + +logger = structlog.get_logger(__name__) +_PAGE_SIZE = 50 +_running_searches: dict[int, asyncio.Task[dict[str, int]]] = {} + + +async def search_story_arc_issues(story_arc_id: int) -> dict[str, int]: + """Search the finite current arc without monitoring any parent series.""" + totals = {"wanted": 0, "sent": 0, "queued": 0} + factory = get_session_factory() + async with factory() as session: + ceiling = await story_arc_search_ceiling(session, story_arc_id) + cursor = 0 + while cursor < ceiling: + async with factory() as session: + targets = await load_story_arc_missing_search_targets( + session, + story_arc_id, + after_issue_id=cursor, + ceiling_issue_id=ceiling, + limit=_PAGE_SIZE, + ) + if not targets: + break + cursor = targets[-1].issue_id + by_series: dict[int, list[int]] = defaultdict(list) + for target in targets: + by_series[target.series_id].append(target.issue_id) + for series_id, issue_ids in by_series.items(): + result = await search_series_issues( + series_id, + story_arc_id=story_arc_id, + issue_ids=issue_ids, + ) + for name in totals: + totals[name] += result[name] + await asyncio.sleep(0) + logger.info("story_arc_search_completed", story_arc_id=story_arc_id, **totals) + return totals + + +def _search_finished(story_arc_id: int, task: asyncio.Task[dict[str, int]]) -> None: + if _running_searches.get(story_arc_id) is task: + _running_searches.pop(story_arc_id, None) + if task.cancelled(): + return + try: + task.result() + except Exception: + logger.exception("story_arc_search_failed", story_arc_id=story_arc_id) + + +def schedule_story_arc_search(story_arc_id: int) -> bool: + """Keep one live explicit search per arc in the application worker.""" + existing = _running_searches.get(story_arc_id) + if existing is not None and not existing.done(): + return False + task = asyncio.create_task(search_story_arc_issues(story_arc_id)) + _running_searches[story_arc_id] = task + task.add_done_callback(lambda done: _search_finished(story_arc_id, done)) + return True diff --git a/src/pullbox/tasks/story_arc_sync_task.py b/src/pullbox/tasks/story_arc_sync_task.py new file mode 100644 index 00000000..a567e18f --- /dev/null +++ b/src/pullbox/tasks/story_arc_sync_task.py @@ -0,0 +1,71 @@ +"""Scheduled bounded drain for durable story-arc synchronization work.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import structlog + +from pullbox.core.scheduler import get_scheduler, scheduled_task +from pullbox.services.import_activity import has_active_import_scheduler_protection +from pullbox.services.story_arc_sync_queue import ( + STORY_ARC_SYNC_TASK_ID, + process_story_arc_sync_work, +) + +logger = structlog.get_logger(__name__) + + +@scheduled_task( + task_id=STORY_ARC_SYNC_TASK_ID, + trigger="interval", + display_name="Synchronize Story Arc Placements", + seconds=300, +) +async def scheduled_sync_story_arc_placements() -> None: + """Process one bounded batch and schedule the next durable continuation.""" + import_only = await has_active_import_scheduler_protection() + result = await process_story_arc_sync_work(import_only=import_only) + if result.import_jobs_evaluated: + from pullbox.tasks.import_task import publish_story_arc_import_updates + + await publish_story_arc_import_updates( + result.import_jobs_evaluated, + completed_job_ids=result.import_jobs_completed, + ) + if result.import_jobs_rollback_ready: + from pullbox.tasks.import_task import trigger_import_rollback + + for job_id in result.import_jobs_rollback_ready: + trigger_import_rollback(job_id) + scheduler = get_scheduler() + if result.has_more: + scheduler.schedule_task_continuation( + STORY_ARC_SYNC_TASK_ID, + run_at=datetime.now(UTC) + timedelta(seconds=1), + interval_seconds=300, + ) + elif result.next_retry_at is not None: + scheduler.schedule_task_continuation( + STORY_ARC_SYNC_TASK_ID, + run_at=result.next_retry_at, + interval_seconds=300, + ) + else: + scheduler.clear_task_continuation(STORY_ARC_SYNC_TASK_ID) + logger.info( + "story_arc_sync_done", + discovered=result.discovered, + claimed=result.claimed, + completed=result.completed, + failed=result.failed, + retrying=result.retrying, + cancelled=result.cancelled, + lost_claims=result.lost_claims, + has_more=result.has_more, + import_only=import_only, + import_jobs_evaluated=len(result.import_jobs_evaluated), + import_jobs_completed=len(result.import_jobs_completed), + import_jobs_stalled=len(result.import_jobs_stalled), + import_jobs_rollback_ready=len(result.import_jobs_rollback_ready), + ) diff --git a/src/pullbox/ui/comicvine_provider.py b/src/pullbox/ui/comicvine_provider.py new file mode 100644 index 00000000..d1d32b0e --- /dev/null +++ b/src/pullbox/ui/comicvine_provider.py @@ -0,0 +1,54 @@ +"""Shared Comic Vine provider lifecycle for UI discovery.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any + +from pullbox.services.comicvine_persistent_cache import PersistentComicVineCacheProvider + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +class ComicVineNotConfiguredError(RuntimeError): + """Raised when a Comic Vine-backed UI action has no configured API key.""" + + +@asynccontextmanager +async def open_comicvine_ui_provider( + session: AsyncSession, + *, + session_factory: async_sessionmaker[AsyncSession] | None = None, + prefer_catalog: bool = False, +) -> AsyncIterator[Any]: + """Open the configured Comic Vine provider for one UI operation.""" + from pullbox.core.comicvine_key import get_comicvine_api_key + from pullbox.services.catalog.lookup import CatalogLookupService + from pullbox.services.catalog.reader import get_catalog_reader + + reader = get_catalog_reader() + if prefer_catalog and reader.available: + yield CatalogLookupService(reader) + return + + api_key = await get_comicvine_api_key(session) + await session.rollback() + if not api_key: + raise ComicVineNotConfiguredError("Comic Vine is not configured") + + # Import lazily so provider test doubles remain isolated from app startup. + from pullbox.providers.metadata.comicvine import ComicVineProvider + + provider = ComicVineProvider(api_key=api_key) + opened: Any = ( + PersistentComicVineCacheProvider(provider, session_factory) + if session_factory is not None + else provider + ) + try: + yield opened + finally: + await provider.close() diff --git a/src/pullbox/ui/formatters.py b/src/pullbox/ui/formatters.py index 150512fb..c859091b 100644 --- a/src/pullbox/ui/formatters.py +++ b/src/pullbox/ui/formatters.py @@ -9,6 +9,7 @@ from pullbox.core.duration_format import format_duration_ms_label from pullbox.core.html_sanitizer import sanitize_rich_html +from pullbox.core.issue_numbers import format_issue_number as _format_issue_number from pullbox.models.series import SeriesStatus @@ -19,7 +20,7 @@ def sanitize_rich_html_filter(value: str | None) -> Markup: def format_issue_number(value: float) -> str: """Format issue number, stripping unnecessary trailing zeros.""" - return f"{value:g}" + return _format_issue_number(value) def format_filesize(value: int) -> str: diff --git a/src/pullbox/ui/import_conflict_review.py b/src/pullbox/ui/import_conflict_review.py index 3e4f9059..db0c026b 100644 --- a/src/pullbox/ui/import_conflict_review.py +++ b/src/pullbox/ui/import_conflict_review.py @@ -2,9 +2,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING -from sqlalchemy import select +from sqlalchemy import func, select from pullbox.composition.services import build_import_control_service from pullbox.core.exceptions import NotFoundError @@ -40,8 +40,21 @@ async def _load_import_conflict_review_context( if job is None: raise NotFoundError("ImportJob", job_id) + allowed_sort_fields = {"series", "conflict", "files", "signal", "status"} + sort_field = sort.removeprefix("-") + if sort_field not in allowed_sort_fields: + sort = "series" + + page_size = 25 svc = build_import_control_service() - conflict_groups = await svc.get_conflict_groups(session, job_id) + conflict_page = await svc.get_conflict_groups_page( + session, + job_id, + page=page, + page_size=page_size, + sort=sort, + ) + conflict_groups = list(conflict_page.items) metadata_extractor = SourceMetadataExtractor() def _series_label(imp_series: ImportedSeries) -> str: @@ -128,118 +141,98 @@ def _distinct_filename_series_names(files: list[ImportedFile]) -> list[str]: seen[normalized] = label return sorted(seen.values(), key=lambda value: NameMatcher.normalize(value)) - def _conflict_sort_issue_number(group: dict[str, object]) -> tuple[int, float, str]: - value = group.get("display_issue_number") - if value is None: - return (1, 0.0, "") - if isinstance(value, int | float): - return (0, float(value), str(value)) - if isinstance(value, str): - try: - return (0, float(value), value) - except ValueError: - return (0, 0.0, value) - return (0, 0.0, str(value)) + series_by_id: dict[int, ImportedSeries] = {} + for group in conflict_groups: + group_series = group.get("series") + if isinstance(group_series, ImportedSeries): + series_by_id[group_series.id] = group_series - def _default_conflict_sort_key( - group: dict[str, object], - ) -> tuple[str, int, tuple[int, float, str], str]: - series_label = str(group.get("series_name") or group.get("raw_series_name") or "") - kind_order = 0 if group.get("kind") == "series_conflict" else 1 - return ( - NameMatcher.normalize(series_label), - kind_order, - _conflict_sort_issue_number(group), - str(group.get("conflict_group_id") or ""), + requested_series_ids = { + int(series_id) + for group in conflict_groups + if (series_id := group.get("series_id")) is not None + } + missing_series_ids = requested_series_ids.difference(series_by_id) + if missing_series_ids: + series_result = await session.execute( + select(ImportedSeries).where(ImportedSeries.id.in_(missing_series_ids)) ) + series_by_id.update({item.id: item for item in series_result.scalars().all()}) - def _conflict_signal_label(group: dict[str, object]) -> str: - diagnostics = group.get("diagnostics") - diagnostics = diagnostics if isinstance(diagnostics, dict) else {} - selected_candidate = diagnostics.get("selected_candidate") - if isinstance(selected_candidate, dict) and selected_candidate.get("title"): - return str(selected_candidate["title"]) - if group.get("kind") == "series_conflict": - return "candidate needs review" - return "auto-selected" if group.get("has_preferred") else "needs choice" + issue_ids = { + int(issue_id) + for group in conflict_groups + if (issue_id := group.get("matched_issue_id")) is not None + } + issues_by_id: dict[int, Issue] = {} + if issue_ids: + issue_result = await session.execute(select(Issue).where(Issue.id.in_(issue_ids))) + issues_by_id = {item.id: item for item in issue_result.scalars().all()} - def _conflict_status_label(group: dict[str, object]) -> str: - if group.get("kind") == "series_conflict": - return "series match conflict" - return "auto-selected" if group.get("has_preferred") else "needs choice" + series_conflicts = [ + group for group in conflict_groups if group.get("kind") == "series_conflict" + ] + sibling_series_by_name: dict[str, list[ImportedSeries]] = {} + related_files_by_series_id: dict[int, list[ImportedFile]] = {} + if series_conflicts: + conflict_series = [ + series_by_id[int(group["series_id"])] + for group in series_conflicts + if int(group["series_id"]) in series_by_id + ] + raw_names = {item.raw_series_name.casefold() for item in conflict_series} + sibling_result = await session.execute( + select(ImportedSeries).where( + ImportedSeries.import_job_id == job_id, + func.lower(ImportedSeries.raw_series_name).in_(raw_names), + ) + ) + sibling_series = list(sibling_result.scalars().all()) + for sibling in sibling_series: + sibling_series_by_name.setdefault( + NameMatcher.normalize(sibling.raw_series_name), [] + ).append(sibling) - def _sortable_conflict_key(group: dict[str, object]) -> Any: - match sort_field: - case "conflict": - return ( - 0 if group.get("kind") == "series_conflict" else 1, - _conflict_sort_issue_number(group), - _default_conflict_sort_key(group), - ) - case "files": - return ( - _object_to_int(group.get("file_count")), - _default_conflict_sort_key(group), - ) - case "signal": - return ( - NameMatcher.normalize(_conflict_signal_label(group)), - _default_conflict_sort_key(group), - ) - case "status": - return ( - NameMatcher.normalize(_conflict_status_label(group)), - _default_conflict_sort_key(group), + sibling_ids = [sibling.id for sibling in sibling_series] + if sibling_ids: + related_file_limit = max(24, len(series_conflicts) * 24) + sibling_files_result = await session.execute( + select(ImportedFile) + .where(ImportedFile.import_series_id.in_(sibling_ids)) + .order_by(ImportedFile.has_comicinfo.desc(), ImportedFile.id.asc()) + .limit(related_file_limit) + ) + for sibling_file in sibling_files_result.scalars().all(): + related_files_by_series_id.setdefault(sibling_file.import_series_id, []).append( + sibling_file ) - case _: - return _default_conflict_sort_key(group) - - allowed_sort_fields = {"series", "conflict", "files", "signal", "status"} - sort_desc = sort.startswith("-") - sort_field = sort.removeprefix("-") - if sort_field not in allowed_sort_fields: - sort_field = "series" - sort_desc = False - sort = "series" enriched_groups: list[dict[str, object]] = [] for group in conflict_groups: if group.get("kind") == "series_conflict": - imp_series = await session.get(ImportedSeries, group["series_id"]) + imp_series = series_by_id.get(int(group["series_id"])) if imp_series is None: continue - files = list(group.get("files", [])) + files = [item for item in group.get("files", []) if isinstance(item, ImportedFile)] parsed_series_names = _distinct_filename_series_names(files) - sibling_series_result = await session.execute( - select(ImportedSeries).where( - ImportedSeries.import_job_id == job_id, - ImportedSeries.id != imp_series.id, - ) - ) normalized_title = NameMatcher.normalize(imp_series.raw_series_name or "") sibling_series = [ sibling - for sibling in sibling_series_result.scalars().all() - if NameMatcher.normalize(sibling.raw_series_name or "") == normalized_title + for sibling in sibling_series_by_name.get(normalized_title, []) + if sibling.id != imp_series.id ] sibling_by_id = {sibling.id: sibling for sibling in sibling_series} related_source_files: list[dict[str, object]] = [] current_series_year = imp_series.raw_year if sibling_by_id: - sibling_files_result = await session.execute( - select(ImportedFile) - .where(ImportedFile.import_series_id.in_(list(sibling_by_id))) - .order_by(ImportedFile.import_series_id.asc(), ImportedFile.id.asc()) - ) - for sibling_file in sibling_files_result.scalars().all(): - related_summary = _build_source_file_summary( - sibling_file, - current_series_label=_series_label( - sibling_by_id[sibling_file.import_series_id] - ), - ) - if related_summary["comicinfo"]: - related_source_files.append(related_summary) + for sibling_id, sibling in sibling_by_id.items(): + for sibling_file in related_files_by_series_id.get(sibling_id, []): + related_summary = _build_source_file_summary( + sibling_file, + current_series_label=_series_label(sibling), + ) + if related_summary["comicinfo"]: + related_source_files.append(related_summary) related_source_files.sort( key=lambda item: ( 0 @@ -262,7 +255,8 @@ def _sortable_conflict_key(group: dict[str, object]) -> Any: "raw_series_name": imp_series.raw_series_name, "source_folder": imp_series.source_folder or "", "has_preferred": False, - "file_count": len(files), + "file_count": _object_to_int(group.get("file_count"), len(files)), + "files_truncated": bool(group.get("files_truncated")), "display_issue_number": None, "parsed_series_names": parsed_series_names, "mixed_series_bucket": False, @@ -283,31 +277,31 @@ def _sortable_conflict_key(group: dict[str, object]) -> Any: series_name = "" source_folder = "" if group["matched_issue_id"]: - issue = await session.get(Issue, group["matched_issue_id"]) - if group["files"]: - first_file: ImportedFile = group["files"][0] - imp_series = await session.get(ImportedSeries, first_file.import_series_id) - if imp_series: - series_name = imp_series.raw_series_name - if imp_series.raw_year: - series_name += f" ({imp_series.raw_year})" - source_folder = imp_series.source_folder or "" + issue = issues_by_id.get(int(group["matched_issue_id"])) + group_files = [item for item in group.get("files", []) if isinstance(item, ImportedFile)] + group_series_id = group.get("series_id") + imp_series = series_by_id.get(int(group_series_id)) if group_series_id is not None else None + if imp_series: + series_name = imp_series.raw_series_name + if imp_series.raw_year: + series_name += f" ({imp_series.raw_year})" + source_folder = imp_series.source_folder or "" parsed_issue_numbers = sorted( - {f.parsed_issue_number for f in group["files"] if f.parsed_issue_number is not None} + {f.parsed_issue_number for f in group_files if f.parsed_issue_number is not None} ) - parsed_series_names = _distinct_filename_series_names(group["files"]) + parsed_series_names = _distinct_filename_series_names(group_files) display_issue_number = ( issue.issue_number if issue is not None else (parsed_issue_numbers[0] if parsed_issue_numbers else None) ) - has_preferred = any(f.is_preferred for f in group["files"]) - preferred_file = next((f for f in group["files"] if f.is_preferred), None) + has_preferred = any(f.is_preferred for f in group_files) + preferred_file = next((f for f in group_files if f.is_preferred), None) diagnostics = ( (preferred_file.diagnostics if preferred_file is not None else None) - or (group["files"][0].diagnostics if group["files"] else {}) + or (group_files[0].diagnostics if group_files else {}) or {} ) enriched_groups.append( @@ -315,12 +309,13 @@ def _sortable_conflict_key(group: dict[str, object]) -> Any: "kind": group.get("kind", "file_conflict"), "conflict_group_id": group["conflict_group_id"], "matched_issue_id": group["matched_issue_id"], - "files": group["files"], + "files": group_files, "issue": issue, "series_name": series_name, "source_folder": source_folder, "has_preferred": has_preferred, - "file_count": len(group["files"]), + "file_count": _object_to_int(group.get("file_count"), len(group_files)), + "files_truncated": bool(group.get("files_truncated")), "display_issue_number": display_issue_number, "parsed_series_names": parsed_series_names, "mixed_series_bucket": len(parsed_series_names) > 1, @@ -328,28 +323,20 @@ def _sortable_conflict_key(group: dict[str, object]) -> Any: } ) - enriched_groups.sort(key=_sortable_conflict_key, reverse=sort_desc) - - page_size = 25 - total_groups = len(enriched_groups) - file_conflict_groups = [g for g in enriched_groups if g.get("kind") == "file_conflict"] - auto_resolved = sum(1 for g in file_conflict_groups if g["has_preferred"]) - needs_decision = sum(1 for g in file_conflict_groups if not g["has_preferred"]) - series_candidate_conflicts = sum( - 1 for g in enriched_groups if g.get("kind") == "series_conflict" - ) + total_groups = conflict_page.total total_pages = max(1, (total_groups + page_size - 1) // page_size) - current_page = min(page, total_pages) - visible_groups = enriched_groups[(current_page - 1) * page_size : current_page * page_size] - visible_file_conflict_groups = [g for g in visible_groups if g.get("kind") == "file_conflict"] + current_page = conflict_page.page + visible_file_conflict_groups = [ + group for group in enriched_groups if group.get("kind") == "file_conflict" + ] return { "job": job, - "conflict_groups": visible_groups, - "auto_resolved": auto_resolved, - "needs_decision": needs_decision, - "series_candidate_conflicts": series_candidate_conflicts, - "file_conflict_group_count": len(file_conflict_groups), + "conflict_groups": enriched_groups, + "auto_resolved": conflict_page.auto_resolved, + "needs_decision": conflict_page.needs_decision, + "series_candidate_conflicts": conflict_page.series_candidate_conflicts, + "file_conflict_group_count": conflict_page.file_conflict_groups, "visible_file_conflict_group_count": len(visible_file_conflict_groups), "total_groups": total_groups, "page": current_page, diff --git a/src/pullbox/ui/import_follow_up.py b/src/pullbox/ui/import_follow_up.py new file mode 100644 index 00000000..871ed729 --- /dev/null +++ b/src/pullbox/ui/import_follow_up.py @@ -0,0 +1,229 @@ +"""Job-grouped context for import follow-up work.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import and_, func, or_, select + +from pullbox.core.exceptions import NotFoundError +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobStatus, + ImportSeriesStatus, +) +from pullbox.models.series import IssueCatalogState, Series +from pullbox.models.story_arc import ImportedStoryArcStatus +from pullbox.models.story_arc_import import ImportedStoryArc +from pullbox.ui.import_orphaned_routes import load_import_orphaned_context +from pullbox.ui.import_results_context import load_import_results_context + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql import Select + from sqlalchemy.sql.elements import ColumnElement + + +_FOLLOW_UP_PAGE_SIZE = 25 +_FOLLOW_UP_JOB_STATUSES = (ImportJobStatus.COMPLETED, ImportJobStatus.FAILED) +_FOLLOW_UP_SERIES_STATUSES = ( + ImportSeriesStatus.NO_MATCH, + ImportSeriesStatus.RECOVERY_PENDING, + ImportSeriesStatus.FAILED, +) +_FOLLOW_UP_FILE_STATUSES = ( + ImportedFileStatus.NO_MATCH, + ImportedFileStatus.FAILED, + ImportedFileStatus.SAFETY_BLOCKED, + ImportedFileStatus.CONFLICT, +) +_VERIFIED_CROSS_FOLDER_METHODS = ( + "verified_cross_folder_issue_identity", + "verified_cross_folder_series_issue_filename", +) + + +def _follow_up_job_filter() -> ColumnElement[bool]: + misplaced = ImportedFile.diagnostics["mylar3_cross_folder_reconciliation"] + cleanup = ImportedFile.diagnostics["misplaced_source_cleanup"] + unresolved_series = ( + select(ImportedSeries.id) + .where( + ImportedSeries.import_job_id == ImportJob.id, + ImportedSeries.status.in_(_FOLLOW_UP_SERIES_STATUSES), + ) + .exists() + ) + unresolved_files = ( + select(ImportedFile.id) + .where( + ImportedFile.import_job_id == ImportJob.id, + ImportedFile.status.in_(_FOLLOW_UP_FILE_STATUSES), + ) + .exists() + ) + misplaced_files = ( + select(ImportedFile.id) + .where( + ImportedFile.import_job_id == ImportJob.id, + misplaced["method"].as_string().in_(_VERIFIED_CROSS_FOLDER_METHODS), + or_( + and_( + ImportedFile.status == ImportedFileStatus.IMPORTED, + misplaced["role"].as_string() == "canonical", + misplaced["restored_at"].as_string().is_(None), + ImportedFile.library_file_id.is_not(None), + ), + and_( + ImportedFile.status == ImportedFileStatus.DUPLICATE_FILE, + misplaced["role"].as_string() == "identical_duplicate", + cleanup["action"].as_string().is_(None), + ImportedFile.duplicate_of_file_id.is_not(None), + ImportedFile.content_hash.is_not(None), + ), + ), + ) + .exists() + ) + unresolved_story_arcs = ( + select(ImportedStoryArc.id) + .where( + ImportedStoryArc.import_job_id == ImportJob.id, + ImportedStoryArc.materialized_story_arc_id.is_(None), + ImportedStoryArc.status != ImportedStoryArcStatus.SKIPPED, + ) + .exists() + ) + failed_metadata = ( + select(ImportedSeries.id) + .join(Series, Series.id == ImportedSeries.series_id) + .where( + ImportedSeries.import_job_id == ImportJob.id, + ImportedSeries.status == ImportSeriesStatus.IMPORTED, + Series.issue_catalog_state == IssueCatalogState.FAILED, + ) + .exists() + ) + return or_( + unresolved_series, + unresolved_files, + misplaced_files, + unresolved_story_arcs, + failed_metadata, + ) + + +def _follow_up_jobs_statement() -> Select[tuple[ImportJob]]: + return select(ImportJob).where( + ImportJob.status.in_(_FOLLOW_UP_JOB_STATUSES), + ImportJob.archived_at.is_(None), + _follow_up_job_filter(), + ) + + +async def count_import_follow_up_jobs(session: AsyncSession) -> int: + """Return the number of imports with actionable follow-up.""" + return int( + ( + await session.scalar( + select(func.count()).select_from(_follow_up_jobs_statement().subquery()) + ) + ) + or 0 + ) + + +async def _count_dismissed_import_series(session: AsyncSession) -> int: + return int( + ( + await session.scalar( + select(func.count()) + .select_from(ImportedSeries) + .join(ImportJob, ImportedSeries.import_job_id == ImportJob.id) + .where( + ImportedSeries.status == ImportSeriesStatus.SKIPPED, + ImportJob.status == ImportJobStatus.COMPLETED, + ) + ) + ) + or 0 + ) + + +async def load_import_follow_up_context( + session: AsyncSession, + *, + view: str, + requested_page: int, + job_id: int | None, +) -> dict[str, object]: + """Load either grouped import jobs or one job's actionable follow-up.""" + normalized_view = "dismissed" if view == "dismissed" else "all" + follow_up_job_count = await count_import_follow_up_jobs(session) + + if normalized_view == "dismissed": + orphaned = await load_import_orphaned_context( + session, + view="dismissed", + requested_page=requested_page, + ) + return { + **orphaned, + "selected_follow_up_job": None, + "follow_up_jobs": [], + "follow_up_job_count": follow_up_job_count, + } + + if job_id is not None: + job = await session.get(ImportJob, job_id) + if job is None or job.status not in _FOLLOW_UP_JOB_STATUSES or job.archived_at is not None: + raise NotFoundError("ImportJob", job_id) + orphaned = await load_import_orphaned_context( + session, + view="all", + requested_page=requested_page, + job_id=job_id, + ) + return { + **orphaned, + **await load_import_results_context( + session, + job, + include_clean_library=False, + ), + "selected_follow_up_job": job, + "follow_up_jobs": [], + "follow_up_job_count": follow_up_job_count, + } + + total_pages = max( + 1, + (follow_up_job_count + _FOLLOW_UP_PAGE_SIZE - 1) // _FOLLOW_UP_PAGE_SIZE, + ) + page = min(max(1, requested_page), total_pages) + jobs = list( + ( + await session.scalars( + _follow_up_jobs_statement() + .order_by(ImportJob.created_at.desc(), ImportJob.id.desc()) + .offset((page - 1) * _FOLLOW_UP_PAGE_SIZE) + .limit(_FOLLOW_UP_PAGE_SIZE) + ) + ).all() + ) + return { + "items": [], + "total": follow_up_job_count, + "page": page, + "page_size": _FOLLOW_UP_PAGE_SIZE, + "view": "all", + "orphaned_count": 0, + "dismissed_count": await _count_dismissed_import_series(session), + "total_pages": total_pages, + "selected_follow_up_job": None, + "follow_up_jobs": jobs, + "follow_up_job_count": follow_up_job_count, + } diff --git a/src/pullbox/ui/import_history.py b/src/pullbox/ui/import_history.py index 03101c0e..8a448179 100644 --- a/src/pullbox/ui/import_history.py +++ b/src/pullbox/ui/import_history.py @@ -6,7 +6,16 @@ from sqlalchemy import ColumnElement, String, case, cast, func, or_, select -from pullbox.models.import_job import ImportedSeries, ImportJob, ImportJobStatus, ImportSeriesStatus +from pullbox.models.import_job import ( + ImportedFile, + ImportedFileStatus, + ImportedSeries, + ImportJob, + ImportJobStatus, + ImportSeriesStatus, +) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile, LibraryFileStorageMode from pullbox.services.import_workflow_state import ( import_control_state_for_job, snapshot_mode_for_job, @@ -129,19 +138,29 @@ async def _load_import_history_context( search_query: str = "", sort: str = "", requested_page: int = 1, + show_archived: bool = False, ) -> dict[str, object]: """Load the import history page context.""" - clearable_jobs_total: int = ( - await session.execute( - select(func.count(ImportJob.id)).where( - ImportJob.status.in_(_IMPORT_HISTORY_CLEARABLE_STATUSES) - ) + archive_filter = ( + ImportJob.archived_at.is_not(None) if show_archived else ImportJob.archived_at.is_(None) + ) + clearable_jobs_total = 0 + if not show_archived: + clearable_jobs_total = int( + ( + await session.execute( + select(func.count(ImportJob.id)).where( + ImportJob.status.in_(_IMPORT_HISTORY_CLEARABLE_STATUSES), + archive_filter, + ) + ) + ).scalar_one() + or 0 ) - ).scalar_one() normalized_search = (search_query or "").strip() normalized_sort = _normalize_import_history_sort(sort) - history_filters = [] + history_filters: list[ColumnElement[bool]] = [archive_filter] if normalized_search: search_pattern = f"%{normalized_search}%" history_filters.append( @@ -214,6 +233,7 @@ async def _load_import_history_context( } for job in jobs } + clean_library_job_ids: set[int] = set() terminal_result_statuses = { ImportJobStatus.COMPLETED, ImportJobStatus.FAILED, @@ -234,6 +254,26 @@ async def _load_import_history_context( } if jobs: job_ids = [job.id for job in jobs] + clean_library_job_ids = set( + ( + await session.scalars( + select(ImportedFile.import_job_id) + .join(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .join(Issue, Issue.id == LibraryFile.issue_id) + .join(ImportJob, ImportJob.id == ImportedFile.import_job_id) + .where( + ImportedFile.import_job_id.in_(job_ids), + ImportJob.status == ImportJobStatus.COMPLETED, + ImportJob.archived_at.is_(None), + ImportedFile.status == ImportedFileStatus.IMPORTED, + ImportedFile.matched_issue_id == Issue.id, + LibraryFile.storage_mode == LibraryFileStorageMode.REFERENCED, + LibraryFile.file_path == ImportedFile.file_path, + ) + .distinct() + ) + ).all() + ) series_counts_result = await session.execute( select( ImportedSeries.import_job_id, @@ -270,6 +310,7 @@ async def _load_import_history_context( "job_history_metrics": job_history_metrics, "job_control_states": job_control_states, "job_resume_steps": job_resume_steps, + "clean_library_job_ids": clean_library_job_ids, "history_has_live_jobs": any(job.status in live_history_statuses for job in jobs), "history_stats": { "active": int(active_count or 0), @@ -282,4 +323,5 @@ async def _load_import_history_context( "clearable_jobs_total": clearable_jobs_total, "search_query": normalized_search, "sort": normalized_sort, + "show_archived": show_archived, } diff --git a/src/pullbox/ui/import_orphaned_routes.py b/src/pullbox/ui/import_orphaned_routes.py index 640c3518..8fbab0f1 100644 --- a/src/pullbox/ui/import_orphaned_routes.py +++ b/src/pullbox/ui/import_orphaned_routes.py @@ -63,6 +63,7 @@ async def load_import_orphaned_context( *, view: str, requested_page: int, + job_id: int | None = None, ) -> dict[str, object]: """Load unmatched-series page data for the requested view and page.""" from pullbox.composition.services import build_import_control_service @@ -103,6 +104,7 @@ async def load_import_orphaned_context( session, page=requested_page, page_size=_IMPORT_ORPHANED_PAGE_SIZE, + job_id=job_id, ) total_pages = max(1, (total + _IMPORT_ORPHANED_PAGE_SIZE - 1) // _IMPORT_ORPHANED_PAGE_SIZE) page = min(requested_page, total_pages) @@ -111,9 +113,10 @@ async def load_import_orphaned_context( session, page=page, page_size=_IMPORT_ORPHANED_PAGE_SIZE, + job_id=job_id, ) - orphaned_count = await svc.get_orphaned_count(session) + orphaned_count = await svc.get_orphaned_count(session, job_id=job_id) dismissed_q = ( select(func.count()) @@ -165,11 +168,19 @@ async def import_orphaned_cv_search( if query_text: parsed_query = parse_comicvine_series_query(query_text) api_key = await get_comicvine_api_key(session) - if api_key: + from pullbox.services.catalog.lookup import CatalogLookupService + from pullbox.services.catalog.reader import get_catalog_reader + + catalog = get_catalog_reader() + if api_key or catalog.available: try: - provider = wrap_comicvine_provider_for_ui_cache( - ComicVineProvider(api_key=api_key, rate_limit=10), - request, + provider = ( + CatalogLookupService(catalog) + if catalog.available + else wrap_comicvine_provider_for_ui_cache( + ComicVineProvider(api_key=api_key, rate_limit=10), + request, + ) ) cv_results, _total_results = await provider.search_series_globally( parsed_query.title_query, diff --git a/src/pullbox/ui/import_redirect_routes.py b/src/pullbox/ui/import_redirect_routes.py index 31335d3a..ec706a22 100644 --- a/src/pullbox/ui/import_redirect_routes.py +++ b/src/pullbox/ui/import_redirect_routes.py @@ -29,8 +29,8 @@ async def import_orphaned_redirect( tab: str = Query("all"), page: int = Query(1, ge=1), ) -> Response: - """Redirect legacy unmatched-series URLs to the unified Import workspace.""" - params = {"tab": "unmatched", "view": "dismissed" if tab == "dismissed" else "all"} + """Redirect legacy unmatched-series URLs to the Follow-up workspace.""" + params = {"tab": "follow-up", "view": "dismissed" if tab == "dismissed" else "all"} if page != 1: params["page"] = str(page) return RedirectResponse(url=f"/import?{urlencode(params)}", status_code=307) diff --git a/src/pullbox/ui/import_results_context.py b/src/pullbox/ui/import_results_context.py index 1ac69539..617243c9 100644 --- a/src/pullbox/ui/import_results_context.py +++ b/src/pullbox/ui/import_results_context.py @@ -6,20 +6,227 @@ from sqlalchemy import case, func, select +from pullbox.core.library_policy import load_effective_library_ingest_policy from pullbox.models.import_job import ( ImportedFile, ImportedFileStatus, ImportedSeries, ImportJob, + ImportJobAction, + ImportJobActionStatus, + ImportJobStatus, ImportSeriesStatus, + ImportSourceType, ) +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile, LibraryFileStorageMode, LibraryRoot from pullbox.models.series import IssueCatalogState, Series +from pullbox.models.story_arc import ( + ImportedStoryArcStatus, + IssueStoryArc, + StoryArcPlacement, + StoryArcPlacementMode, + StoryArcPlacementOwnership, + StoryArcPlacementState, + StoryArcResolutionState, + StoryArcSourceKind, +) +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry +from pullbox.models.story_arc_sync import StoryArcSyncWork, StoryArcSyncWorkState +from pullbox.services.import_completed_cleanup import ( + CompletedImportCleanupAction, + summarize_completed_import_cleanup_scope, +) +from pullbox.services.import_misplaced_source_cleanup import ( + MisplacedSourceCleanupAction, + count_misplaced_source_cleanup_files, +) +from pullbox.services.import_safety_diagnostics import ( + ImportSafetyCategory, + import_safety_category_label, +) +from pullbox.services.import_terminal_recovery import allows_terminal_import_recovery from pullbox.services.import_workflow_state import import_control_state_for_job if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession +_STORY_ARC_ACTION_PAGE_SIZE = 1_000 +_FAILED_SAFETY_DETAIL_LIMIT = 100 +_STORY_ARC_MANAGED_ACTION = "story_arc_managed_placement_requested" +_STORY_ARC_REFERENCE_ACTION = "story_arc_referenced_placement_attached" +_MANAGED_PAYLOAD_KEYS = frozenset( + { + "schema_version", + "sync_work_id", + "membership_id", + "desired_generation", + "imported_story_arc_id", + "imported_story_arc_entry_id", + "source_import_job_id", + } +) +_REFERENCE_PAYLOAD_KEYS = frozenset( + { + "schema_version", + "journal_state", + "placement_id", + "issue_story_arc_id", + "imported_story_arc_entry_id", + "placement_path", + "source_kind", + "source_import_job_id", + "expected_after", + } +) +_MANAGED_PLACEMENT_MODES = frozenset( + { + StoryArcPlacementMode.COPY, + StoryArcPlacementMode.HARDLINK, + StoryArcPlacementMode.SYMLINK, + } +) + + +def _library_paths_overlap(first: str, second: str) -> bool: + from pathlib import Path + + first_path = Path(first).resolve(strict=False) + second_path = Path(second).resolve(strict=False) + return ( + first_path == second_path + or first_path.is_relative_to(second_path) + or second_path.is_relative_to(first_path) + ) + + +async def load_clean_library_summary( + session: AsyncSession, + job_id: int, +) -> dict[str, object]: + active_clean_job: ImportJob | None = None + active_jobs = list( + ( + await session.scalars( + select(ImportJob) + .where( + ImportJob.status.in_( + { + ImportJobStatus.IMPORTING, + ImportJobStatus.PAUSING, + ImportJobStatus.PAUSED, + ImportJobStatus.STALLED, + ImportJobStatus.CANCELLING, + ImportJobStatus.ROLLING_BACK, + } + ) + ) + .order_by(ImportJob.id.desc()) + ) + ).all() + ) + for candidate in active_jobs: + progress = dict(candidate.progress_snapshot or {}) + if ( + progress.get("clean_library_adoption") is True + and int(progress.get("source_import_job_id") or 0) == job_id + ): + active_clean_job = candidate + break + active_payload = ( + { + "id": int(active_clean_job.id), + "status": active_clean_job.status.value, + "progress_snapshot": dict(active_clean_job.progress_snapshot or {}), + } + if active_clean_job is not None + else None + ) + eligibility = ( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.IMPORTED, + ImportedFile.matched_issue_id == Issue.id, + ImportedFile.library_file_id == LibraryFile.id, + LibraryFile.issue_id == Issue.id, + LibraryFile.storage_mode == LibraryFileStorageMode.REFERENCED, + LibraryFile.file_path == ImportedFile.file_path, + ) + count_row = ( + await session.execute( + select( + func.count(ImportedFile.id), + func.count(func.distinct(Issue.series_id)), + func.coalesce(func.sum(LibraryFile.file_size), 0), + ) + .select_from(ImportedFile) + .join(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .join(Issue, Issue.id == LibraryFile.issue_id) + .where(*eligibility) + ) + ).one() + reference_count = int(count_row[0] or 0) + if reference_count == 0: + active_snapshot = dict(active_clean_job.progress_snapshot or {}) if active_clean_job else {} + source_snapshot = active_snapshot.get("clean_library_source_snapshot") + source_snapshot = source_snapshot if isinstance(source_snapshot, dict) else {} + return { + "clean_library_reference_count": int(source_snapshot.get("file_count") or 0), + "clean_library_reference_series_count": int(source_snapshot.get("series_count") or 0), + "clean_library_reference_bytes": int(source_snapshot.get("total_bytes") or 0), + "clean_library_target_roots": [], + "clean_library_active_job": active_payload, + } + source_root_paths = set( + ( + await session.scalars( + select(LibraryRoot.path) + .select_from(ImportedFile) + .join(LibraryFile, LibraryFile.id == ImportedFile.library_file_id) + .join(Issue, Issue.id == LibraryFile.issue_id) + .join(LibraryRoot, LibraryRoot.id == LibraryFile.library_root_id) + .where(*eligibility) + .distinct() + ) + ).all() + ) + roots = list( + ( + await session.scalars( + select(LibraryRoot) + .where( + LibraryRoot.enabled.is_(True), + LibraryRoot.allow_managed_writes.is_(True), + ) + .order_by(LibraryRoot.name, LibraryRoot.id) + ) + ).all() + ) + targets: list[dict[str, object]] = [] + for root in roots: + if any(_library_paths_overlap(root.path, source) for source in source_root_paths): + continue + policy = await load_effective_library_ingest_policy(session, root) + targets.append( + { + "id": root.id, + "name": root.name, + "path": root.path, + "rename_on_import": policy.rename_on_import, + "normalize_to_cbz": policy.normalize_imported_archives_to_cbz, + "update_comicinfo": policy.update_embedded_comicinfo_from_match, + "skip_existing": policy.skip_existing_files, + } + ) + return { + "clean_library_reference_count": reference_count, + "clean_library_reference_series_count": int(count_row[1] or 0), + "clean_library_reference_bytes": int(count_row[2] or 0), + "clean_library_target_roots": targets, + "clean_library_active_job": active_payload, + } + + async def _count_series_status( session: AsyncSession, job_id: int, @@ -61,16 +268,236 @@ async def _load_files_for_status( session: AsyncSession, job_id: int, status: ImportedFileStatus, + *, + limit: int | None = None, ) -> list[ImportedFile]: - result = await session.execute( - select(ImportedFile).where( + query = ( + select(ImportedFile) + .where( ImportedFile.import_job_id == job_id, ImportedFile.status == status, ) + .order_by(ImportedFile.id) ) + if limit is not None: + query = query.limit(limit) + result = await session.execute(query) return list(result.scalars().all()) +_SAFETY_ACTION_BY_CATEGORY = { + ImportSafetyCategory.SOURCE_MISSING: ( + CompletedImportCleanupAction.DISMISS_MISSING_REFERENCES, + "safe_action", + ), + ImportSafetyCategory.SINGLE_PAGE_COMIC: ( + CompletedImportCleanupAction.SKIP_PROBABLE_COVERS, + "safe_action", + ), + ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT: ( + CompletedImportCleanupAction.ALLOW_OVERSIZED_FILES, + "safe_action", + ), + ImportSafetyCategory.PERMISSION_UNREADABLE: ( + CompletedImportCleanupAction.RETRY_SOURCE_INSPECTION, + "safe_action", + ), + ImportSafetyCategory.ARCHIVE_INSPECTION_FAILED: ( + CompletedImportCleanupAction.RETRY_SOURCE_INSPECTION, + "safe_action", + ), + ImportSafetyCategory.SOURCE_CHANGED: ( + CompletedImportCleanupAction.RETRY_SOURCE_INSPECTION, + "safe_action", + ), + ImportSafetyCategory.ZERO_BYTE: ( + CompletedImportCleanupAction.SKIP_UNUSABLE_FILES, + "safe_action", + ), + ImportSafetyCategory.ARCHIVE_NO_PAGES: ( + CompletedImportCleanupAction.SKIP_UNUSABLE_FILES, + "safe_action", + ), + ImportSafetyCategory.UNSUPPORTED_FILE_TYPE: ( + CompletedImportCleanupAction.SKIP_UNUSABLE_FILES, + "safe_action", + ), +} + +_CLEANUP_ACTION_PRESENTATION = { + CompletedImportCleanupAction.RECHECK_DEFERRED_FILES: { + "label": "Recheck deferred files", + "description": ( + "Group repeated file records, recognize completed imports, and recover exact issue " + "matches. Missing series catalogs are checked in the background. " + "Files that still need a decision remain here." + ), + "button_label": "Recheck files", + "tone": "warning", + }, + CompletedImportCleanupAction.DISMISS_MISSING_REFERENCES: { + "label": "Dismiss stale Mylar references", + "description": ( + "Clear database references whose source files no longer exist. No files are deleted." + ), + "button_label": "Dismiss references", + "tone": "neutral", + }, + CompletedImportCleanupAction.SKIP_PROBABLE_COVERS: { + "label": "Skip one-page archives", + "description": ( + "Exclude one-page image archives from this import while preserving the source files. " + "They may be cover art, damaged archives, or intentional one-page comics." + ), + "button_label": "Skip from import", + "tone": "neutral", + }, + CompletedImportCleanupAction.SKIP_UNUSABLE_FILES: { + "label": "Skip unusable files", + "description": ( + "Clear empty, unsupported, or page-less files that cannot become library issues." + ), + "button_label": "Skip unusable files", + "tone": "neutral", + }, + CompletedImportCleanupAction.ALLOW_OVERSIZED_FILES: { + "label": "Allow oversized files once", + "description": ( + "Retry legitimate large books once without weakening the global archive safety policy." + ), + "button_label": "Allow once and retry", + "tone": "warning", + }, + CompletedImportCleanupAction.RETRY_SOURCE_INSPECTION: { + "label": "Retry source inspection", + "description": ( + "Recheck files that were unreadable, changed, or could not be " + "inspected during the original run." + ), + "button_label": "Recheck sources", + "tone": "warning", + }, + CompletedImportCleanupAction.NORMALIZE_ALREADY_OWNED: { + "label": "Recognize already-owned issues", + "description": ( + "Resolve conflicts that point to issues already registered in the Pullbox library." + ), + "button_label": "Mark already owned", + "tone": "neutral", + }, + CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS: { + "label": "Accept recommended conflict choices", + "description": ( + "Import the single high-confidence preferred file in each eligible " + "conflict group and skip its alternatives." + ), + "button_label": "Accept recommendations", + "tone": "warning", + }, + CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES: { + "label": "Resolve mixed-folder files", + "description": ( + "Use exact embedded ComicInfo identity to assign misplaced files to the correct " + "Pullbox series and issue. Mylar folders and source files remain unchanged." + ), + "button_label": "Resolve and retry", + "tone": "warning", + }, + CompletedImportCleanupAction.RECOVER_KNOWN_SERIES: { + "label": "Recover known series", + "description": ( + "Retry files with agreeing saved series and issue IDs from an older import. " + "Conflicting files, skips, and safety decisions stay in Follow-up. " + "Source files remain unchanged." + ), + "button_label": "Recover and retry", + "tone": "warning", + }, +} + + +async def _load_cleanup_action_summaries( + session: AsyncSession, + job_id: int, +) -> list[dict[str, object]]: + summaries: list[dict[str, object]] = [] + for action, presentation in _CLEANUP_ACTION_PRESENTATION.items(): + summary = await summarize_completed_import_cleanup_scope( + session, + job_id, + action, + ) + if summary.affected_count == 0: + continue + summaries.append( + { + "action": action.value, + "affected_count": summary.affected_count, + "affected_file_count": summary.affected_file_count, + "item_unit": ( + "group" + if action is CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS + else "follow-up item" + if action is CompletedImportCleanupAction.RECHECK_DEFERRED_FILES + else "file" + ), + "examples": summary.examples, + **presentation, + } + ) + return summaries + + +async def _load_safety_category_summaries( + session: AsyncSession, + job_id: int, +) -> list[dict[str, object]]: + category_expression = ImportedFile.diagnostics["safety_block"]["category"].as_string() + rows = ( + await session.execute( + select(category_expression, func.count(ImportedFile.id)) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + ) + .group_by(category_expression) + .order_by(func.count(ImportedFile.id).desc(), category_expression.asc()) + ) + ).all() + summaries: list[dict[str, object]] = [] + for raw_category, count in rows: + try: + category = ImportSafetyCategory(str(raw_category)) + except ValueError: + category = ImportSafetyCategory.UNKNOWN + action, bucket = _SAFETY_ACTION_BY_CATEGORY.get(category, (None, "needs_review")) + examples = tuple( + ( + await session.scalars( + select(ImportedFile.file_name) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.SAFETY_BLOCKED, + category_expression == raw_category, + ) + .order_by(ImportedFile.id) + .limit(3) + ) + ).all() + ) + summaries.append( + { + "category": category.value, + "label": import_safety_category_label(category), + "count": int(count), + "examples": examples, + "action": action.value if action is not None else None, + "bucket": bucket, + } + ) + return summaries + + async def _orphaned_file_no_match_count(session: AsyncSession, job_id: int) -> int: return int( ( @@ -128,11 +555,363 @@ async def _load_catalog_sync_series(session: AsyncSession, job_id: int) -> list[ return list(result.unique().scalars().all()) +def _positive_int(value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None + + +def _managed_story_arc_action_is_completed( + *, + job_id: int, + action: ImportJobAction, + work: StoryArcSyncWork | None, + placements: list[StoryArcPlacement], + staged_arc: ImportedStoryArc | None, + staged_entry: ImportedStoryArcEntry | None, + membership: IssueStoryArc | None, + library_file: LibraryFile | None, +) -> bool: + payload = dict(action.payload or {}) + if ( + action.phase != "story_arc_placements" + or set(payload) != _MANAGED_PAYLOAD_KEYS + or _positive_int(payload.get("schema_version")) != 1 + or work is None + or work.state is not StoryArcSyncWorkState.COMPLETED + or work.origin_import_action_id != action.id + or work.origin_import_job_id != job_id + or _positive_int(payload.get("sync_work_id")) != work.id + or _positive_int(payload.get("membership_id")) != work.issue_story_arc_id + or payload.get("desired_generation") != work.desired_generation + or _positive_int(payload.get("imported_story_arc_id")) != work.origin_imported_story_arc_id + or _positive_int(payload.get("imported_story_arc_entry_id")) + != work.origin_imported_story_arc_entry_id + or _positive_int(payload.get("source_import_job_id")) != job_id + or staged_arc is None + or staged_arc.import_job_id != job_id + or staged_arc.status is not ImportedStoryArcStatus.IMPORTED + or staged_entry is None + or staged_entry.imported_story_arc_id != staged_arc.id + or staged_entry.materialized_membership_id != work.issue_story_arc_id + or staged_entry.resolution_state is not StoryArcResolutionState.RESOLVED + or membership is None + or staged_arc.materialized_story_arc_id != membership.story_arc_id + or library_file is None + or staged_entry.matched_issue_id != library_file.issue_id + or membership.issue_id != library_file.issue_id + or len(placements) != 1 + ): + return False + placement = placements[0] + return bool( + placement.issue_story_arc_id == work.issue_story_arc_id + and placement.library_file_id == work.library_file_id + and placement.source_import_job_id == job_id + and placement.creating_action_id == action.id + and placement.ownership is StoryArcPlacementOwnership.MANAGED + and placement.mode in _MANAGED_PLACEMENT_MODES + and placement.state is StoryArcPlacementState.CURRENT + and placement.source_kind is StoryArcSourceKind.PULLBOX + and placement.policy_schema_version == work.policy_schema_version + and placement.rendered_reading_order == work.membership_sequence + and placement.operation_token is None + and dict(placement.last_result or {}).get("status") == "complete" + ) + + +def _referenced_story_arc_action_is_completed( + *, + job_id: int, + action: ImportJobAction, + placements: list[StoryArcPlacement], + staged_arc: ImportedStoryArc | None, + staged_entry: ImportedStoryArcEntry | None, +) -> bool: + payload = dict(action.payload or {}) + placement_id = _positive_int(payload.get("placement_id")) + membership_id = _positive_int(payload.get("issue_story_arc_id")) + if ( + action.phase != "story_arcs" + or set(payload) != _REFERENCE_PAYLOAD_KEYS + or _positive_int(payload.get("schema_version")) != 1 + or payload.get("journal_state") != "completed" + or placement_id is None + or membership_id is None + or _positive_int(payload.get("source_import_job_id")) != job_id + or not isinstance(payload.get("expected_after"), dict) + or staged_arc is None + or staged_arc.import_job_id != job_id + or staged_arc.status is not ImportedStoryArcStatus.IMPORTED + or staged_entry is None + or staged_entry.imported_story_arc_id != staged_arc.id + or staged_entry.materialized_membership_id != membership_id + or staged_entry.resolution_state is not StoryArcResolutionState.RESOLVED + or staged_entry.source_kind.value != payload.get("source_kind") + or len(placements) != 1 + ): + return False + placement = placements[0] + return bool( + placement.id == placement_id + and placement.issue_story_arc_id == membership_id + and placement.placement_path == payload.get("placement_path") + and placement.mode is StoryArcPlacementMode.REFERENCE_ONLY + and placement.ownership is StoryArcPlacementOwnership.REFERENCED + and placement.source_kind.value == payload.get("source_kind") + and placement.source_import_job_id == job_id + and placement.creating_action_id == action.id + ) + + +async def _load_story_arc_ownership_counts( + session: AsyncSession, + job_id: int, +) -> dict[str, int]: + """Count only durable, completed, import-owned Story Arc placements.""" + counts = {"managed": 0, "referenced": 0} + after_action_id = 0 + while True: + actions = list( + ( + await session.scalars( + select(ImportJobAction) + .where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.status == ImportJobActionStatus.COMPLETED, + ImportJobAction.action_type.in_( + [_STORY_ARC_MANAGED_ACTION, _STORY_ARC_REFERENCE_ACTION] + ), + ImportJobAction.id > after_action_id, + ) + .order_by(ImportJobAction.id.asc()) + .limit(_STORY_ARC_ACTION_PAGE_SIZE) + ) + ).all() + ) + if not actions: + break + action_ids = [int(action.id) for action in actions] + works = list( + ( + await session.scalars( + select(StoryArcSyncWork).where( + StoryArcSyncWork.origin_import_action_id.in_(action_ids) + ) + ) + ).all() + ) + work_by_action_id = { + int(work.origin_import_action_id): work + for work in works + if work.origin_import_action_id is not None + } + placements_by_action_id: dict[int, list[StoryArcPlacement]] = {} + for placement in ( + await session.scalars( + select(StoryArcPlacement).where( + StoryArcPlacement.creating_action_id.in_(action_ids) + ) + ) + ).all(): + if placement.creating_action_id is not None: + placements_by_action_id.setdefault(int(placement.creating_action_id), []).append( + placement + ) + + imported_arc_ids = { + int(work.origin_imported_story_arc_id) + for work in works + if work.origin_imported_story_arc_id is not None + } + imported_entry_ids = { + int(work.origin_imported_story_arc_entry_id) + for work in works + if work.origin_imported_story_arc_entry_id is not None + } + for action in actions: + payload = dict(action.payload or {}) + arc_id = _positive_int(payload.get("imported_story_arc_id")) + entry_id = _positive_int(payload.get("imported_story_arc_entry_id")) + if arc_id is not None: + imported_arc_ids.add(arc_id) + if entry_id is not None: + imported_entry_ids.add(entry_id) + entries_by_id = { + int(entry.id): entry + for entry in ( + await session.scalars( + select(ImportedStoryArcEntry).where( + ImportedStoryArcEntry.id.in_(imported_entry_ids) + ) + ) + ).all() + } + imported_arc_ids.update( + int(entry.imported_story_arc_id) for entry in entries_by_id.values() + ) + arcs_by_id = { + int(arc.id): arc + for arc in ( + await session.scalars( + select(ImportedStoryArc).where(ImportedStoryArc.id.in_(imported_arc_ids)) + ) + ).all() + } + membership_ids = {int(work.issue_story_arc_id) for work in works} + library_file_ids = {int(work.library_file_id) for work in works} + memberships_by_id = { + int(membership.id): membership + for membership in ( + await session.scalars( + select(IssueStoryArc).where(IssueStoryArc.id.in_(membership_ids)) + ) + ).all() + } + library_files_by_id = { + int(library_file.id): library_file + for library_file in ( + await session.scalars( + select(LibraryFile).where(LibraryFile.id.in_(library_file_ids)) + ) + ).all() + } + for action in actions: + placements = placements_by_action_id.get(int(action.id), []) + if action.action_type == _STORY_ARC_REFERENCE_ACTION: + entry_id = _positive_int( + dict(action.payload or {}).get("imported_story_arc_entry_id") + ) + entry = entries_by_id.get(entry_id) if entry_id is not None else None + arc = ( + arcs_by_id.get(int(entry.imported_story_arc_id)) if entry is not None else None + ) + if _referenced_story_arc_action_is_completed( + job_id=job_id, + action=action, + placements=placements, + staged_arc=arc, + staged_entry=entry, + ): + counts["referenced"] += 1 + continue + work = work_by_action_id.get(int(action.id)) + if work is None: + continue + if _managed_story_arc_action_is_completed( + job_id=job_id, + action=action, + work=work, + placements=placements, + staged_arc=arcs_by_id.get(int(work.origin_imported_story_arc_id or 0)), + staged_entry=entries_by_id.get(int(work.origin_imported_story_arc_entry_id or 0)), + membership=memberships_by_id.get(int(work.issue_story_arc_id)), + library_file=library_files_by_id.get(int(work.library_file_id)), + ): + counts["managed"] += 1 + after_action_id = int(actions[-1].id) + return counts + + +async def _load_rollback_journal_summary( + session: AsyncSession, + job_id: int, +) -> dict[str, int]: + """Summarize file ownership without materializing a large action journal.""" + storage_mode = ImportJobAction.payload["storage_mode"].as_string() + transfer_method = ImportJobAction.payload["transfer_method"].as_string() + ownership = case( + ( + (storage_mode == "referenced") | (transfer_method == "leave_in_place"), + "referenced", + ), + else_="managed", + ).label("ownership") + result = await session.execute( + select( + ownership, + ImportJobAction.status, + func.count(ImportJobAction.id), + ) + .where( + ImportJobAction.import_job_id == job_id, + ImportJobAction.action_type == "library_file_registered", + ) + .group_by(ownership, ImportJobAction.status) + ) + + completed = {"managed": 0, "referenced": 0} + for owner, status, count in result.all(): + if status == ImportJobActionStatus.COMPLETED: + completed[str(owner)] = int(count or 0) + story_arc_completed = await _load_story_arc_ownership_counts(session, job_id) + completed["managed"] += story_arc_completed["managed"] + completed["referenced"] += story_arc_completed["referenced"] + + action_status_result = await session.execute( + select(ImportJobAction.status, func.count(ImportJobAction.id)) + .where(ImportJobAction.import_job_id == job_id) + .group_by(ImportJobAction.status) + ) + action_status_counts = {status: int(count or 0) for status, count in action_status_result.all()} + completed_action_count = action_status_counts.get(ImportJobActionStatus.COMPLETED, 0) + rolled_back_action_count = action_status_counts.get(ImportJobActionStatus.ROLLED_BACK, 0) + manual_recovery_count = action_status_counts.get( + ImportJobActionStatus.ROLLBACK_FAILED, + 0, + ) + return { + "managed_artifacts_created": completed["managed"], + "referenced_files_registered": completed["referenced"], + # These are journal candidates, not an assertion that the on-disk artifact + # is still unchanged. Rollback revalidates ownership and fingerprints. + "rollback_managed_candidates": completed["managed"], + "rollback_reference_candidates": completed["referenced"], + "rollback_manual_recovery_count": manual_recovery_count, + "rollback_action_count": sum(action_status_counts.values()), + "rollback_actions_pending": completed_action_count, + "rollback_actions_rolled_back": rolled_back_action_count, + } + + +async def _load_story_arc_results_summary( + session: AsyncSession, + job_id: int, +) -> dict[str, int]: + """Summarize created arcs separately from retained follow-up evidence.""" + created_count, follow_up_count = ( + await session.execute( + select( + func.sum( + case( + (ImportedStoryArc.materialized_story_arc_id.is_not(None), 1), + else_=0, + ) + ), + func.sum( + case( + ( + ImportedStoryArc.materialized_story_arc_id.is_(None) + & (ImportedStoryArc.status != ImportedStoryArcStatus.SKIPPED), + 1, + ), + else_=0, + ) + ), + ).where(ImportedStoryArc.import_job_id == job_id) + ) + ).one() + return { + "story_arcs_created_count": int(created_count or 0), + "story_arcs_follow_up_count": int(follow_up_count or 0), + } + + async def load_import_results_context( session: AsyncSession, job: ImportJob, + *, + include_clean_library: bool = True, ) -> dict[str, object]: - """Load aggregate counts and detail rows for the Step 5 results template.""" + """Load import results, optionally including History-only organizer data.""" job_id = int(job.id) imported_count = await _count_series_status(session, job_id, ImportSeriesStatus.IMPORTED) failed_count = await _count_series_status(session, job_id, ImportSeriesStatus.FAILED) @@ -201,10 +980,102 @@ async def load_import_results_context( file_status_counts.get(ImportedFileStatus.FAILED.value, 0), job.total_files_failed or 0, ) + files_skipped = file_status_counts.get(ImportedFileStatus.SKIPPED.value, 0) + failed_files = ( + await _load_files_for_status(session, job_id, ImportedFileStatus.FAILED) + if files_failed > 0 + else [] + ) + source_changed_files = sum( + 1 + for imported_file in failed_files + if dict(dict(imported_file.diagnostics or {}).get("source_revalidation") or {}).get("code") + == "source_changed" + ) files_safety_blocked = file_status_counts.get( ImportedFileStatus.SAFETY_BLOCKED.value, 0, ) + safety_blocked_files = ( + await _load_files_for_status( + session, + job_id, + ImportedFileStatus.SAFETY_BLOCKED, + limit=_FAILED_SAFETY_DETAIL_LIMIT, + ) + if files_safety_blocked > 0 and job.status is ImportJobStatus.FAILED + else [] + ) + safety_category_summaries = ( + await _load_safety_category_summaries(session, job_id) if files_safety_blocked > 0 else [] + ) + recovery_actions_available = allows_terminal_import_recovery(job) + cleanup_action_summaries = ( + await _load_cleanup_action_summaries(session, job_id) if recovery_actions_available else [] + ) + misplaced_source_restore_count = 0 + misplaced_source_duplicate_count = 0 + if ( + job.status is ImportJobStatus.COMPLETED + and job.archived_at is None + and job.source_type is ImportSourceType.MYLAR3 + ): + misplaced_source_restore_count = await count_misplaced_source_cleanup_files( + session, + job_id, + MisplacedSourceCleanupAction.RESTORE_RECORDED_PATH, + ) + misplaced_source_duplicate_count = await count_misplaced_source_cleanup_files( + session, + job_id, + MisplacedSourceCleanupAction.TRASH_IDENTICAL_DUPLICATE, + ) + clean_library_summary = ( + await load_clean_library_summary(session, job_id) + if include_clean_library + and job.status is ImportJobStatus.COMPLETED + and job.archived_at is None + else { + "clean_library_reference_count": 0, + "clean_library_reference_series_count": 0, + "clean_library_reference_bytes": 0, + "clean_library_target_roots": [], + } + ) + cleanup_by_action = {str(item["action"]): item for item in cleanup_action_summaries} + mixed_folder_summary = cleanup_by_action.get( + CompletedImportCleanupAction.RESOLVE_MIXED_FOLDER_FILES.value, + {}, + ) + clean_library_summary["clean_library_mixed_folder_repair_count"] = ( + _positive_int(mixed_folder_summary.get("affected_file_count")) or 0 + ) + recommended_summary = cleanup_by_action.get( + CompletedImportCleanupAction.ACCEPT_RECOMMENDED_CONFLICTS.value, + {}, + ) + already_owned_summary = cleanup_by_action.get( + CompletedImportCleanupAction.NORMALIZE_ALREADY_OWNED.value, + {}, + ) + recommended_conflict_groups = _positive_int(recommended_summary.get("affected_count")) or 0 + recommended_conflict_files = _positive_int(recommended_summary.get("affected_file_count")) or 0 + already_owned_conflict_files = ( + _positive_int(already_owned_summary.get("affected_file_count")) or 0 + ) + remaining_conflict_files = max( + files_conflict - recommended_conflict_files - already_owned_conflict_files, + 0, + ) + cleanup_safe_action_count = sum( + _positive_int(item["affected_file_count"]) or 0 for item in cleanup_action_summaries + ) + manual_safety_count = sum( + _positive_int(item["count"]) or 0 + for item in safety_category_summaries + if item.get("bucket") == "needs_review" + ) + cleanup_needs_review_count = manual_safety_count + remaining_conflict_files files_total = sum(file_status_counts.values()) orphaned_file_no_match_count = await _orphaned_file_no_match_count(session, job_id) identified_series_file_no_match_count = max( @@ -218,14 +1089,38 @@ async def load_import_results_context( if series.issue_catalog_state == IssueCatalogState.FAILED ) catalog_sync_pending_count = len(catalog_sync_series) - catalog_sync_failed_count + rollback_journal_summary = await _load_rollback_journal_summary(session, job_id) + story_arc_results_summary = await _load_story_arc_results_summary(session, job_id) + follow_up_group_count = ( + int(unmatched_queue_count > 0) + + len(cleanup_action_summaries) + + int(cleanup_needs_review_count > 0) + + int(misplaced_source_restore_count > 0) + + int(misplaced_source_duplicate_count > 0) + + int(failed_count > 0) + + int(files_failed > 0) + + int(job.status is ImportJobStatus.FAILED and files_safety_blocked > 0) + + int(story_arc_results_summary["story_arcs_follow_up_count"] > 0) + + int(catalog_sync_failed_count > 0) + ) + rollback_incomplete = bool( + rollback_journal_summary["rollback_manual_recovery_count"] + and job.status == ImportJobStatus.FAILED + and dict(job.progress_snapshot or {}).get("mode") == "rollback" + ) + can_rollback = bool(import_control_state_for_job(job).get("can_rollback")) and not ( + rollback_incomplete + ) return { - "can_rollback": bool(import_control_state_for_job(job).get("can_rollback")), + "can_rollback": can_rollback, + "rollback_incomplete": rollback_incomplete, "imported_count": imported_count, "failed_count": failed_count, "duplicate_count": duplicate_count, "no_match_count": no_match_count, "unmatched_queue_count": unmatched_queue_count, + "follow_up_group_count": follow_up_group_count, "failed_series": failed_series, "files_total": files_total, "files_imported": files_imported, @@ -241,19 +1136,30 @@ async def load_import_results_context( "catalog_sync_attention_count": len(catalog_sync_series), "catalog_sync_series": catalog_sync_series, "files_failed": files_failed, - "failed_files": ( - await _load_files_for_status(session, job_id, ImportedFileStatus.FAILED) - if files_failed > 0 - else [] - ), + "files_skipped": files_skipped, + "source_changed_files": source_changed_files, + "failed_files": failed_files, "files_safety_blocked": files_safety_blocked, - "safety_blocked_files": ( - await _load_files_for_status( - session, - job_id, - ImportedFileStatus.SAFETY_BLOCKED, - ) - if files_safety_blocked > 0 - else [] + # Completed results use category summaries. Failed jobs retain a + # bounded detail list so interrupted safety decisions remain actionable. + "safety_blocked_files": safety_blocked_files, + "safety_blocked_files_truncated": max( + files_safety_blocked - len(safety_blocked_files), + 0, ), + "safety_category_summaries": safety_category_summaries, + "cleanup_action_summaries": cleanup_action_summaries, + "recovery_actions_available": recovery_actions_available, + "recommended_conflict_groups": recommended_conflict_groups, + "recommended_conflict_files": recommended_conflict_files, + "already_owned_conflict_files": already_owned_conflict_files, + "remaining_conflict_files": remaining_conflict_files, + "cleanup_no_action_count": files_duplicate + files_already_owned + files_skipped, + "cleanup_safe_action_count": cleanup_safe_action_count, + "cleanup_needs_review_count": cleanup_needs_review_count, + "misplaced_source_restore_count": misplaced_source_restore_count, + "misplaced_source_duplicate_count": misplaced_source_duplicate_count, + **clean_library_summary, + **rollback_journal_summary, + **story_arc_results_summary, } diff --git a/src/pullbox/ui/import_review_context.py b/src/pullbox/ui/import_review_context.py index 4a20493b..366fb742 100644 --- a/src/pullbox/ui/import_review_context.py +++ b/src/pullbox/ui/import_review_context.py @@ -16,8 +16,18 @@ ) from pullbox.models.library import LibraryRoot from pullbox.services.import_review_selection import load_import_review_selection_state +from pullbox.services.import_safety_diagnostics import normalize_import_safety_diagnostics +from pullbox.services.import_split_series import load_selected_split_series_review +from pullbox.services.import_story_arc_review import ( + ImportedStoryArcReviewRow, + load_import_story_arc_review_page, +) +from pullbox.services.library_root_management import list_library_roots from pullbox.ui.import_conflict_review import _load_import_conflict_review_context -from pullbox.ui.import_review_summary import load_import_review_summary +from pullbox.ui.import_review_summary import ( + load_import_review_summary, + load_import_safety_failure_summary, +) from pullbox.ui.import_review_tables import ( _get_import_review_series_order_by, _load_import_review_file_detail_groups, @@ -28,6 +38,11 @@ _safety_blocked_files_filter, _series_conflict_kind_filter, ) +from pullbox.ui.import_story_arc_entry_review import ( + ImportedStoryArcEntryReviewRow, + StoryArcEntryResolutionFilter, + load_import_story_arc_entry_review_page, +) if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -47,7 +62,7 @@ def _resolve_review_view(status: str | None) -> tuple[str, ImportSeriesStatus | current_view = "series" if status == "conflicts": current_view = "conflicts" - elif status in {"needs_series", "needs_issue", "safety_blocked"}: + elif status in {"needs_series", "needs_issue", "safety_blocked", "story_arcs"}: current_view = status elif status: with contextlib.suppress(ValueError): @@ -165,8 +180,44 @@ async def _load_status_counts( return status_counts +async def _load_safety_review_counts( + session: AsyncSession, + job_id: int, +) -> tuple[int, int]: + row = ( + await session.execute( + select( + func.count(func.distinct(ImportedFile.import_series_id)), + func.count(ImportedFile.id), + ).where( + ImportedFile.import_job_id == job_id, + ImportedFile.status.in_( + (ImportedFileStatus.SAFETY_BLOCKED, ImportedFileStatus.SAFETY_APPROVED) + ), + ) + ) + ).one() + return int(row[0] or 0), int(row[1] or 0) + + +async def has_pending_import_safety_rematch(session: AsyncSession, job_id: int) -> bool: + """Return whether a review job has an approved file actively awaiting rematch.""" + pending_file_id = await session.scalar( + select(ImportedFile.id) + .join(ImportedSeries, ImportedSeries.id == ImportedFile.import_series_id) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.status == ImportedFileStatus.SAFETY_APPROVED, + ImportedSeries.diagnostics["rematch_pending"].as_boolean().is_(True), + ) + .limit(1) + ) + return pending_file_id is not None + + async def _load_safety_blocked_files_by_series_id( session: AsyncSession, + job_id: int, visible_series_ids: list[int], ) -> dict[int, list[ImportedFile]]: safety_blocked_files_by_series_id: dict[int, list[ImportedFile]] = { @@ -178,6 +229,7 @@ async def _load_safety_blocked_files_by_series_id( safety_files_result = await session.execute( select(ImportedFile) .where( + ImportedFile.import_job_id == job_id, ImportedFile.import_series_id.in_(visible_series_ids), ImportedFile.status.in_( [ImportedFileStatus.SAFETY_BLOCKED, ImportedFileStatus.SAFETY_APPROVED] @@ -193,6 +245,38 @@ async def _load_safety_blocked_files_by_series_id( return safety_blocked_files_by_series_id +def _build_safety_block_context_by_file_id( + files_by_series_id: Mapping[int, list[ImportedFile]], +) -> dict[int, dict[str, object]]: + """Normalize persisted/legacy blocks without mutating ORM diagnostics.""" + result: dict[int, dict[str, object]] = {} + for files in files_by_series_id.values(): + for imp_file in files: + diagnostics = imp_file.diagnostics or {} + if not isinstance(diagnostics, Mapping): + continue + safety_block = diagnostics.get("safety_block") + if isinstance(safety_block, Mapping): + result[imp_file.id] = normalize_import_safety_diagnostics(safety_block) + return result + + +def _build_safety_file_display_name_by_file_id( + files_by_series_id: Mapping[int, list[ImportedFile]], +) -> dict[int, str]: + """Return bounded basename-only labels for safety review rows.""" + result: dict[int, str] = {} + for files in files_by_series_id.values(): + for imp_file in files: + normalized = str(imp_file.file_name).replace("\\", "/").rstrip("/") + leaf = normalized.rsplit("/", maxsplit=1)[-1] + safe_leaf = "".join( + character for character in leaf if character >= " " and character != "\x7f" + ) + result[imp_file.id] = (safe_leaf or "File")[:200] + return result + + async def load_import_review_context( session: AsyncSession, job: ImportJob, @@ -200,6 +284,9 @@ async def load_import_review_context( status: str | None, page: int, sort: str | None, + story_arc_id: int | None = None, + arc_entry_state: StoryArcEntryResolutionFilter = StoryArcEntryResolutionFilter.ALL, + arc_entry_page: int = 1, ) -> dict[str, object]: """Load the template context for the Step 3 review table.""" job_id = int(job.id) @@ -212,6 +299,13 @@ async def load_import_review_context( conflict_review_ctx: dict[str, object] | None = None matched_file_targets_by_series_id: dict[int, list[dict[str, object]]] = {} review_file_groups_by_series_id: dict[int, list[dict[str, object]]] = {} + story_arc_items: tuple[ImportedStoryArcReviewRow, ...] = () + story_arc_total = 0 + story_arc_selected_item: ImportedStoryArcReviewRow | None = None + story_arc_entry_items: tuple[ImportedStoryArcEntryReviewRow, ...] = () + story_arc_entry_total = 0 + story_arc_entry_page_size = 25 + safety_blocked_files_by_series_id: dict[int, list[ImportedFile]] = {} safety_rematch_pending = False if current_view == "safety_blocked": @@ -228,7 +322,41 @@ async def load_import_review_context( current_view = "series" requested_series_status = None - if current_view == "conflicts": + if current_view == "story_arcs": + library_roots_result = await session.execute( + select(LibraryRoot).where(LibraryRoot.enabled.is_(True)).order_by(LibraryRoot.id) + ) + library_roots = list(library_roots_result.scalars().all()) + story_arc_page = await load_import_story_arc_review_page( + session, + job_id, + page=page, + page_size=page_size, + ) + story_arc_items = story_arc_page.items + story_arc_total = story_arc_page.total + total = story_arc_page.total + page = story_arc_page.page + page_size = story_arc_page.page_size + normalized_sort = "source_order" + story_arc_selected_item = next( + (item for item in story_arc_items if item.id == story_arc_id), + story_arc_items[0] if story_arc_items else None, + ) + if story_arc_selected_item is not None: + story_arc_entry_page_result = await load_import_story_arc_entry_review_page( + session, + job_id=job_id, + imported_story_arc_id=story_arc_selected_item.id, + resolution_state=arc_entry_state.resolution_state, + page=arc_entry_page, + page_size=story_arc_entry_page_size, + ) + story_arc_entry_items = story_arc_entry_page_result.items + story_arc_entry_total = story_arc_entry_page_result.total + arc_entry_page = story_arc_entry_page_result.page + story_arc_entry_page_size = story_arc_entry_page_result.page_size + elif current_view == "conflicts": conflict_review_ctx = await _load_import_conflict_review_context( job_id, session, @@ -239,7 +367,6 @@ async def load_import_review_context( page = _object_to_int(conflict_review_ctx["page"]) page_size = _object_to_int(conflict_review_ctx["page_size"]) normalized_sort = str(conflict_review_ctx["sort"]) - safety_blocked_files_by_series_id: dict[int, list[ImportedFile]] = {} else: filters = _review_filters( job_id=job_id, @@ -260,20 +387,18 @@ async def load_import_review_context( visible_series_ids = [item.id for item in series_items] safety_blocked_files_by_series_id = await _load_safety_blocked_files_by_series_id( session, + job_id, visible_series_ids, ) - safety_rematch_pending = any( - imp_file.status == ImportedFileStatus.SAFETY_APPROVED - for files in safety_blocked_files_by_series_id.values() - for imp_file in files - ) if visible_series_ids: matched_file_targets_by_series_id = await _load_import_review_matched_file_targets( session, + job_id, visible_series_ids, ) review_file_groups_by_series_id = await _load_import_review_file_detail_groups( session, + job_id, series_items, ) library_roots_result = await session.execute( @@ -281,9 +406,38 @@ async def load_import_review_context( ) library_roots = list(library_roots_result.scalars().all()) + split_series_review = await load_selected_split_series_review(session, job) + safety_review_series_count, safety_review_file_count = await _load_safety_review_counts( + session, job_id + ) + safety_rematch_pending = await has_pending_import_safety_rematch(session, job_id) + managed_library_root_options: list[dict[str, Any]] = [] + if split_series_review.requires_preferred_destination: + managed_library_root_options = [ + root + for root in await list_library_roots(session) + if bool(root["enabled"]) + and bool(root["allow_managed_writes"]) + and bool(root["available"]) + and bool(root["readable"]) + and bool(root["writable"]) + ] + template_ctx: dict[str, object] = { "job": job, "series_items": series_items, + "story_arc_items": story_arc_items, + "story_arc_total": story_arc_total, + "story_arc_selected_item": story_arc_selected_item, + "story_arc_selected_id": ( + story_arc_selected_item.id if story_arc_selected_item is not None else None + ), + "story_arc_entry_items": story_arc_entry_items, + "story_arc_entry_total": story_arc_entry_total, + "story_arc_entry_page": arc_entry_page, + "story_arc_entry_page_size": story_arc_entry_page_size, + "story_arc_entry_state_filter": arc_entry_state, + "story_arc_entry_state_options": tuple(StoryArcEntryResolutionFilter), "library_roots": library_roots, "total": total, "page": page, @@ -292,13 +446,24 @@ async def load_import_review_context( "current_view": current_view, "sort": normalized_sort, "status_counts": await _load_status_counts(session, job_id), + "safety_review_series_count": safety_review_series_count, + "safety_review_file_count": safety_review_file_count, "review_summary": await load_import_review_summary(session, job), + "split_series_review": split_series_review, + "managed_library_root_options": managed_library_root_options, + "safety_failure_summary": await load_import_safety_failure_summary(session, job), "selected_series_ids": await _load_selected_review_series_ids(session, job_id), "duplicate_selected_file_counts": await _load_duplicate_selected_file_counts( session, job_id, ), "safety_blocked_files_by_series_id": safety_blocked_files_by_series_id, + "safety_block_context_by_file_id": _build_safety_block_context_by_file_id( + safety_blocked_files_by_series_id + ), + "safety_file_display_name_by_file_id": _build_safety_file_display_name_by_file_id( + safety_blocked_files_by_series_id + ), "safety_rematch_pending": safety_rematch_pending, "matched_file_targets_by_series_id": matched_file_targets_by_series_id, "review_file_groups_by_series_id": review_file_groups_by_series_id, diff --git a/src/pullbox/ui/import_review_summary.py b/src/pullbox/ui/import_review_summary.py index 82f71711..858f119e 100644 --- a/src/pullbox/ui/import_review_summary.py +++ b/src/pullbox/ui/import_review_summary.py @@ -2,9 +2,10 @@ from __future__ import annotations +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, cast -from sqlalchemy import func, select +from sqlalchemy import func, or_, select from pullbox.models.import_job import ( ImportedFile, @@ -13,7 +14,14 @@ ImportJobStatus, ImportSeriesStatus, ) +from pullbox.models.story_arc import ImportedStoryArcStatus, StoryArcResolutionState +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry from pullbox.services.import_review_selection import load_import_review_selection_state +from pullbox.services.import_safety_diagnostics import ( + ImportSafetyCategory, + ImportSafetyFailureSummaryAccumulator, + normalize_import_safety_diagnostics, +) if TYPE_CHECKING: from sqlalchemy.ext.asyncio import AsyncSession @@ -103,6 +111,73 @@ async def load_import_review_summary( ) ).scalar_one() ) + story_arc_counts_result = await session.execute( + select(ImportedStoryArc.status, func.count(ImportedStoryArc.id)) + .where(ImportedStoryArc.import_job_id == job.id) + .group_by(ImportedStoryArc.status) + ) + story_arc_counts = { + status.value if hasattr(status, "value") else str(status): int(count) + for status, count in story_arc_counts_result.all() + } + story_arc_entry_counts_result = await session.execute( + select(ImportedStoryArcEntry.resolution_state, func.count(ImportedStoryArcEntry.id)) + .join( + ImportedStoryArc, + ImportedStoryArc.id == ImportedStoryArcEntry.imported_story_arc_id, + ) + .where(ImportedStoryArc.import_job_id == job.id) + .group_by(ImportedStoryArcEntry.resolution_state) + ) + story_arc_entry_counts = { + state.value if hasattr(state, "value") else str(state): int(count) + for state, count in story_arc_entry_counts_result.all() + } + story_arcs_selected = int( + await session.scalar( + select(func.count(ImportedStoryArc.id)).where( + ImportedStoryArc.import_job_id == job.id, + ImportedStoryArc.selected_for_import.is_(True), + ) + ) + or 0 + ) + story_arc_reviewable_statuses = { + ImportedStoryArcStatus.DETECTED.value, + ImportedStoryArcStatus.NEEDS_REVIEW.value, + ImportedStoryArcStatus.READY.value, + ImportedStoryArcStatus.CONFIRMED.value, + } + story_arcs_reviewable = sum( + count + for status, count in story_arc_counts.items() + if status in story_arc_reviewable_statuses + ) + attention_file_statuses = ( + ImportedFileStatus.NO_MATCH, + ImportedFileStatus.CONFLICT, + ImportedFileStatus.SAFETY_BLOCKED, + ImportedFileStatus.FAILED, + ) + needs_attention_series_total = int( + await session.scalar( + select(func.count(func.distinct(ImportedSeries.id))) + .outerjoin(ImportedFile, ImportedFile.import_series_id == ImportedSeries.id) + .where( + ImportedSeries.import_job_id == job.id, + or_( + ImportedSeries.status.in_( + [ImportSeriesStatus.NO_MATCH, ImportSeriesStatus.FAILED] + ), + ImportedFile.status.in_(attention_file_statuses), + ), + ) + ) + or 0 + ) + needs_attention_files_total = sum( + file_counts.get(status.value, 0) for status in attention_file_statuses + ) row_summary = { "series_total": sum(series_counts.values()), @@ -134,9 +209,50 @@ async def load_import_review_summary( "duplicate_series_importable": duplicate_importable_series_count, "duplicate_series_selected": duplicate_selected_series_count, "selected_series_total": matched_selected_series_count + duplicate_selected_series_count, + # Story Arcs are optional follow-up work, not canonical import items. "selected_items_total": _object_to_int(selection_state["selected_item_count"]), "importable_items_total": _object_to_int(selection_state["importable_item_count"]), + "ready_to_import_total": _object_to_int(selection_state["importable_item_count"]), + "needs_attention_total": needs_attention_series_total, + "needs_attention_files_total": needs_attention_files_total, "resolved_file_conflict_groups": resolved_file_conflict_groups, + "story_arcs_total": sum(story_arc_counts.values()), + "story_arcs_detected": story_arc_counts.get(ImportedStoryArcStatus.DETECTED.value, 0), + "story_arcs_needs_review": story_arc_counts.get( + ImportedStoryArcStatus.NEEDS_REVIEW.value, + 0, + ), + "story_arcs_ready": story_arc_counts.get(ImportedStoryArcStatus.READY.value, 0), + "story_arcs_selected": story_arcs_selected, + "story_arcs_reviewable": story_arcs_reviewable, + "deferred_story_arcs_total": story_arcs_reviewable, + "deferred_follow_up_total": story_arcs_reviewable, + "story_arcs_skipped": story_arc_counts.get(ImportedStoryArcStatus.SKIPPED.value, 0), + "story_arc_entries_total": sum(story_arc_entry_counts.values()), + "story_arc_entries_resolved": story_arc_entry_counts.get( + StoryArcResolutionState.RESOLVED.value, + 0, + ), + "story_arc_entries_missing": story_arc_entry_counts.get( + StoryArcResolutionState.MISSING.value, + 0, + ), + "story_arc_entries_ambiguous": story_arc_entry_counts.get( + StoryArcResolutionState.AMBIGUOUS.value, + 0, + ), + "story_arc_entries_conflict": story_arc_entry_counts.get( + StoryArcResolutionState.CONFLICT.value, + 0, + ), + "story_arc_entries_pending": story_arc_entry_counts.get( + StoryArcResolutionState.PENDING.value, + 0, + ), + "story_arc_entries_skipped": story_arc_entry_counts.get( + StoryArcResolutionState.SKIPPED.value, + 0, + ), "duplicate_files_duplicate": duplicate_file_counts.get( ImportedFileStatus.DUPLICATE_FILE.value, 0, @@ -195,3 +311,60 @@ async def load_import_review_summary( } return row_summary + + +async def load_import_safety_failure_summary( + session: AsyncSession, + job: ImportJob, + *, + page_size: int = 1_000, +) -> list[dict[str, object]]: + """Return complete safety categories from one bounded, narrow-field stream.""" + if page_size < 1 or page_size > 5_000: + raise ValueError("Safety summary page_size must be between 1 and 5000") + + accumulator = ImportSafetyFailureSummaryAccumulator() + bulk_overrideable_counts: dict[str, int] = {} + # Avoid rescanning/sorting failures for every page or loading their source metadata. + result = await session.stream( + select( + ImportedFile.file_name, + ImportedFile.diagnostics["safety_block"].label("safety_block"), + ImportedFile.diagnostics["source_revalidation"].label("source_revalidation"), + ImportedFile.status, + ) + .where( + ImportedFile.import_job_id == job.id, + ImportedFile.status.in_([ImportedFileStatus.SAFETY_BLOCKED, ImportedFileStatus.FAILED]), + ) + .order_by(ImportedFile.id) + .execution_options(yield_per=page_size) + ) + try: + async for rows in result.partitions(page_size): + for file_name, safety_block, source_revalidation, status in rows: + if isinstance(safety_block, Mapping): + accumulator.add(str(file_name), safety_block) + normalized = normalize_import_safety_diagnostics(safety_block) + category = str(normalized["category"]) + if ( + status == ImportedFileStatus.SAFETY_BLOCKED + and category == ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT.value + and normalized["overrideable"] is True + ): + bulk_overrideable_counts[category] = ( + bulk_overrideable_counts.get(category, 0) + 1 + ) + continue + if isinstance(source_revalidation, Mapping): + accumulator.add(str(file_name), source_revalidation) + finally: + await result.close() + + summaries = accumulator.summaries() + for summary in summaries: + category = str(summary["category"]) + bulk_overrideable_count = bulk_overrideable_counts.get(category, 0) + summary["bulk_overrideable_count"] = bulk_overrideable_count + summary["bulk_overrideable"] = bulk_overrideable_count > 0 + return summaries diff --git a/src/pullbox/ui/import_review_tables.py b/src/pullbox/ui/import_review_tables.py index 44fed8b1..3441d796 100644 --- a/src/pullbox/ui/import_review_tables.py +++ b/src/pullbox/ui/import_review_tables.py @@ -6,6 +6,7 @@ from sqlalchemy import ColumnElement, and_, case, func, or_, select +from pullbox.core.issue_numbers import format_issue_number from pullbox.models.import_job import ( ImportedFile, ImportedFileStatus, @@ -95,6 +96,7 @@ def _safety_blocked_files_filter() -> ColumnElement[bool]: return ( select(ImportedFile.id) .where( + ImportedFile.import_job_id == ImportedSeries.import_job_id, ImportedFile.import_series_id == ImportedSeries.id, ImportedFile.status.in_( [ImportedFileStatus.SAFETY_BLOCKED, ImportedFileStatus.SAFETY_APPROVED] @@ -226,9 +228,7 @@ def _format_import_review_issue_number(issue_number: object) -> str | None: numeric_issue = float(issue_number) except (TypeError, ValueError): return str(issue_number).strip() or None - if numeric_issue.is_integer(): - return str(int(numeric_issue)) - return f"{numeric_issue:g}" + return format_issue_number(numeric_issue) def _comicvine_issue_url(issue_cv_id: int) -> str: @@ -238,6 +238,7 @@ def _comicvine_issue_url(issue_cv_id: int) -> str: async def _load_import_review_matched_file_targets( session: AsyncSession, + job_id: int, series_ids: list[int], ) -> dict[int, list[dict[str, object]]]: """Load compact matched file-target rows for visible Step 3 review rows.""" @@ -247,6 +248,7 @@ async def _load_import_review_matched_file_targets( files_result = await session.execute( select(ImportedFile) .where( + ImportedFile.import_job_id == job_id, ImportedFile.import_series_id.in_(series_ids), ImportedFile.status.in_(_IMPORT_REVIEW_MATCHED_FILE_TARGET_STATUSES), ) @@ -392,6 +394,7 @@ def _import_review_file_group_key( async def _load_import_review_file_detail_groups( session: AsyncSession, + job_id: int, series_items: list[ImportedSeries], ) -> dict[int, list[dict[str, object]]]: """Load grouped per-file details for visible Step 3 review rows.""" @@ -402,7 +405,10 @@ async def _load_import_review_file_detail_groups( series_ids = list(series_by_id) files_result = await session.execute( select(ImportedFile) - .where(ImportedFile.import_series_id.in_(series_ids)) + .where( + ImportedFile.import_job_id == job_id, + ImportedFile.import_series_id.in_(series_ids), + ) .order_by( ImportedFile.import_series_id.asc(), case( diff --git a/src/pullbox/ui/import_routes.py b/src/pullbox/ui/import_routes.py index 159bdee6..ed5a313d 100644 --- a/src/pullbox/ui/import_routes.py +++ b/src/pullbox/ui/import_routes.py @@ -1,15 +1,18 @@ """Import workspace UI routes and loaders.""" from collections.abc import Callable, Mapping +from html import escape +from typing import Annotated -from fastapi import APIRouter, Query, Request +from fastapi import APIRouter, Form, HTTPException, Query, Request from fastapi.responses import HTMLResponse, JSONResponse from fastapi.templating import Jinja2Templates from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from starlette.responses import Response -from pullbox.api.deps import AuthenticatedUser, DbSession +from pullbox.api.deps import AuthenticatedUser, DbSession, InteractiveOperatorUser +from pullbox.core.exceptions import NotFoundError, ValidationError from pullbox.models.import_job import ( ImportJob, ImportJobLog, @@ -17,7 +20,37 @@ ImportSeriesStatus, ) from pullbox.models.library import LibraryRoot -from pullbox.services.import_workflow_state import ACTIVE_IMPORT_JOB_STATUSES +from pullbox.services.audit_service import source_ip_from_request +from pullbox.services.import_completed_cleanup import ( + CompletedImportCleanupAction, + list_completed_import_cleanup_files, +) +from pullbox.services.import_misplaced_source_cleanup import ( + MisplacedSourceCleanupAction, + apply_misplaced_source_cleanup, + apply_verified_misplaced_source_cleanup, + list_misplaced_source_cleanup_files, + preview_misplaced_source_cleanup, + preview_verified_misplaced_source_cleanup, +) +from pullbox.services.import_safety_bulk_review import ( + ImportSafetyBulkInterruptedError, + ImportSafetyBulkPreview, + allow_import_safety_category_once, + preview_import_safety_category, + preview_import_safety_category_skip, + skip_import_safety_category, +) +from pullbox.services.import_safety_diagnostics import ImportSafetyCategory +from pullbox.services.import_safety_source_cleanup import ( + move_one_page_source_to_trash, + preview_one_page_source_cleanup, +) +from pullbox.services.import_workflow_state import ( + ACTIVE_IMPORT_JOB_STATUSES, + snapshot_mode_for_job, +) +from pullbox.tasks.import_task import trigger_import_safety_bulk_rematch from pullbox.ui import import_orphaned_routes from pullbox.ui.comicvine_series_search import ( COMICVINE_SERIES_SEARCH_LIMIT, @@ -29,15 +62,23 @@ wrap_comicvine_provider_for_ui_cache, ) from pullbox.ui.import_conflict_review import _load_import_conflict_review_context +from pullbox.ui.import_follow_up import ( + count_import_follow_up_jobs, + load_import_follow_up_context, +) from pullbox.ui.import_history import ( _history_resume_step_for_job, _load_import_history_context, ) from pullbox.ui.import_progress_snapshot import build_import_progress_snapshot from pullbox.ui.import_results_context import load_import_results_context -from pullbox.ui.import_review_context import load_import_review_context +from pullbox.ui.import_review_context import ( + has_pending_import_safety_rematch, + load_import_review_context, +) from pullbox.ui.import_review_summary import load_import_review_summary from pullbox.ui.import_series_details_context import load_import_series_details_context +from pullbox.ui.import_story_arc_entry_review import StoryArcEntryResolutionFilter router = APIRouter() @@ -126,7 +167,11 @@ def _can_resume_collection_job(job: ImportJob, requested_step: int | None) -> bo ImportJobStatus.ROLLING_BACK, ImportJobStatus.COMPLETED, ImportJobStatus.FAILED, - } and (job.status != ImportJobStatus.STALLED or job.import_started_at is not None) + } and ( + job.status != ImportJobStatus.STALLED + or job.import_started_at is not None + or snapshot_mode_for_job(job) in {"import", "rollback"} + ) if step == 3: return job.status == ImportJobStatus.REVIEW and job.import_started_at is None if step == 2: @@ -228,6 +273,17 @@ async def _load_import_collection_context(session: AsyncSession) -> dict[str, ob """Load the collection import wizard context.""" roots_result = await session.execute(select(LibraryRoot).order_by(LibraryRoot.name)) library_roots = list(roots_result.scalars().all()) + from pullbox.services.library_root_management import list_library_roots + + enabled_root_options = [ + root for root in await list_library_roots(session) if bool(root["enabled"]) + ] + enabled_root_options.sort( + key=lambda root: ( + not bool(root["is_default_managed_destination"]), + str(root["name"]).casefold(), + ) + ) jobs_result = await session.execute( select(ImportJob).order_by(ImportJob.created_at.desc()).limit(10) @@ -236,6 +292,7 @@ async def _load_import_collection_context(session: AsyncSession) -> dict[str, ob return { "library_roots": library_roots, + "library_root_options": enabled_root_options, "recent_jobs": recent_jobs, "resume_step": None, "resume_job_id": None, @@ -287,12 +344,8 @@ async def _load_import_progress_snapshot( async def _load_import_workspace_counts(session: AsyncSession) -> dict[str, object]: """Load shared counts used by the unified Import workspace tabs.""" - from pullbox.composition.services import build_import_control_service - - svc = build_import_control_service() - return { - "unmatched_count": await svc.get_orphaned_count(session), + "follow_up_count": await count_import_follow_up_jobs(session), } @@ -306,11 +359,16 @@ async def import_page( search: str = Query(""), sort: str = Query(""), page: int = Query(1, ge=1), + show_archived: bool = Query(False), + job_id: int | None = Query(None), resume_job_id: int | None = Query(None), resume_step: int | None = Query(None), ) -> Response: """Render the unified Import workspace and its tab-scoped partials.""" - normalized_tab = tab if tab in {"collection", "unmatched", "history"} else "collection" + normalized_tab = "follow-up" if tab in {"follow-up", "unmatched"} else tab + if normalized_tab not in {"collection", "follow-up", "history"}: + normalized_tab = "collection" + selected_follow_up_job_id = job_id if isinstance(job_id, int) else None workspace_ctx: dict[str, object] if normalized_tab == "history": @@ -319,12 +377,14 @@ async def import_page( search_query=search, sort=sort, requested_page=page, + show_archived=show_archived, ) - elif normalized_tab == "unmatched": - workspace_ctx = await _load_import_orphaned_context( + elif normalized_tab == "follow-up": + workspace_ctx = await load_import_follow_up_context( session, view=view, requested_page=page, + job_id=selected_follow_up_job_id, ) else: workspace_ctx = await _load_import_collection_context(session) @@ -384,7 +444,7 @@ async def import_page( "partials/import_history_panel_bundle.html", ctx, ) - if hx_target == "import-orphaned-results" and normalized_tab == "unmatched": + if hx_target == "import-orphaned-results" and normalized_tab == "follow-up": return _templates().TemplateResponse( request, "partials/import_orphaned_content_bundle.html", @@ -468,9 +528,71 @@ async def import_review_partial( status: str | None = Query(None), page: int = Query(1, ge=1), sort: str | None = Query(None), + story_arc_id: int | None = Query(None, ge=1), + arc_entry_state: StoryArcEntryResolutionFilter = StoryArcEntryResolutionFilter.ALL, + arc_entry_page: int = Query(1, ge=1), ) -> Response: """Render the review table partial for an import job.""" - from pullbox.core.exceptions import NotFoundError + + return await _render_import_review_partial( + job_id, + request, + user, + session, + status=status, + page=page, + sort=sort, + story_arc_id=story_arc_id, + arc_entry_state=arc_entry_state, + arc_entry_page=arc_entry_page, + ) + + +@router.get( + "/import/{job_id}/review-rematch-status", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_review_rematch_status( + job_id: int, + _user: AuthenticatedUser, + session: DbSession, +) -> Response: + """Poll safety rematch completion without replacing the review surface.""" + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + + pending = await has_pending_import_safety_rematch(session, job_id) + if pending: + escaped_job_id = escape(str(job_id), quote=True) + return HTMLResponse( + f'' + ) + + response = HTMLResponse('') + response.headers["HX-Trigger"] = "import:review-refresh" + return response + + +async def _render_import_review_partial( + job_id: int, + request: Request, + user: object, + session: AsyncSession, + *, + status: str | None, + page: int, + sort: str | None, + story_arc_id: int | None = None, + arc_entry_state: StoryArcEntryResolutionFilter = StoryArcEntryResolutionFilter.ALL, + arc_entry_page: int = 1, + extra_context: Mapping[str, object] | None = None, +) -> Response: + """Render the canonical review partial with optional route-local state.""" job = await session.get(ImportJob, job_id) if job is None: @@ -482,7 +604,12 @@ async def import_review_partial( status=status, page=page, sort=sort, + story_arc_id=story_arc_id, + arc_entry_state=arc_entry_state, + arc_entry_page=arc_entry_page, ) + if extra_context: + template_ctx.update(extra_context) return _templates().TemplateResponse( request, @@ -495,6 +622,347 @@ async def import_review_partial( ) +async def _load_import_safety_bulk_preview( + session: AsyncSession, + *, + job_id: int, + category: ImportSafetyCategory, + actor_id: int, +) -> ImportSafetyBulkPreview: + """Load an authoritative signed preview or hide unavailable actions.""" + try: + preview = await preview_import_safety_category( + session, + job_id, + category, + actor_id=actor_id, + ) + except ValidationError as exc: + raise HTTPException(status_code=409, detail=exc.message) from exc + if not preview.overrideable or preview.preview_token is None: + raise HTTPException(status_code=404, detail="Bulk safety action not available.") + return preview + + +async def _load_import_safety_bulk_skip_preview( + session: AsyncSession, + *, + job_id: int, + category: ImportSafetyCategory, + actor_id: int, +) -> ImportSafetyBulkPreview: + """Load an authoritative preview for a source-preserving category skip.""" + try: + preview = await preview_import_safety_category_skip( + session, + job_id, + category, + actor_id=actor_id, + ) + except ValidationError as exc: + raise HTTPException(status_code=409, detail=exc.message) from exc + if preview.preview_token is None: + raise HTTPException(status_code=404, detail="Bulk safety action not available.") + return preview + + +@router.get( + "/import/{job_id}/safety/categories/{category}/preview", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_review_preview_safety_category( + job_id: int, + category: ImportSafetyCategory, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, + status: str | None = Query("safety_blocked"), + page: int = Query(1, ge=1), + sort: str | None = Query(None), +) -> Response: + """Render a signed, exact, read-only category preview for Step 3.""" + preview = await _load_import_safety_bulk_preview( + session, + job_id=job_id, + category=category, + actor_id=user.id, + ) + return await _render_import_review_partial( + job_id, + request, + user, + session, + status=status, + page=page, + sort=sort, + extra_context={"safety_bulk_preview": preview}, + ) + + +@router.post( + "/import/{job_id}/safety/categories/{category}/allow-once", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_review_allow_safety_category_once( + job_id: int, + category: ImportSafetyCategory, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, + preview_token: Annotated[str, Form(min_length=1, max_length=4096)], + status: str | None = Query("safety_blocked"), + page: int = Query(1, ge=1), + sort: str | None = Query(None), +) -> Response: + """Apply one exact signed category preview, then refresh Step 3.""" + if category is not ImportSafetyCategory.DECOMPRESSION_SIZE_LIMIT: + raise HTTPException(status_code=404, detail="Bulk safety action not available.") + try: + await allow_import_safety_category_once( + session, + job_id, + category, + actor_id=user.id, + actor_username=user.username, + source_ip=source_ip_from_request(request), + preview_token=preview_token, + ) + except ImportSafetyBulkInterruptedError: + trigger_import_safety_bulk_rematch(job_id) + return await _render_import_review_partial( + job_id, + request, + user, + session, + status=status, + page=page, + sort=sort, + extra_context={ + "safety_bulk_error": ( + "The bulk action stopped because the import job changed. " + "The review below shows the latest state." + ), + "safety_bulk_error_category": category.value, + }, + ) + except ValidationError as exc: + await session.rollback() + return await _render_import_review_partial( + job_id, + request, + user, + session, + status=status, + page=page, + sort=sort, + extra_context={ + "safety_bulk_error": exc.message, + "safety_bulk_error_category": category.value, + }, + ) + + trigger_import_safety_bulk_rematch(job_id) + return await import_review_partial( + job_id, + request, + user, + session, + status=status, + page=page, + sort=sort, + ) + + +@router.get( + "/import/{job_id}/safety/categories/{category}/skip-preview", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_review_preview_safety_category_skip( + job_id: int, + category: ImportSafetyCategory, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, + status: str | None = Query("safety_blocked"), + page: int = Query(1, ge=1), + sort: str | None = Query(None), +) -> Response: + """Render a signed preview for excluding one-page archives from this import.""" + preview = await _load_import_safety_bulk_skip_preview( + session, + job_id=job_id, + category=category, + actor_id=user.id, + ) + return await _render_import_review_partial( + job_id, + request, + user, + session, + status=status, + page=page, + sort=sort, + extra_context={ + "safety_bulk_preview": preview, + "safety_bulk_action": "skip", + }, + ) + + +@router.post( + "/import/{job_id}/safety/categories/{category}/skip", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_review_skip_safety_category( + job_id: int, + category: ImportSafetyCategory, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, + preview_token: Annotated[str, Form(min_length=1, max_length=4096)], + status: str | None = Query("safety_blocked"), + page: int = Query(1, ge=1), + sort: str | None = Query(None), +) -> Response: + """Exclude a signed one-page archive set while leaving source files untouched.""" + try: + await skip_import_safety_category( + session, + job_id, + category, + actor_id=user.id, + actor_username=user.username, + source_ip=source_ip_from_request(request), + preview_token=preview_token, + ) + except (ImportSafetyBulkInterruptedError, ValidationError) as exc: + await session.rollback() + message = ( + exc.message + if isinstance(exc, ValidationError) + else "The bulk skip stopped because the import job changed. Review the latest state." + ) + return await _render_import_review_partial( + job_id, + request, + user, + session, + status=status, + page=page, + sort=sort, + extra_context={ + "safety_bulk_error": message, + "safety_bulk_error_category": category.value, + }, + ) + return await import_review_partial( + job_id, + request, + user, + session, + status=status, + page=page, + sort=sort, + ) + + +@router.get( + "/import/{job_id}/files/{file_id}/safety/source-cleanup-preview", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_review_source_cleanup_preview( + job_id: int, + file_id: int, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, + status: str | None = Query("safety_blocked"), + page: int = Query(1, ge=1), + sort: str | None = Query(None), + return_to: str = Query("review"), +) -> Response: + """Preview the explicitly destructive source cleanup for one one-page archive.""" + try: + preview = await preview_one_page_source_cleanup( + session, + job_id, + file_id, + actor_id=user.id, + ) + except ValidationError as exc: + raise HTTPException(status_code=409, detail=exc.message) from exc + return _templates().TemplateResponse( + request, + "partials/import_source_cleanup_modal.html", + _ctx( + request, + user, + preview=preview, + status=status or "safety_blocked", + page=page, + sort=sort or "confidence", + return_to=(return_to if return_to in {"results", "follow-up"} else "review"), + error="", + ), + ) + + +@router.post( + "/import/{job_id}/files/{file_id}/safety/source-cleanup", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_review_source_cleanup( + job_id: int, + file_id: int, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, + preview_token: Annotated[str, Form(min_length=1, max_length=4096)], + status: str | None = Query("safety_blocked"), + page: int = Query(1, ge=1), + sort: str | None = Query(None), + return_to: str = Query("review"), +) -> Response: + """Move one explicitly confirmed source file to configured Trash.""" + try: + await move_one_page_source_to_trash( + session, + job_id, + file_id, + actor_id=user.id, + actor_username=user.username, + source_ip=source_ip_from_request(request), + preview_token=preview_token, + ) + except ValidationError as exc: + await session.rollback() + raise HTTPException(status_code=409, detail=exc.message) from exc + if return_to == "results": + return Response( + status_code=204, + headers={ + "HX-Redirect": (f"/import?tab=collection&resume_job_id={job_id}&resume_step=5") + }, + ) + if return_to == "follow-up": + return await import_follow_up_job_partial(job_id, request, user, session) + return await import_review_partial( + job_id, + request, + user, + session, + status=status, + page=page, + sort=sort, + ) + + @router.post( "/import/{job_id}/files/{file_id}/safety/allow-once", response_class=HTMLResponse, @@ -635,7 +1103,11 @@ async def import_results_partial( if job.status not in {ImportJobStatus.COMPLETED, ImportJobStatus.FAILED}: raise ValidationError("Results are only available for completed or failed imports.") progress_snapshot = await _load_import_progress_snapshot(session, job) - results_context = await load_import_results_context(session, job) + results_context = await load_import_results_context( + session, + job, + include_clean_library=False, + ) return _templates().TemplateResponse( request, @@ -652,6 +1124,239 @@ async def import_results_partial( ) +async def import_follow_up_job_partial( + job_id: int, + request: Request, + user: AuthenticatedUser, + session: DbSession, +) -> Response: + """Refresh one job-scoped Follow-up surface after a mutation.""" + follow_up_context = await load_import_follow_up_context( + session, + view="all", + requested_page=1, + job_id=job_id, + ) + return _templates().TemplateResponse( + request, + "partials/import_orphaned_results.html", + _ctx(request, user, tab="follow-up", **follow_up_context), + ) + + +@router.get( + "/import/{job_id}/clean-library-panel", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_clean_library_panel( + job_id: int, + request: Request, + user: AuthenticatedUser, + session: DbSession, +) -> Response: + """Render optional clean-library organization from Import History.""" + job = await session.get(ImportJob, job_id) + if job is None: + raise NotFoundError("ImportJob", job_id) + if job.status is not ImportJobStatus.COMPLETED or job.archived_at is not None: + raise ValidationError("Library organization is available for current completed imports.") + results_context = await load_import_results_context( + session, + job, + include_clean_library=True, + ) + return _templates().TemplateResponse( + request, + "partials/import_clean_library_modal.html", + _ctx(request, user, job=job, **results_context), + ) + + +@router.get( + "/import/{job_id}/cleanup/{action}/files", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_completed_cleanup_files_partial( + job_id: int, + action: CompletedImportCleanupAction, + request: Request, + user: AuthenticatedUser, + session: DbSession, + page: int = Query(1, ge=1), +) -> Response: + """Render one bounded page of files in a completed cleanup scope.""" + cleanup_page = await list_completed_import_cleanup_files( + session, + job_id, + action, + page=page, + ) + return _templates().TemplateResponse( + request, + "partials/import_completed_cleanup_files.html", + _ctx( + request, + user, + job_id=job_id, + cleanup_action=action.value, + cleanup_page=cleanup_page, + ), + ) + + +@router.get( + "/import/{job_id}/misplaced-source-cleanup/{action}/files", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_misplaced_source_cleanup_files_partial( + job_id: int, + action: MisplacedSourceCleanupAction, + request: Request, + user: AuthenticatedUser, + session: DbSession, + page: int = Query(1, ge=1), +) -> Response: + """Render one bounded page of optional source cleanup candidates.""" + cleanup_page = await list_misplaced_source_cleanup_files( + session, + job_id, + action, + page=page, + ) + return _templates().TemplateResponse( + request, + "partials/import_misplaced_source_cleanup_files.html", + _ctx( + request, + user, + job_id=job_id, + cleanup_action=action.value, + cleanup_page=cleanup_page, + ), + ) + + +@router.get( + "/import/{job_id}/misplaced-source-cleanup/restore-recorded-path/preview-all", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_misplaced_source_cleanup_preview_all( + job_id: int, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, +) -> Response: + """Preview moving every currently eligible verified misplaced source.""" + try: + preview = await preview_verified_misplaced_source_cleanup( + session, + job_id, + actor_id=user.id, + ) + except ValidationError as exc: + raise HTTPException(status_code=409, detail=exc.message) from exc + return _templates().TemplateResponse( + request, + "partials/import_misplaced_source_cleanup_bulk_modal.html", + _ctx(request, user, preview=preview), + ) + + +@router.post( + "/import/{job_id}/misplaced-source-cleanup/restore-recorded-path/apply-all", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_misplaced_source_cleanup_apply_all( + job_id: int, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, + preview_token: Annotated[str, Form(min_length=1, max_length=4096)], +) -> Response: + """Apply a signed exact-scope move and refresh Step 5 without navigation.""" + try: + await apply_verified_misplaced_source_cleanup( + session, + job_id, + actor_id=user.id, + actor_username=user.username, + source_ip=source_ip_from_request(request), + preview_token=preview_token, + ) + except ValidationError as exc: + await session.rollback() + raise HTTPException(status_code=409, detail=exc.message) from exc + return await import_follow_up_job_partial(job_id, request, user, session) + + +@router.get( + "/import/{job_id}/files/{file_id}/misplaced-source-cleanup/{action}/preview", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_misplaced_source_cleanup_preview( + job_id: int, + file_id: int, + action: MisplacedSourceCleanupAction, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, +) -> Response: + """Preview one physical source cleanup without changing the library.""" + try: + preview = await preview_misplaced_source_cleanup( + session, + job_id, + file_id, + action, + actor_id=user.id, + ) + except ValidationError as exc: + raise HTTPException(status_code=409, detail=exc.message) from exc + return _templates().TemplateResponse( + request, + "partials/import_misplaced_source_cleanup_modal.html", + _ctx(request, user, preview=preview, error=""), + ) + + +@router.post( + "/import/{job_id}/files/{file_id}/misplaced-source-cleanup/{action}", + response_class=HTMLResponse, + include_in_schema=False, +) +async def import_misplaced_source_cleanup_apply( + job_id: int, + file_id: int, + action: MisplacedSourceCleanupAction, + request: Request, + user: InteractiveOperatorUser, + session: DbSession, + preview_token: Annotated[str, Form(min_length=1, max_length=4096)], +) -> Response: + """Apply one signed source cleanup and return to the completed results.""" + try: + await apply_misplaced_source_cleanup( + session, + job_id, + file_id, + action, + actor_id=user.id, + actor_username=user.username, + source_ip=source_ip_from_request(request), + preview_token=preview_token, + ) + except ValidationError as exc: + await session.rollback() + raise HTTPException(status_code=409, detail=exc.message) from exc + return await import_follow_up_job_partial(job_id, request, user, session) + + @router.get( "/import/{job_id}/series/{series_id}/details-partial", response_class=HTMLResponse, @@ -777,11 +1482,19 @@ async def import_cv_search( if query_text: parsed_query = parse_comicvine_series_query(query_text) api_key = await get_comicvine_api_key(session) - if api_key: + from pullbox.services.catalog.lookup import CatalogLookupService + from pullbox.services.catalog.reader import get_catalog_reader + + catalog = get_catalog_reader() + if api_key or catalog.available: try: - provider = wrap_comicvine_provider_for_ui_cache( - ComicVineProvider(api_key=api_key, rate_limit=10), - request, + provider = ( + CatalogLookupService(catalog) + if catalog.available + else wrap_comicvine_provider_for_ui_cache( + ComicVineProvider(api_key=api_key, rate_limit=10), + request, + ) ) cv_results, _total_results = await provider.search_series_globally( parsed_query.title_query, diff --git a/src/pullbox/ui/import_series_details_context.py b/src/pullbox/ui/import_series_details_context.py index 593588b6..c5a77056 100644 --- a/src/pullbox/ui/import_series_details_context.py +++ b/src/pullbox/ui/import_series_details_context.py @@ -7,6 +7,7 @@ from sqlalchemy import case, select from pullbox.core.exceptions import NotFoundError +from pullbox.core.issue_numbers import format_issue_number from pullbox.core.release_parser import parse_release_title from pullbox.core.source_metadata import SourceMetadataExtractor from pullbox.models.import_job import ( @@ -37,9 +38,7 @@ def is_actionable_duplicate_merge(series_item: ImportedSeries | None) -> bool: def _format_issue_number(issue_number: float | None) -> str | None: if issue_number is None: return None - if float(issue_number).is_integer(): - return str(int(issue_number)) - return f"{issue_number:g}" + return format_issue_number(issue_number) def _duplicate_reason_label(reason: str | None) -> str: diff --git a/src/pullbox/ui/import_story_arc_entry_review.py b/src/pullbox/ui/import_story_arc_entry_review.py new file mode 100644 index 00000000..f4e0138a --- /dev/null +++ b/src/pullbox/ui/import_story_arc_entry_review.py @@ -0,0 +1,163 @@ +"""Bounded presentation queries for Step 3 story-arc entries.""" + +from __future__ import annotations + +import enum +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from sqlalchemy import func, select + +from pullbox.core.exceptions import ValidationError +from pullbox.core.issue_numbers import format_issue_number +from pullbox.models.issue import Issue +from pullbox.models.series import Series +from pullbox.models.story_arc import StoryArcResolutionState, StoryArcSourceKind +from pullbox.models.story_arc_import import ImportedStoryArc, ImportedStoryArcEntry + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + + +class StoryArcEntryResolutionFilter(enum.StrEnum): + """Typed UI filter for all or one staged-entry resolution state.""" + + ALL = "all" + PENDING = StoryArcResolutionState.PENDING.value + RESOLVED = StoryArcResolutionState.RESOLVED.value + MISSING = StoryArcResolutionState.MISSING.value + AMBIGUOUS = StoryArcResolutionState.AMBIGUOUS.value + CONFLICT = StoryArcResolutionState.CONFLICT.value + SKIPPED = StoryArcResolutionState.SKIPPED.value + + @property + def resolution_state(self) -> StoryArcResolutionState | None: + """Return the database state represented by this UI filter.""" + if self is StoryArcEntryResolutionFilter.ALL: + return None + return StoryArcResolutionState(self.value) + + +@dataclass(frozen=True, slots=True) +class ImportedStoryArcEntryReviewRow: + """Safe presentation evidence for one staged story-arc entry.""" + + id: int + source_ordinal: int + reading_order: int | None + reading_order_text: str | None + source_kind: StoryArcSourceKind + source_series_name: str | None + source_issue_number_text: str | None + source_issue_title: str | None + resolution_state: StoryArcResolutionState + resolution_method: str | None + resolution_confidence: float | None + matched_issue_id: int | None + matched_series_title: str | None + matched_issue_number_text: str | None + matched_issue_title: str | None + source_location_present: bool + selected_for_import: bool + + +@dataclass(frozen=True, slots=True) +class ImportedStoryArcEntryReviewPage: + """One independently paginated entry page for a visible staged arc.""" + + items: tuple[ImportedStoryArcEntryReviewRow, ...] + total: int + page: int + page_size: int + + +async def load_import_story_arc_entry_review_page( + session: AsyncSession, + *, + job_id: int, + imported_story_arc_id: int, + resolution_state: StoryArcResolutionState | None, + page: int = 1, + page_size: int = 25, +) -> ImportedStoryArcEntryReviewPage: + """Load one entry page with a constant count-and-page query shape.""" + if page < 1: + raise ValidationError("Story arc entry page must be at least 1") + if page_size < 1 or page_size > 100: + raise ValidationError("Story arc entry page_size must be between 1 and 100") + + filters = [ + ImportedStoryArc.import_job_id == job_id, + ImportedStoryArcEntry.imported_story_arc_id == imported_story_arc_id, + ] + if resolution_state is not None: + filters.append(ImportedStoryArcEntry.resolution_state == resolution_state) + + total = int( + await session.scalar( + select(func.count(ImportedStoryArcEntry.id)) + .join( + ImportedStoryArc, + ImportedStoryArc.id == ImportedStoryArcEntry.imported_story_arc_id, + ) + .where(*filters) + ) + or 0 + ) + total_pages = max(1, (total + page_size - 1) // page_size) + bounded_page = min(page, total_pages) + + result = await session.execute( + select(ImportedStoryArcEntry, Issue, Series) + .join( + ImportedStoryArc, + ImportedStoryArc.id == ImportedStoryArcEntry.imported_story_arc_id, + ) + .outerjoin(Issue, Issue.id == ImportedStoryArcEntry.matched_issue_id) + .outerjoin(Series, Series.id == Issue.series_id) + .where(*filters) + .order_by( + ImportedStoryArcEntry.source_ordinal.asc(), + ImportedStoryArcEntry.id.asc(), + ) + .offset((bounded_page - 1) * page_size) + .limit(page_size) + ) + + rows: list[ImportedStoryArcEntryReviewRow] = [] + for entry, issue, series in result.all(): + matched_issue_number_text: str | None = None + if issue is not None: + matched_issue_number_text = issue.issue_number_text or format_issue_number( + issue.issue_number + ) + rows.append( + ImportedStoryArcEntryReviewRow( + id=int(entry.id), + source_ordinal=int(entry.source_ordinal), + reading_order=entry.reading_order, + reading_order_text=entry.reading_order_raw, + source_kind=entry.source_kind, + source_series_name=entry.source_series_name, + source_issue_number_text=entry.source_issue_number_text, + source_issue_title=entry.source_issue_title, + resolution_state=entry.resolution_state, + resolution_method=entry.resolution_method, + resolution_confidence=entry.resolution_confidence, + matched_issue_id=entry.matched_issue_id, + matched_series_title=series.title if series is not None else None, + matched_issue_number_text=matched_issue_number_text, + matched_issue_title=issue.title if issue is not None else None, + source_location_present=bool( + entry.source_location and entry.source_location.strip() + ), + selected_for_import=bool(entry.selected_for_import), + ) + ) + + return ImportedStoryArcEntryReviewPage( + items=tuple(rows), + total=total, + page=bounded_page, + page_size=page_size, + ) diff --git a/src/pullbox/ui/intervention_context_loaders.py b/src/pullbox/ui/intervention_context_loaders.py index fe705d55..d3befe5e 100644 --- a/src/pullbox/ui/intervention_context_loaders.py +++ b/src/pullbox/ui/intervention_context_loaders.py @@ -214,6 +214,7 @@ async def load_intervention_queue_context( ("usenet", "Usenet"), ("torrent", "Torrent"), ("direct", "Direct"), + ("dc", "Direct Connect"), ], "high_count": confidence_counts.get("high", 0), "medium_count": confidence_counts.get("medium", 0), @@ -345,6 +346,7 @@ async def load_intervention_history_context( ("usenet", "Usenet"), ("torrent", "Torrent"), ("direct", "Direct"), + ("dc", "Direct Connect"), ], } diff --git a/src/pullbox/ui/intervention_filter_helpers.py b/src/pullbox/ui/intervention_filter_helpers.py index 0e3cafaf..d10d9f66 100644 --- a/src/pullbox/ui/intervention_filter_helpers.py +++ b/src/pullbox/ui/intervention_filter_helpers.py @@ -18,7 +18,7 @@ INTERVENTION_TABS = {"queue", "recovery", "history"} INTERVENTION_LANES = {"review", "recovery"} INTERVENTION_CONFIDENCE_FILTERS = {"high", "medium", "low"} -INTERVENTION_PROTOCOL_FILTERS = {"usenet", "torrent", "direct"} +INTERVENTION_PROTOCOL_FILTERS = {"usenet", "torrent", "direct", "dc"} INTERVENTION_REASON_LABELS = { "fuzzy_series": "Fuzzy series match", "issue_mismatch": "Issue mismatch", @@ -168,17 +168,19 @@ def intervention_protocol_clause(protocol: str) -> ColumnElement[bool]: PendingMatch.match_details["source_kind"].as_string(), "", ) - if protocol == "direct": - return source_kind == "direct" + if protocol in {"direct", "dc"}: + return source_kind == protocol if protocol == "torrent": - return and_(PendingMatch.is_torrent.is_(True), source_kind != "direct") - return and_(PendingMatch.is_torrent.is_(False), source_kind != "direct") + return and_(PendingMatch.is_torrent.is_(True), source_kind.not_in(("direct", "dc"))) + return and_(PendingMatch.is_torrent.is_(False), source_kind.not_in(("direct", "dc"))) def intervention_protocol_label(is_torrent: bool, source_kind: str = "") -> str: """Return the human-readable protocol label for a pending match.""" if source_kind == "direct": return "Direct" + if source_kind == "dc": + return "Direct Connect" return "Torrent" if is_torrent else "Usenet" @@ -257,6 +259,7 @@ def get_intervention_history_order_by(sort: str) -> list[ColumnElement[object]]: "", ) protocol_sort = case( + (source_kind == "dc", 3), (source_kind == "direct", 2), (PendingMatch.is_torrent.is_(True), 0), else_=1, diff --git a/src/pullbox/ui/intervention_routes.py b/src/pullbox/ui/intervention_routes.py index 168cb153..2e7531ca 100644 --- a/src/pullbox/ui/intervention_routes.py +++ b/src/pullbox/ui/intervention_routes.py @@ -678,7 +678,7 @@ async def htmx_intervention_approve( is_direct_pending_match, ) - if is_direct_pending_match(pm): + if is_direct_pending_match(pm) or (pm.match_details or {}).get("source_kind") == "dc": svc = InterventionService() else: from pullbox.composition.services import build_domain_download_service diff --git a/src/pullbox/ui/routes.py b/src/pullbox/ui/routes.py index 23026bf9..ed8c313d 100644 --- a/src/pullbox/ui/routes.py +++ b/src/pullbox/ui/routes.py @@ -49,6 +49,7 @@ series_detail_routes, series_routes, settings_routes, + story_arc_routes, system_routes, utilities_routes, whats_new_routes, @@ -560,6 +561,12 @@ async def _post_processing_live_status_map_bridge( import_review_partial = import_routes.import_review_partial import_series_reconcile = import_routes.import_series_reconcile import_results_partial = import_routes.import_results_partial +import_clean_library_panel = import_routes.import_clean_library_panel +import_misplaced_source_cleanup_files_partial = ( + import_routes.import_misplaced_source_cleanup_files_partial +) +import_misplaced_source_cleanup_preview = import_routes.import_misplaced_source_cleanup_preview +import_misplaced_source_cleanup_apply = import_routes.import_misplaced_source_cleanup_apply import_series_details_partial = import_routes.import_series_details_partial import_conflicts_partial = import_routes.import_conflicts_partial import_log_panel = import_routes.import_log_panel @@ -764,6 +771,16 @@ async def _load_sidebar_health_counts_bridge(session: AsyncSession) -> tuple[int reading_workspace = reading_routes.reading_workspace +story_arc_routes.configure_story_arc_routes( + get_templates=lambda: templates, + build_context=_ctx, +) +router.include_router(story_arc_routes.router) + +story_arc_list = story_arc_routes.story_arc_list +story_arc_detail = story_arc_routes.story_arc_detail + + router.include_router(series_routes.htmx_router) diff --git a/src/pullbox/ui/series_detail_routes.py b/src/pullbox/ui/series_detail_routes.py index 255a5ba7..2e0d8cb2 100644 --- a/src/pullbox/ui/series_detail_routes.py +++ b/src/pullbox/ui/series_detail_routes.py @@ -5,7 +5,7 @@ from collections.abc import AsyncIterator, Callable, Mapping from contextlib import suppress from typing import Annotated -from urllib.parse import unquote, urlsplit +from urllib.parse import unquote, urlencode, urlsplit import structlog from fastapi import APIRouter, Query, Request @@ -26,6 +26,7 @@ from pullbox.models.issue import Issue, IssueStatus from pullbox.models.library import LibraryFile from pullbox.models.series import Series +from pullbox.models.story_arc import IssueStoryArc, StoryArc from pullbox.services.airdcpp_route_tokens import get_airdcpp_route_token_store from pullbox.services.airdcpp_search_types import AirDcppSearchProgress from pullbox.services.reader_state_service import load_reader_state @@ -269,6 +270,8 @@ async def series_detail( series=series, file_count=file_count, delete_file_count=delete_context.linked_file_count, + delete_managed_file_count=delete_context.managed_file_count, + delete_referenced_file_count=delete_context.referenced_file_count, detail_origin=source if source == "pull-list" else None, detail_back_url=( _pull_list_return_url(return_to) if source == "pull-list" else "/series" @@ -285,12 +288,33 @@ async def issue_detail( user: AuthenticatedUser, session: DbSession, read: str | None = Query(None), + source: str | None = Query(None, max_length=50), + story_arc_id: Annotated[int | None, Query(ge=1)] = None, + story_arc_page: Annotated[int, Query(ge=1)] = 1, + story_arc_per_page: Annotated[int, Query(ge=1, le=100)] = 25, ) -> Response: """Render the issue detail page, fetching metadata on-demand if missing.""" issue = await _load_issue_detail_record(session, issue_id) if issue is None: return RedirectResponse(url="/series", status_code=302) + story_arc_origin = None + if source == "story-arc" and story_arc_id is not None: + story_arc_origin = await session.scalar( + select(StoryArc) + .join(IssueStoryArc, IssueStoryArc.story_arc_id == StoryArc.id) + .where( + StoryArc.id == story_arc_id, + IssueStoryArc.issue_id == issue.id, + ) + ) + issue_detail_back_url = ( + f"/story-arcs/{story_arc_origin.id}?" + f"{urlencode({'page': story_arc_page, 'per_page': story_arc_per_page})}" + if story_arc_origin is not None + else f"/series/{issue.series_id}" + ) + # On-demand metadata enrichment: fetch description from ComicVine if missing. if issue.comicvine_id and not issue.description: try: @@ -342,6 +366,8 @@ async def issue_detail( request, user, issue=issue, + issue_story_arc_origin=story_arc_origin, + issue_detail_back_url=issue_detail_back_url, reader_enabled=reader_enabled, issue_reading=issue_reading, open_reader_on_load=open_reader_on_load, diff --git a/src/pullbox/ui/series_routes.py b/src/pullbox/ui/series_routes.py index afa8b3b9..cd6d9b07 100644 --- a/src/pullbox/ui/series_routes.py +++ b/src/pullbox/ui/series_routes.py @@ -21,9 +21,10 @@ from pullbox.models.library import LibraryFile, LibraryRoot from pullbox.models.publisher import Publisher from pullbox.models.series import IssueCatalogState, Series, SeriesStatus -from pullbox.services.comicvine_persistent_cache import PersistentComicVineCacheProvider from pullbox.services.cover_url_service import build_series_cover_url +from pullbox.services.library_root_management import list_library_roots from pullbox.services.reading_query_service import load_series_reading_aggregates +from pullbox.ui.comicvine_provider import open_comicvine_ui_provider from pullbox.ui.comicvine_series_search import ( ADD_SERIES_PER_PAGE, COMICVINE_SERIES_SEARCH_LIMIT, @@ -525,10 +526,25 @@ async def add_series_page( search_mode: str | None = Query(None), ) -> Response: """Render the add series page with ComicVine search.""" - roots_result = await session.execute( - select(LibraryRoot).where(LibraryRoot.enabled.is_(True)).order_by(LibraryRoot.name) + roots = [ + root + for root in await list_library_roots(session) + if bool(root["enabled"]) + and bool(root["allow_managed_writes"]) + and bool(root["available"]) + and bool(root["writable"]) + ] + roots.sort( + key=lambda root: ( + not bool(root["is_default_managed_destination"]), + str(root["name"]).casefold(), + int(root["id"]), + ) ) - roots = list(roots_result.scalars().all()) + if not roots or not bool(roots[0]["is_default_managed_destination"]): + # The page intentionally has no arbitrary first-root fallback. A + # managed default must be selected through root management first. + roots = [] add_series_sort_options = COMICVINE_SERIES_SORT_OPTIONS add_series_search_ctx = await load_add_series_search_context( @@ -585,17 +601,24 @@ async def load_add_series_search_context( search_mode: str | None = None, session_factory: async_sessionmaker[AsyncSession] | None = None, ) -> dict[str, object]: + from pullbox.services.catalog.reader import get_catalog_reader + + local_catalog = get_catalog_reader().available per_page = ADD_SERIES_PER_PAGE normalized_query = (query or "").strip() normalized_sort = normalize_add_series_sort(sort) requested_page = max(1, page) roots_count = ( await session.scalar( - select(func.count(LibraryRoot.id)).where(LibraryRoot.enabled.is_(True)) + select(func.count(LibraryRoot.id)).where( + LibraryRoot.enabled.is_(True), + LibraryRoot.allow_managed_writes.is_(True), + ) ) ) or 0 base_context: dict[str, object] = { + "local_catalog": local_catalog, "search_query": normalized_query, "add_series_sort": normalized_sort, "is_preview_search": False, @@ -628,13 +651,6 @@ async def load_add_series_search_context( base_context["add_series_full_search_url"] = full_search_url try: - from pullbox.core.comicvine_key import get_comicvine_api_key - from pullbox.providers.metadata.comicvine import ComicVineProvider - - api_key = await get_comicvine_api_key(session) - provider: Any = ComicVineProvider(api_key=api_key) - if session_factory is not None: - provider = PersistentComicVineCacheProvider(provider, session_factory) naming_config = await _system_config_values( session, ( @@ -646,17 +662,22 @@ async def load_add_series_search_context( folder_template = naming_config.get("series_folder_template", "{Series} ({Year})") replace_illegal = naming_config.get("replace_illegal_characters", "true") == "true" colon_replacement = naming_config.get("colon_replacement", "dash") - if preview_mode: - cv_results, _total_results = await provider.search_series_page( - parsed_query.title_query, - parsed_query.year_hint, - limit=per_page, - ) - else: - cv_results, _total_results = await provider.search_series_globally( - parsed_query.title_query, - max_results=COMICVINE_SERIES_SEARCH_LIMIT, - ) + async with open_comicvine_ui_provider( + session, + session_factory=session_factory, + prefer_catalog=True, + ) as provider: + if preview_mode: + cv_results, _total_results = await provider.search_series_page( + parsed_query.title_query, + parsed_query.year_hint, + limit=per_page, + ) + else: + cv_results, _total_results = await provider.search_series_globally( + parsed_query.title_query, + max_results=COMICVINE_SERIES_SEARCH_LIMIT, + ) searchable_total = len(cv_results) total_pages = max(1, (searchable_total + per_page - 1) // per_page) resolved_page = min(requested_page, total_pages) @@ -679,7 +700,11 @@ async def load_add_series_search_context( ) except Exception: logger.exception("comicvine_search_failed", query=normalized_query) - base_context["search_error"] = "ComicVine search failed. Check your API key in settings." + base_context["search_error"] = ( + "Local catalog search failed. Check its status in Metadata settings." + if local_catalog + else "ComicVine search failed. Check your API key in settings." + ) return base_context in_library_count = sum(1 for item in search_results if bool(item.get("already_added"))) diff --git a/src/pullbox/ui/settings_routes.py b/src/pullbox/ui/settings_routes.py index aa18ba09..3204092c 100644 --- a/src/pullbox/ui/settings_routes.py +++ b/src/pullbox/ui/settings_routes.py @@ -258,6 +258,19 @@ async def load_settings_tab(request: Request, session: DbSession, tab: str) -> d ctx["configs"] = configs runtime = runtime_snapshot_to_dict(get_runtime_status_snapshot()) ctx["host_info"] = {"library_root": runtime["library_root"]["value"]} + from pullbox.services.library_root_management import list_library_roots + + root_states = await list_library_roots(session) + root_states.sort( + key=lambda root: ( + not bool(root["is_default_managed_destination"]), + str(root["name"]).casefold(), + ) + ) + ctx["library_roots"] = root_states + from pullbox.services.story_arc_file_defaults import load_story_arc_file_defaults + + ctx["arc_file_defaults"] = await load_story_arc_file_defaults(session) elif tab == "metadata": result = await session.execute(select(SystemConfig).order_by(SystemConfig.key)) ctx["configs"] = {c.key: c.value for c in result.scalars().all()} diff --git a/src/pullbox/ui/standalone_shell.py b/src/pullbox/ui/standalone_shell.py index 295d2c85..d0d8b02b 100644 --- a/src/pullbox/ui/standalone_shell.py +++ b/src/pullbox/ui/standalone_shell.py @@ -19,6 +19,8 @@ _STATIC_DIR / "js" / "idiomorph-ext.min.js", _STATIC_DIR / "js" / "alpine.min.js", _STATIC_DIR / "js" / "pullbox.js", + _STATIC_DIR / "js" / "story-arc-preview.js", + _STATIC_DIR / "js" / "story-arc-detail.js", ) diff --git a/src/pullbox/ui/static/css/input.css b/src/pullbox/ui/static/css/input.css index 575c6933..eae4ca10 100644 --- a/src/pullbox/ui/static/css/input.css +++ b/src/pullbox/ui/static/css/input.css @@ -540,6 +540,82 @@ line-height: var(--pb-control-line-height); border-radius: var(--pb-control-radius); } + .header-add-menu { + @apply relative inline-flex shrink-0; + } + .header-add-menu__primary { + @apply gap-2 whitespace-nowrap; + border-top-right-radius: 0; + border-bottom-right-radius: 0; + padding-right: 0.625rem; + } + .header-add-menu__trigger { + @apply gap-2 whitespace-nowrap; + min-width: 7rem; + padding-inline: 0.5rem; + justify-content: space-between; + border-top-left-radius: 0; + border-bottom-left-radius: 0; + border-left: 1px solid color-mix(in srgb, var(--pb-text-inverse) 26%, transparent); + } + .header-add-menu__chevron { + width: 0.875rem; + height: 0.875rem; + transition: transform 150ms ease; + } + .header-add-menu__panel { + @apply absolute left-0 z-50 p-1.5; + min-width: 17rem; + top: calc(100% + 0.5rem); + border: 1px solid var(--pb-border); + border-radius: 0.75rem; + background: var(--pb-bg-card); + box-shadow: var(--pb-shadow-2); + } + .header-add-menu__item { + @apply flex w-full items-center gap-2.5 px-3 py-2 text-left text-sm font-medium; + min-height: 2.5rem; + border-radius: 0.5rem; + color: var(--pb-text); + transition: color 150ms ease, background-color 150ms ease; + } + .header-add-menu__item:hover, + .header-add-menu__item:focus-visible { + color: var(--pb-interactive); + background: var(--pb-interactive-dim); + } + .header-add-menu__item--selected { + color: var(--pb-interactive); + background: var(--pb-interactive-dim); + } + .header-add-menu__item--disabled { + color: var(--pb-text-ter); + cursor: not-allowed; + opacity: 0.72; + } + .header-add-menu__item--disabled:hover { + color: var(--pb-text-ter); + background: transparent; + } + .header-add-menu__item-label { + @apply min-w-0 flex-1; + } + .header-add-menu__check { + color: var(--pb-success); + } + .header-add-menu__soon { + @apply shrink-0 whitespace-nowrap text-xs font-semibold uppercase tracking-wide; + color: var(--pb-text-ter); + } + .header-add-menu__item svg { + @apply h-4 w-4 shrink-0; + } + @media (prefers-reduced-motion: reduce) { + .header-add-menu__chevron, + .header-add-menu__item { + transition: none; + } + } .btn-ghost { @apply inline-flex items-center justify-center bg-transparent border border-pb-border-hover text-pb-text-sec rounded-pb-md font-sans font-medium @@ -3876,6 +3952,9 @@ align-items: start; gap: 1.25rem; } + .story-arc-detail-hero-inner { + grid-template-columns: 130px minmax(0, 1fr) minmax(18rem, 19rem); + } .issue-domain-hero-inner { grid-template-columns: 120px minmax(0, 1fr) minmax(13rem, 16rem); } @@ -4100,6 +4179,9 @@ padding-left: 1.25rem; border-left: 1px solid var(--pb-border-subtle); } + .series-domain-actions-panel-wide { + min-width: 18rem; + } .series-domain-actions-title { font-family: "Syne", sans-serif; font-size: 0.88rem; @@ -4143,6 +4225,13 @@ .series-domain-actions-panel .series-domain-actions-buttons .series-domain-inline-toggle { justify-content: space-between; } + .series-domain-action-form { + width: 100%; + } + .series-domain-action-form > button { + width: 100%; + justify-content: center; + } .series-domain-inline-toggle { display: inline-flex; align-items: center; @@ -4787,7 +4876,9 @@ color: var(--pb-text-primary); } html[data-series-view="list"] .series-view-toggle-item[data-series-view-option="list"], - html[data-series-view="grid"] .series-view-toggle-item[data-series-view-option="grid"] { + html[data-series-view="grid"] .series-view-toggle-item[data-series-view-option="grid"], + html[data-story-arc-view="list"] .series-view-toggle-item[data-story-arc-view-option="list"], + html[data-story-arc-view="grid"] .series-view-toggle-item[data-story-arc-view-option="grid"] { background: var(--pb-interactive); color: var(--pb-text-inverse); } @@ -5335,6 +5426,71 @@ left: 9px; z-index: 3; } + .story-arc-wall-review { + position: absolute; + top: 9px; + right: 9px; + z-index: 4; + min-width: 1.75rem; + height: 1.75rem; + padding-inline: 0.38rem; + border: 1px solid var(--pb-border-default); + border-radius: 999px; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 0.2rem; + background: color-mix(in srgb, var(--pb-surface-card) 92%, transparent); + box-shadow: var(--pb-shadow-1); + font-family: "JetBrains Mono", monospace; + font-size: 0.65rem; + font-weight: 700; + } + .story-arc-wall-review svg { + width: 0.8rem; + height: 0.8rem; + } + .story-arc-wall-review-warning { + color: var(--pb-warning); + border-color: color-mix(in srgb, var(--pb-warning) 48%, transparent); + } + .story-arc-wall-review-error { + color: var(--pb-error); + border-color: color-mix(in srgb, var(--pb-error) 48%, transparent); + } + .story-arc-review-indicator { + min-width: 2.5rem; + min-height: 1.75rem; + padding-inline: 0.45rem; + border: 1px solid var(--pb-border-default); + border-radius: 999px; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 0.28rem; + font-family: "JetBrains Mono", monospace; + font-size: 0.68rem; + font-weight: 700; + } + .story-arc-review-indicator svg { + width: 0.82rem; + height: 0.82rem; + } + .story-arc-review-indicator-success { + color: var(--pb-success); + background: var(--pb-success-dim); + border-color: color-mix(in srgb, var(--pb-success) 32%, transparent); + } + .story-arc-review-indicator-warning { + color: var(--pb-warning); + background: var(--pb-warning-dim); + border-color: color-mix(in srgb, var(--pb-warning) 32%, transparent); + } + .story-arc-review-indicator-error { + color: var(--pb-error); + background: var(--pb-error-dim); + border-color: color-mix(in srgb, var(--pb-error) 32%, transparent); + } .series-wall-catalog-state-badge { position: absolute; top: 46px; @@ -7095,6 +7251,9 @@ width: var(--pb-control-icon-size); height: var(--pb-control-icon-size); } + .import-advanced-disclosure[open] > summary .import-advanced-disclosure-chevron { + @apply rotate-90; + } .dropdown-select-panel { @apply fixed left-0 top-0 z-[80] max-h-60 overflow-y-auto rounded-lg border border-pb-border-hover bg-pb-card py-1 shadow-2xl shadow-black/20; max-height: var(--pb-dropdown-panel-max-height, 15rem); @@ -7225,7 +7384,7 @@ height: 0; } .search-field-clear { - @apply absolute top-1/2 z-10 -translate-y-1/2 rounded p-0.5 text-pb-text-sec transition-colors hover:text-pb-text; + @apply absolute top-1/2 z-10 inline-flex h-6 w-6 -translate-y-1/2 items-center justify-center rounded p-0.5 text-pb-text-sec transition-colors hover:text-pb-text; right: var(--pb-control-clear-right); } .search-history-panel { @@ -8403,12 +8562,14 @@ input[type="range"].range-pb::-moz-range-thumb { from { transform: translateX(-130%); } to { transform: translateX(280%); } } -.downloads-progress-fill.is-indeterminate { +.downloads-progress-fill.is-indeterminate, +.app-progress-fill.is-indeterminate { animation: downloads-progress-indeterminate 1.25s ease-in-out infinite; will-change: transform; } @media (prefers-reduced-motion: reduce) { - .downloads-progress-fill.is-indeterminate { + .downloads-progress-fill.is-indeterminate, + .app-progress-fill.is-indeterminate { animation: none; opacity: 0.7; transform: translateX(60%); diff --git a/src/pullbox/ui/static/css/tailwind.css b/src/pullbox/ui/static/css/tailwind.css index b6192583..14c8ea04 100644 --- a/src/pullbox/ui/static/css/tailwind.css +++ b/src/pullbox/ui/static/css/tailwind.css @@ -1,2 +1,2 @@ /*! tailwindcss v4.3.3 | MIT License | https://tailwindcss.com */ -@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-scale-x:1;--tw-scale-y:1;--tw-scale-z:1;--tw-space-y-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-ordinal:initial;--tw-slashed-zero:initial;--tw-numeric-figure:initial;--tw-numeric-spacing:initial;--tw-numeric-fraction:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-outline-style:solid;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-backdrop-blur:initial;--tw-backdrop-brightness:initial;--tw-backdrop-contrast:initial;--tw-backdrop-grayscale:initial;--tw-backdrop-hue-rotate:initial;--tw-backdrop-invert:initial;--tw-backdrop-opacity:initial;--tw-backdrop-saturate:initial;--tw-backdrop-sepia:initial;--tw-duration:initial;--tw-ease:initial}}}@layer theme{:root,:host{--color-slate-950:oklch(12.9% .042 264.695);--color-black:#000;--color-white:#fff;--spacing:.25rem;--container-xs:20rem;--container-sm:24rem;--container-md:28rem;--container-lg:32rem;--container-xl:36rem;--container-2xl:42rem;--container-3xl:48rem;--container-4xl:56rem;--container-5xl:64rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-lg:1.125rem;--text-lg--line-height:calc(1.75 / 1.125);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--text-3xl:1.875rem;--text-3xl--line-height:calc(2.25 / 1.875);--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--font-weight-bold:700;--font-weight-extrabold:800;--font-weight-black:900;--tracking-tight:-.025em;--tracking-wide:.025em;--tracking-wider:.05em;--leading-tight:1.25;--leading-relaxed:1.625;--radius-xs:.125rem;--radius-sm:.25rem;--radius-md:.375rem;--radius-lg:.5rem;--radius-xl:.75rem;--radius-2xl:1rem;--radius-3xl:1.5rem;--ease-in:cubic-bezier(.4, 0, 1, 1);--ease-out:cubic-bezier(0, 0, .2, 1);--animate-spin:spin 1s linear infinite;--animate-pulse:pulse 2s cubic-bezier(.4, 0, .6, 1) infinite;--blur-xs:4px;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1)}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;font-feature-settings:normal;font-variation-settings:normal;-webkit-tap-highlight-color:transparent;font-family:DM Sans,sans-serif;line-height:1.5}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-feature-settings:normal;font-variation-settings:normal;font-family:JetBrains Mono,Fira Code,monospace;font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring:where(:not(iframe)){outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab, red, red)){::placeholder{color:color-mix(in oklab, currentcolor 50%, transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{padding-block:0}::-webkit-datetime-edit-month-field{padding-block:0}::-webkit-datetime-edit-day-field{padding-block:0}::-webkit-datetime-edit-hour-field{padding-block:0}::-webkit-datetime-edit-minute-field{padding-block:0}::-webkit-datetime-edit-second-field{padding-block:0}::-webkit-datetime-edit-millisecond-field{padding-block:0}::-webkit-datetime-edit-meridiem-field{padding-block:0}::-webkit-calendar-picker-indicator{line-height:1}:-moz-ui-invalid{box-shadow:none}button,input:where([type=button],[type=reset],[type=submit]){appearance:button}::file-selector-button{appearance:button}::-webkit-inner-spin-button{height:auto}::-webkit-outer-spin-button{height:auto}[hidden]:where(:not([hidden=until-found])){display:none!important}html{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}:root,[data-theme=dark]{--pb-surface-app:#101824;--pb-surface-shell:#141e2c;--pb-surface-card:#172231;--pb-surface-raised:#1d2a3d;--pb-surface-input:#132030;--pb-surface-overlay:#070b12b8;--pb-surface-selected:#8fb9ee29;--pb-text-primary:#f7f1e8;--pb-text-secondary:#cbd5e1;--pb-text-tertiary:#9aaaba;--pb-text-inverse:#101824;--pb-border-subtle:#cbd5e11f;--pb-border-default:#cbd5e133;--pb-border-strong:#cbd5e152;--pb-interactive:#8fb9ee;--pb-interactive-hover:#7da9df;--pb-interactive-active:#6a96c8;--pb-interactive-selected:#8fb9ee2e;--pb-brand:#c6a17b;--pb-brand-muted:#c6a17b29;--pb-status-success:#68b88b;--pb-status-warning:#d7a15b;--pb-status-danger:#e38473;--pb-status-info:#8fb9ee;--pb-focus-ring:#8fb9ee47;--pb-focus-outline:#8fb9ee85;--pb-selection:#8fb9ee38;--pb-shadow-0:none;--pb-shadow-1:0 1px 2px #070b1238, 0 10px 24px #070b1229;--pb-shadow-2:0 2px 6px #070b1247, 0 18px 40px #070b1238;--pb-shadow-overlay:0 20px 56px #00000075;--pb-brand-hover:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-hover:color-mix(in srgb, var(--pb-brand) 84%, black)}}:root,[data-theme=dark]{--pb-brand-dim:var(--pb-brand-muted);--pb-brand-border:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-border:color-mix(in srgb, var(--pb-brand) 28%, transparent)}}:root,[data-theme=dark]{--pb-brand-signal:var(--pb-status-warning);--pb-brand-signal-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-signal-dim:color-mix(in srgb, var(--pb-status-warning) 16%, transparent)}}:root,[data-theme=dark]{--pb-brand-signal-hover:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-signal-hover:color-mix(in srgb, var(--pb-status-warning) 22%, transparent)}}:root,[data-theme=dark]{--pb-brand-signal-border:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-signal-border:color-mix(in srgb, var(--pb-status-warning) 28%, transparent)}}:root,[data-theme=dark]{--pb-interactive-dim:var(--pb-interactive-selected);--pb-interactive-border:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-interactive-border:color-mix(in srgb, var(--pb-interactive) 30%, transparent)}}:root,[data-theme=dark]{--pb-success:var(--pb-status-success);--pb-success-dim:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-success-dim:color-mix(in srgb, var(--pb-status-success) 18%, transparent)}}:root,[data-theme=dark]{--pb-warning:var(--pb-status-warning);--pb-warning-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-warning-dim:color-mix(in srgb, var(--pb-status-warning) 20%, transparent)}}:root,[data-theme=dark]{--pb-error:var(--pb-status-danger);--pb-error-dim:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-error-dim:color-mix(in srgb, var(--pb-status-danger) 18%, transparent)}}:root,[data-theme=dark]{--pb-info:var(--pb-status-info);--pb-info-dim:var(--pb-status-info)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-info-dim:color-mix(in srgb, var(--pb-status-info) 18%, transparent)}}:root,[data-theme=dark]{--pb-purple:#a88bda;--pb-purple-dim:#a88bda2e;--pb-bg-base:var(--pb-surface-app);--pb-bg-surface:var(--pb-surface-shell);--pb-bg-card:var(--pb-surface-card);--pb-bg-card-hover:var(--pb-surface-card)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-bg-card-hover:color-mix(in srgb, var(--pb-surface-card) 72%, var(--pb-surface-raised))}}:root,[data-theme=dark]{--pb-bg-input:var(--pb-surface-input);--pb-bg-overlay:var(--pb-surface-overlay);--pb-table-row-stripe:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-table-row-stripe:color-mix(in srgb, var(--pb-bg-card-hover) 68%, transparent)}}:root,[data-theme=dark]{--pb-table-row-hover:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-table-row-hover:color-mix(in srgb, var(--pb-bg-card-hover) 88%, transparent)}}:root,[data-theme=dark]{--pb-table-cell-x:1.25rem;--pb-table-cell-x-tight:1rem;--pb-page-footer-clearance:1.5rem;--pb-text-dim:var(--pb-text-tertiary);--pb-border:var(--pb-border-default);--pb-border-hover:var(--pb-border-strong)}[data-theme=light]{--pb-surface-app:#f6f1e8;--pb-surface-shell:#fbf6ee;--pb-surface-card:#fffcf7;--pb-surface-raised:#fff;--pb-surface-input:#fffdfa;--pb-surface-overlay:#1a16136b;--pb-surface-selected:#2f5e8c1f;--pb-text-primary:#1e1a17;--pb-text-secondary:#51473d;--pb-text-tertiary:#74675b;--pb-text-inverse:#f7f1e8;--pb-border-subtle:#51473d24;--pb-border-default:#51473d33;--pb-border-strong:#51473d57;--pb-interactive:#2f5e8c;--pb-interactive-hover:#274f76;--pb-interactive-active:#21435f;--pb-interactive-selected:#2f5e8c24;--pb-brand:#855e3d;--pb-brand-muted:#8c68471f;--pb-status-success:#2e6a4f;--pb-status-warning:#8a5a1f;--pb-status-danger:#b24432;--pb-status-info:#2f5e8c;--pb-focus-ring:#2f5e8c42;--pb-focus-outline:#2f5e8cb8;--pb-selection:#2f5e8c33;--pb-shadow-0:none;--pb-shadow-1:0 1px 2px #1a161314, 0 10px 24px #1a16130f;--pb-shadow-2:0 2px 6px #1a16131a, 0 18px 40px #1a161314;--pb-shadow-overlay:0 20px 56px #1a161333;--pb-brand-hover:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-hover:color-mix(in srgb, var(--pb-brand) 84%, black)}}[data-theme=light]{--pb-brand-dim:var(--pb-brand-muted);--pb-brand-border:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-border:color-mix(in srgb, var(--pb-brand) 28%, transparent)}}[data-theme=light]{--pb-brand-signal:var(--pb-status-warning);--pb-brand-signal-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-signal-dim:color-mix(in srgb, var(--pb-status-warning) 12%, transparent)}}[data-theme=light]{--pb-brand-signal-hover:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-signal-hover:color-mix(in srgb, var(--pb-status-warning) 18%, transparent)}}[data-theme=light]{--pb-brand-signal-border:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-signal-border:color-mix(in srgb, var(--pb-status-warning) 24%, transparent)}}[data-theme=light]{--pb-interactive-dim:var(--pb-interactive-selected);--pb-interactive-border:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-interactive-border:color-mix(in srgb, var(--pb-interactive) 28%, transparent)}}[data-theme=light]{--pb-success:var(--pb-status-success);--pb-success-dim:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-success-dim:color-mix(in srgb, var(--pb-status-success) 12%, transparent)}}[data-theme=light]{--pb-warning:var(--pb-status-warning);--pb-warning-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-warning-dim:color-mix(in srgb, var(--pb-status-warning) 14%, transparent)}}[data-theme=light]{--pb-error:var(--pb-status-danger);--pb-error-dim:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-error-dim:color-mix(in srgb, var(--pb-status-danger) 12%, transparent)}}[data-theme=light]{--pb-info:var(--pb-status-info);--pb-info-dim:var(--pb-status-info)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-info-dim:color-mix(in srgb, var(--pb-status-info) 12%, transparent)}}[data-theme=light]{--pb-purple:#7b5c91;--pb-purple-dim:#7b5c9124;--pb-bg-base:var(--pb-surface-app);--pb-bg-surface:var(--pb-surface-shell);--pb-bg-card:var(--pb-surface-card);--pb-bg-card-hover:var(--pb-surface-card)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-bg-card-hover:color-mix(in srgb, var(--pb-surface-card) 72%, var(--pb-surface-raised))}}[data-theme=light]{--pb-bg-input:var(--pb-surface-input);--pb-bg-overlay:var(--pb-surface-overlay);--pb-table-row-stripe:var(--pb-surface-shell)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-table-row-stripe:color-mix(in srgb, var(--pb-surface-shell) 40%, transparent)}}[data-theme=light]{--pb-table-row-hover:var(--pb-surface-shell)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-table-row-hover:color-mix(in srgb, var(--pb-surface-shell) 58%, transparent)}}[data-theme=light]{--pb-table-cell-x:1.25rem;--pb-table-cell-x-tight:1rem;--pb-text-dim:var(--pb-text-tertiary);--pb-border:var(--pb-border-default);--pb-border-hover:var(--pb-border-strong)}@media (prefers-color-scheme:light){:root:not([data-theme]){--pb-surface-app:#f6f1e8;--pb-surface-shell:#fbf6ee;--pb-surface-card:#fffcf7;--pb-surface-raised:#fff;--pb-surface-input:#fffdfa;--pb-surface-overlay:#1a16136b;--pb-surface-selected:#2f5e8c1f;--pb-text-primary:#1e1a17;--pb-text-secondary:#51473d;--pb-text-tertiary:#74675b;--pb-text-inverse:#f7f1e8;--pb-border-subtle:#51473d24;--pb-border-default:#51473d33;--pb-border-strong:#51473d57;--pb-interactive:#2f5e8c;--pb-interactive-hover:#274f76;--pb-interactive-active:#21435f;--pb-interactive-selected:#2f5e8c24;--pb-brand:#855e3d;--pb-brand-muted:#8c68471f;--pb-status-success:#2e6a4f;--pb-status-warning:#8a5a1f;--pb-status-danger:#b24432;--pb-status-info:#2f5e8c;--pb-focus-ring:#2f5e8c42;--pb-focus-outline:#2f5e8cb8;--pb-selection:#2f5e8c33;--pb-shadow-0:none;--pb-shadow-1:0 1px 2px #1a161314, 0 10px 24px #1a16130f;--pb-shadow-2:0 2px 6px #1a16131a, 0 18px 40px #1a161314;--pb-shadow-overlay:0 20px 56px #1a161333;--pb-brand-hover:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-hover:color-mix(in srgb, var(--pb-brand) 84%, black)}}:root:not([data-theme]){--pb-brand-dim:var(--pb-brand-muted);--pb-brand-border:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-border:color-mix(in srgb, var(--pb-brand) 28%, transparent)}}:root:not([data-theme]){--pb-brand-signal:var(--pb-status-warning);--pb-brand-signal-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-signal-dim:color-mix(in srgb, var(--pb-status-warning) 12%, transparent)}}:root:not([data-theme]){--pb-brand-signal-hover:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-signal-hover:color-mix(in srgb, var(--pb-status-warning) 18%, transparent)}}:root:not([data-theme]){--pb-brand-signal-border:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-signal-border:color-mix(in srgb, var(--pb-status-warning) 24%, transparent)}}:root:not([data-theme]){--pb-interactive-dim:var(--pb-interactive-selected);--pb-interactive-border:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-interactive-border:color-mix(in srgb, var(--pb-interactive) 28%, transparent)}}:root:not([data-theme]){--pb-success:var(--pb-status-success);--pb-success-dim:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-success-dim:color-mix(in srgb, var(--pb-status-success) 12%, transparent)}}:root:not([data-theme]){--pb-warning:var(--pb-status-warning);--pb-warning-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-warning-dim:color-mix(in srgb, var(--pb-status-warning) 14%, transparent)}}:root:not([data-theme]){--pb-error:var(--pb-status-danger);--pb-error-dim:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-error-dim:color-mix(in srgb, var(--pb-status-danger) 12%, transparent)}}:root:not([data-theme]){--pb-info:var(--pb-status-info);--pb-info-dim:var(--pb-status-info)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-info-dim:color-mix(in srgb, var(--pb-status-info) 12%, transparent)}}:root:not([data-theme]){--pb-purple:#7b5c91;--pb-purple-dim:#7b5c9124;--pb-bg-base:var(--pb-surface-app);--pb-bg-surface:var(--pb-surface-shell);--pb-bg-card:var(--pb-surface-card);--pb-bg-card-hover:var(--pb-surface-card)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-bg-card-hover:color-mix(in srgb, var(--pb-surface-card) 72%, var(--pb-surface-raised))}}:root:not([data-theme]){--pb-bg-input:var(--pb-surface-input);--pb-bg-overlay:var(--pb-surface-overlay);--pb-table-row-stripe:var(--pb-surface-shell)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-table-row-stripe:color-mix(in srgb, var(--pb-surface-shell) 40%, transparent)}}:root:not([data-theme]){--pb-table-row-hover:var(--pb-surface-shell)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-table-row-hover:color-mix(in srgb, var(--pb-surface-shell) 58%, transparent)}}:root:not([data-theme]){--pb-table-cell-x:1.25rem;--pb-table-cell-x-tight:1rem;--pb-text-dim:var(--pb-text-tertiary);--pb-border:var(--pb-border-default);--pb-border-hover:var(--pb-border-strong)}}html,body{overscroll-behavior:none;overflow:hidden}:focus-visible{outline:2px solid var(--pb-focus-outline);outline-offset:3px;box-shadow:0 0 0 4px var(--pb-focus-ring);border-radius:4px}::selection{background-color:var(--pb-selection)}:where(a[href]:not([aria-disabled=true]),button:not(:disabled):not([aria-disabled=true]),input[type=button]:not(:disabled),input[type=submit]:not(:disabled),input[type=reset]:not(:disabled),input[type=checkbox]:not(:disabled),input[type=radio]:not(:disabled),input[type=file]:not(:disabled),input[type=image]:not(:disabled),select:not(:disabled),summary,[role=button]:not([aria-disabled=true]),[role=link]:not([aria-disabled=true]),label:has(:is(input[type=checkbox],input[type=radio],input[type=file]):not(:disabled))){cursor:pointer}:where(button:disabled,input:disabled,select:disabled,[aria-disabled=true]){cursor:not-allowed}}@layer components{.control-size-sm,.btn-sm,.chip-btn-sm,.icon-btn-sm{--pb-control-min-height:2rem;--pb-control-radius:.5rem;--pb-control-gap:.5rem;--pb-control-px:.75rem;--pb-control-py:.4375rem;--pb-control-font-size:.875rem;--pb-control-line-height:1.25rem;--pb-control-icon-size:.875rem;--pb-control-panel-item-px:.75rem;--pb-control-panel-item-py:.5rem;--pb-control-search-icon-left:.625rem;--pb-control-search-padding-left:2rem;--pb-control-search-padding-right:1.875rem;--pb-control-clear-right:.5rem}.control-size-md,.btn-md,.chip-btn,.icon-btn,.dropdown-select,.search-field,.btn-primary,.btn-warning,.btn-ghost,.btn-danger{--pb-control-min-height:2.5rem;--pb-control-radius:.5rem;--pb-control-gap:.5rem;--pb-control-px:.75rem;--pb-control-py:.5rem;--pb-control-font-size:.875rem;--pb-control-line-height:1.25rem;--pb-control-icon-size:1rem;--pb-control-panel-item-px:.75rem;--pb-control-panel-item-py:.5rem;--pb-control-search-icon-left:.75rem;--pb-control-search-padding-left:2.5rem;--pb-control-search-padding-right:2.25rem;--pb-control-clear-right:.625rem}.control-size-lg,.btn-lg,.chip-btn-lg,.icon-btn-lg{--pb-control-min-height:2.875rem;--pb-control-radius:.625rem;--pb-control-gap:.625rem;--pb-control-px:.875rem;--pb-control-py:.625rem;--pb-control-font-size:.9375rem;--pb-control-line-height:1.375rem;--pb-control-icon-size:1rem;--pb-control-panel-item-px:.875rem;--pb-control-panel-item-py:.625rem;--pb-control-search-icon-left:.875rem;--pb-control-search-padding-left:2.75rem;--pb-control-search-padding-right:2.5rem;--pb-control-clear-right:.75rem}.btn-primary.btn-sm,.btn-warning.btn-sm,.btn-ghost.btn-sm,.btn-danger.btn-sm,.btn-primary.control-size-sm,.btn-warning.control-size-sm,.btn-ghost.control-size-sm,.btn-danger.control-size-sm{--pb-control-min-height:2rem;--pb-control-radius:.5rem;--pb-control-gap:.5rem;--pb-control-px:.75rem;--pb-control-py:.4375rem;--pb-control-font-size:.875rem;--pb-control-line-height:1.25rem;--pb-control-icon-size:.875rem}.btn-primary.btn-md,.btn-warning.btn-md,.btn-ghost.btn-md,.btn-danger.btn-md,.btn-primary.control-size-md,.btn-warning.control-size-md,.btn-ghost.control-size-md,.btn-danger.control-size-md{--pb-control-min-height:2.5rem;--pb-control-radius:.5rem;--pb-control-gap:.5rem;--pb-control-px:.75rem;--pb-control-py:.5rem;--pb-control-font-size:.875rem;--pb-control-line-height:1.25rem;--pb-control-icon-size:1rem}.btn-primary.btn-lg,.btn-warning.btn-lg,.btn-ghost.btn-lg,.btn-danger.btn-lg,.btn-primary.control-size-lg,.btn-warning.control-size-lg,.btn-ghost.control-size-lg,.btn-danger.control-size-lg{--pb-control-min-height:2.875rem;--pb-control-radius:.625rem;--pb-control-gap:.625rem;--pb-control-px:.875rem;--pb-control-py:.625rem;--pb-control-font-size:.9375rem;--pb-control-line-height:1.375rem;--pb-control-icon-size:1rem}.btn-primary{background-color:var(--pb-interactive);--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-weight:var(--font-weight-medium);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:10px;justify-content:center;align-items:center;display:inline-flex}@media (hover:hover){.btn-primary:hover{background-color:var(--pb-interactive-hover)}}.btn-primary:disabled{cursor:not-allowed;opacity:.5}.btn-primary{color:var(--pb-text-inverse);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.btn-ghost{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background-color:#0000;border-radius:10px;justify-content:center;align-items:center;display:inline-flex}@media (hover:hover){.btn-ghost:hover{border-color:var(--pb-border-strong);color:var(--pb-text-primary)}}.btn-ghost:disabled{cursor:not-allowed;opacity:.5}.btn-ghost{min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.btn-warning{--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-weight:var(--font-weight-medium);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:10px;justify-content:center;align-items:center;display:inline-flex}.btn-warning:disabled{cursor:not-allowed;opacity:.5}.btn-warning{background-color:var(--pb-warning);color:var(--pb-text-inverse);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.btn-warning:hover{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.btn-warning:hover{background-color:color-mix(in srgb, var(--pb-warning) 86%, var(--pb-text-primary))}}.btn-danger{justify-content:center;align-items:center;gap:var(--pb-control-gap);background-color:var(--pb-error);--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-weight:var(--font-weight-medium);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:10px;display:inline-flex}@media (hover:hover){.btn-danger:hover{--tw-brightness:brightness(90%);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}}.btn-danger:disabled{cursor:not-allowed;opacity:.5}.btn-danger{color:var(--pb-text-inverse);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.pill,.badge-success,.badge-warning,.badge-error,.badge-info,.badge-neutral,.badge-muted,.badge-purple{justify-content:center;align-items:center;gap:var(--spacing);padding-inline:calc(var(--spacing) * 2);padding-block:calc(var(--spacing) * .5);text-align:center;--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-size:11px;font-weight:var(--font-weight-medium);border-radius:3.40282e38px;display:inline-flex}.pill-success,.badge-success{background-color:var(--pb-success-dim);color:var(--pb-success)}.pill-warning,.badge-warning{background-color:var(--pb-warning-dim);color:var(--pb-warning)}.pill-error,.badge-error{background-color:var(--pb-error-dim);color:var(--pb-error)}.pill-info,.badge-info{background-color:var(--pb-info-dim);color:var(--pb-info)}.pill-neutral,.badge-neutral{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary)}.pill-muted,.badge-muted{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card-hover);color:var(--pb-text-dim)}.pill-purple,.badge-purple{background-color:var(--pb-purple-dim);color:var(--pb-purple)}.count-badge{min-width:1.25rem;min-height:1.25rem;padding-inline:calc(var(--spacing) * 1.5);--tw-leading:1;--tw-font-weight:var(--font-weight-bold);font-size:10px;line-height:1;font-weight:var(--font-weight-bold);--tw-numeric-spacing:tabular-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,);border-radius:3.40282e38px;justify-content:center;align-items:center;display:inline-flex}.count-badge-info{background-color:var(--pb-info-dim);color:var(--pb-info)}.count-badge-success{background-color:var(--pb-success-dim);color:var(--pb-success)}.count-badge-warning{background-color:var(--pb-warning-dim);color:var(--pb-warning)}.count-badge-error{background-color:var(--pb-error-dim);color:var(--pb-error)}.count-badge-neutral{background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary)}.table-shell{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card)}.table-shell-header{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4)}.table-shell-header-comfortable{padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4)}.table-shell-content{padding:calc(var(--spacing) * 5)}.table-shell-content-comfortable{padding:calc(var(--spacing) * 6)}.table-toolbar{gap:calc(var(--spacing) * 3);border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);flex-direction:column;display:flex}@media (min-width:64rem){.table-toolbar{flex-direction:row;justify-content:space-between;align-items:flex-end}}:where(.table-toolbar-title-block>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}.table-toolbar-eyebrow{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.table-toolbar-title{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.table-toolbar-title-compact{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.table-toolbar-copy{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.table-toolbar-actions{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;display:flex}.table-header-row{grid-template-columns:minmax(0,1fr) auto;align-items:center;gap:.75rem;display:grid}.table-header-aside{justify-self:end}.table-summary{align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));flex-wrap:wrap;display:flex}.table-scroll{overflow-x:auto}.table-base{border-collapse:collapse;width:100%;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.table-head{text-align:left;font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-tracking:var(--tracking-wider);letter-spacing:var(--tracking-wider);color:var(--pb-text-secondary);text-transform:uppercase}.table-head-row{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border)}.table-head-cell{padding-block:calc(var(--spacing) * 3);--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);padding-left:var(--pb-table-cell-x);padding-right:var(--pb-table-cell-x)}.table-head-cell-tight{padding-block:calc(var(--spacing) * 2.5);--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);padding-left:var(--pb-table-cell-x-tight);padding-right:var(--pb-table-cell-x-tight)}.table-head-cell-right{text-align:right}:where(.table-body>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)));border-color:var(--pb-border)}.table-body-striped>.table-row:nth-child(2n){background-color:var(--pb-table-row-stripe)}.table-row{color:var(--pb-text-secondary)}.table-row-hover>.table-row{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.table-row-hover>.table-row:hover{background-color:var(--pb-table-row-hover)}.table-row-hover>.table-row-detail{transition-property:none}.table-row-hover>.table-row-detail:hover{background-color:#0000}.table-cell{padding-block:calc(var(--spacing) * 3);padding-left:var(--pb-table-cell-x);padding-right:var(--pb-table-cell-x)}.table-cell-tight{padding-block:calc(var(--spacing) * 2.5);padding-left:var(--pb-table-cell-x-tight);padding-right:var(--pb-table-cell-x-tight)}.table-cell-right{text-align:right}.table-cell-muted{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary)}.table-cell-nowrap{white-space:nowrap}.table-footer{border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}.table-empty{padding:calc(var(--spacing) * 8);text-align:center}.table-empty-copy{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}.chip-btn{border-radius:var(--radius-lg);border-style:var(--tw-border-style);--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-width:1px;justify-content:center;align-items:center;display:inline-flex}.chip-btn:disabled{cursor:not-allowed;opacity:.5}.chip-btn{gap:var(--pb-control-gap);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.chip-btn-pill{padding-inline:calc(var(--spacing) * 3.5);padding-block:calc(var(--spacing) * 1.5);--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:.2em;letter-spacing:.2em;text-transform:uppercase;border-radius:9999px}.chip-btn-neutral{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary)}@media (hover:hover){.chip-btn-neutral:hover{border-color:var(--pb-border-strong);color:var(--pb-text-primary)}}.chip-btn-selected{color:var(--pb-info);background-color:var(--pb-info-dim);border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-selected{border-color:color-mix(in srgb, var(--pb-info) 26%, transparent)}}.chip-btn-selected:hover{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-selected:hover{background-color:color-mix(in srgb, var(--pb-info) 20%, var(--pb-bg-card))}}.chip-btn-selected:hover{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-selected:hover{border-color:color-mix(in srgb, var(--pb-info) 34%, transparent)}}.chip-btn-info{color:var(--pb-info);background-color:var(--pb-info-dim);border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-info{border-color:color-mix(in srgb, var(--pb-info) 24%, transparent)}}.chip-btn-info:hover{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-info:hover{background-color:color-mix(in srgb, var(--pb-info) 20%, var(--pb-bg-card))}}.chip-btn-info:hover{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-info:hover{border-color:color-mix(in srgb, var(--pb-info) 32%, transparent)}}.chip-btn-success{color:var(--pb-success);background-color:var(--pb-success-dim);border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.chip-btn-success{border-color:color-mix(in srgb, var(--pb-success) 24%, transparent)}}.chip-btn-success:hover{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.chip-btn-success:hover{background-color:color-mix(in srgb, var(--pb-success) 20%, var(--pb-bg-card))}}.chip-btn-success:hover{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.chip-btn-success:hover{border-color:color-mix(in srgb, var(--pb-success) 32%, transparent)}}.chip-btn-warning{color:var(--pb-warning);background-color:var(--pb-warning-dim);border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.chip-btn-warning{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.chip-btn-warning:hover{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.chip-btn-warning:hover{background-color:color-mix(in srgb, var(--pb-warning) 20%, var(--pb-bg-card))}}.chip-btn-warning:hover{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.chip-btn-warning:hover{border-color:color-mix(in srgb, var(--pb-warning) 32%, transparent)}}.chip-btn-error{color:var(--pb-error);background-color:var(--pb-error-dim);border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.chip-btn-error{border-color:color-mix(in srgb, var(--pb-error) 24%, transparent)}}.chip-btn-error:hover{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.chip-btn-error:hover{background-color:color-mix(in srgb, var(--pb-error) 20%, var(--pb-bg-card))}}.chip-btn-error:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.chip-btn-error:hover{border-color:color-mix(in srgb, var(--pb-error) 32%, transparent)}}.card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);border-radius:14px}.card-hover{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:14px}@media (hover:hover){.card-hover:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}}.auth-side-panel{border-right:1px solid var(--pb-border);background:linear-gradient(160deg, var(--pb-bg-surface) 0%, var(--pb-bg-surface) 42%, var(--pb-bg-card) 100%);position:relative;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.auth-side-panel{background:linear-gradient(160deg, color-mix(in srgb, var(--pb-bg-surface) 90%, var(--pb-brand) 10%) 0%, var(--pb-bg-surface) 42%, color-mix(in srgb, var(--pb-bg-card) 78%, var(--pb-interactive) 22%) 100%)}}.auth-side-panel:before,.auth-side-panel:after{content:"";pointer-events:none;filter:blur(12px);position:absolute;inset:auto}.auth-side-panel:before{background:radial-gradient(circle, var(--pb-brand) 0%, transparent 64%);width:24rem;height:24rem;top:-10%;left:-8%}@supports (color:color-mix(in lab, red, red)){.auth-side-panel:before{background:radial-gradient(circle, color-mix(in srgb, var(--pb-brand) 26%, transparent) 0%, transparent 64%)}}.auth-side-panel:after{background:radial-gradient(circle, var(--pb-interactive) 0%, transparent 66%);width:26rem;height:26rem;bottom:-14%;right:-12%}@supports (color:color-mix(in lab, red, red)){.auth-side-panel:after{background:radial-gradient(circle, color-mix(in srgb, var(--pb-interactive) 22%, transparent) 0%, transparent 66%)}}.auth-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-overlay);border-radius:1.75rem;position:relative;overflow:hidden}.auth-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.auth-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 40%, transparent), transparent)}}.auth-summary-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.auth-summary-card-emphasis{border-color:var(--pb-brand-border);background-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.auth-summary-card-emphasis{background-color:color-mix(in srgb, var(--pb-brand) 8%, var(--pb-bg-card))}}.auth-kicker{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.login-brand-mobile{justify-content:center;display:flex}.login-hero-stack{flex-direction:column;gap:1rem;max-width:29rem;padding-bottom:.25rem;display:flex}.login-hero-headline{letter-spacing:-.02em;color:var(--pb-text-primary);text-wrap:balance;padding-bottom:.1em;font-family:Syne,sans-serif;font-size:clamp(2.3rem,3.1vw,3.35rem);font-weight:800;line-height:1.08}.login-hero-copy{max-width:25rem;color:var(--pb-text-secondary);padding-bottom:.08em;font-size:.95rem;line-height:1.72}.login-side-footer{color:var(--pb-text-tertiary);flex-wrap:wrap;align-items:center;gap:.625rem 1.25rem;padding-bottom:.08em;font-family:JetBrains Mono,monospace;font-size:.68rem;display:flex}.login-side-footer-item{align-items:center;gap:.45rem;display:inline-flex}.login-side-footer-dot{background:var(--pb-success);width:.4rem;height:.4rem;box-shadow:0 0 4px var(--pb-success);border-radius:9999px}@supports (color:color-mix(in lab, red, red)){.login-side-footer-dot{box-shadow:0 0 4px color-mix(in srgb, var(--pb-success) 35%, transparent)}}.login-side-footer-dot{flex-shrink:0}.login-card-title{letter-spacing:-.02em;color:var(--pb-text-primary);padding-bottom:.08em;font-family:Syne,sans-serif;font-size:1.9rem;font-weight:800;line-height:1.08}.login-card-subtitle{color:var(--pb-text-secondary);padding-bottom:.05em;font-size:.875rem;line-height:1.62}.login-card-hint{color:var(--pb-text-tertiary);padding-bottom:.04em;font-size:.75rem;line-height:1.6}.floating-panel{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-overlay);overflow:hidden}.add-series-header-metrics{flex-wrap:wrap;align-items:flex-start;gap:1.5rem;display:flex}.add-series-results-shell{position:relative}.add-series-results-loading{z-index:12;background:var(--pb-bg-base);border-radius:.875rem;justify-content:center;align-items:flex-start;padding:1.25rem;display:flex;position:absolute;inset:0}@supports (color:color-mix(in lab, red, red)){.add-series-results-loading{background:color-mix(in srgb, var(--pb-bg-base) 55%, transparent)}}.add-series-results-loading{-webkit-backdrop-filter:blur(3px);backdrop-filter:blur(3px);visibility:hidden;pointer-events:none}.add-series-results-loading.htmx-request{opacity:1;visibility:visible}.add-series-results-loading-card{border:1px solid var(--pb-border-default);background:var(--pb-bg-card);border-radius:.875rem;align-items:center;gap:.75rem;max-width:24rem;padding:.875rem 1rem;display:flex}@supports (color:color-mix(in lab, red, red)){.add-series-results-loading-card{background:color-mix(in srgb, var(--pb-bg-card) 94%, transparent)}}.add-series-results-loading-card{color:var(--pb-text-primary);box-shadow:var(--pb-shadow-1)}.add-series-results-loading-card svg{color:var(--pb-interactive);flex-shrink:0}.add-series-results-loading-title{font-size:.82rem;font-weight:800;line-height:1.2}.add-series-results-loading-copy{color:var(--pb-text-secondary);margin-top:.125rem;font-size:.74rem;line-height:1.35}.add-series-preview-notice{border:1px solid var(--pb-interactive);border-radius:.875rem;flex-wrap:wrap;align-items:center;gap:.875rem;padding:.875rem 1rem;display:flex}@supports (color:color-mix(in lab, red, red)){.add-series-preview-notice{border:1px solid color-mix(in srgb, var(--pb-interactive) 32%, var(--pb-border-default))}}.add-series-preview-notice{background:linear-gradient(135deg, var(--pb-interactive), transparent 70%), var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.add-series-preview-notice{background:linear-gradient(135deg, color-mix(in srgb, var(--pb-interactive) 12%, transparent), transparent 70%), var(--pb-bg-card)}}.add-series-preview-notice{color:var(--pb-text-primary);box-shadow:var(--pb-shadow-0)}.add-series-preview-notice-icon{background:var(--pb-interactive);border-radius:999px;flex:none;justify-content:center;align-items:center;width:2rem;height:2rem;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.add-series-preview-notice-icon{background:color-mix(in srgb, var(--pb-interactive) 14%, transparent)}}.add-series-preview-notice-icon{color:var(--pb-interactive)}.add-series-preview-notice-body{flex:auto;min-width:0}.add-series-preview-notice-title{letter-spacing:.08em;text-transform:uppercase;font-size:.78rem;font-weight:850;line-height:1.2}.add-series-preview-notice-copy{color:var(--pb-text-secondary);margin-top:.125rem;font-size:.8rem;line-height:1.45}.add-series-preview-notice-action{white-space:nowrap;flex:none;margin-left:auto}@media (max-width:640px){.add-series-preview-notice{align-items:flex-start}.add-series-preview-notice-action{justify-content:center;width:100%;margin-left:0}}.add-series-results-list{gap:calc(var(--spacing) * 2);flex-direction:column;display:flex}.add-series-result-card{border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);border-radius:.875rem;align-items:flex-start;gap:.875rem;padding:.875rem;transition:border-color .14s,background-color .14s;display:flex}.add-series-result-card:hover{border-color:var(--pb-border-strong);background:var(--pb-surface-selected)}@supports (color:color-mix(in lab, red, red)){.add-series-result-card:hover{background:color-mix(in srgb, var(--pb-surface-selected) 35%, var(--pb-bg-card))}}.add-series-result-card-static:hover{border-color:var(--pb-border-subtle);background:var(--pb-bg-card)}.add-series-result-cover{aspect-ratio:2/3;border:1px solid var(--pb-border-subtle);background:var(--pb-surface-shell);width:3.5rem;min-width:3.5rem;color:var(--pb-text-tertiary);border-radius:.625rem;overflow:hidden}.add-series-result-cover-empty{justify-content:center;align-items:center;width:100%;height:100%;display:flex}.add-series-result-main{flex-wrap:wrap;justify-content:space-between;align-items:flex-start;gap:.875rem;display:flex}.add-series-result-title{color:var(--pb-text-primary);font-size:.92rem;font-weight:600;line-height:1.25}.add-series-result-meta{color:var(--pb-text-secondary);flex-wrap:wrap;gap:.25rem .75rem;margin-top:.25rem;font-size:.78rem;display:flex}.add-series-result-description{color:var(--pb-text-tertiary);-webkit-line-clamp:2;-webkit-box-orient:vertical;margin-top:.375rem;font-size:.78rem;line-height:1.55;display:-webkit-box;overflow:hidden}.add-series-result-actions{align-items:center;gap:calc(var(--spacing) * 2);display:flex}.add-series-result-add{white-space:nowrap;flex-shrink:0}.add-series-empty-state{text-align:center;border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);border-radius:.875rem;padding:2.5rem 1.25rem}.add-series-empty-title{text-transform:uppercase;letter-spacing:.04em;color:var(--pb-text-primary);margin-top:.75rem;font-family:Syne,sans-serif;font-size:.9rem;font-weight:700}.add-series-empty-copy{color:var(--pb-text-secondary);margin-top:.25rem;font-size:.82rem;line-height:1.55}.add-series-modal-title{color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:1rem;font-weight:800;line-height:1.1}.add-series-modal-subtitle{color:var(--pb-text-tertiary);font-size:.78rem}.add-series-modal-section-label{border-bottom:1px solid var(--pb-border-subtle);text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);padding-bottom:.375rem;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700}.add-series-modal-row{justify-content:space-between;align-items:center;gap:.75rem;display:flex}.add-series-modal-row-label{color:var(--pb-text-secondary);font-size:.85rem;font-weight:500}.add-series-modal-preview{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-shell);color:var(--pb-text-secondary);border-radius:.625rem;padding:.625rem .75rem;font-family:JetBrains Mono,monospace;font-size:.72rem;line-height:1.5}.add-series-root-display.input-pb:disabled,.add-series-root-display.input-pb[readonly]{opacity:1;color:var(--pb-text-primary);background:var(--pb-surface-input);cursor:default}.media-result-card{gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1);display:flex}.media-result-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.media-result-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.media-result-card:hover{box-shadow:var(--pb-shadow-2)}.media-result-card-static:hover{border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1)}.media-result-cover{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card-hover);flex-shrink:0;width:4.75rem;overflow:hidden}.media-result-cover-lg{width:6.25rem}.media-result-cover-inner{aspect-ratio:2/3;width:100%;height:100%}.media-result-empty{width:100%;height:100%;color:var(--pb-text-dim);justify-content:center;align-items:center;display:flex}.media-result-meta{margin-top:var(--spacing);align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary);flex-wrap:wrap;display:flex}.media-result-divider{color:var(--pb-text-dim)}.match-file-card{gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-direction:column;display:flex}@media (min-width:64rem){.match-file-card{flex-direction:row;align-items:flex-start}}.match-file-card{box-shadow:var(--pb-shadow-1)}.match-file-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.match-file-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 58%, transparent)}}.match-file-card:hover{box-shadow:var(--pb-shadow-2)}.match-file-format{height:calc(var(--spacing) * 14);width:calc(var(--spacing) * 14);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:.18em;letter-spacing:.18em;text-transform:uppercase;border-width:1px;flex-shrink:0;justify-content:center;align-items:center;display:flex}.match-file-format-cbz{color:var(--pb-interactive);border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.match-file-format-cbz{border-color:color-mix(in srgb, var(--pb-interactive) 24%, transparent)}}.match-file-format-cbz{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.match-file-format-cbz{background-color:color-mix(in srgb, var(--pb-interactive) 12%, transparent)}}.match-file-format-cbr{color:var(--pb-purple);border-color:var(--pb-purple)}@supports (color:color-mix(in lab, red, red)){.match-file-format-cbr{border-color:color-mix(in srgb, var(--pb-purple) 24%, transparent)}}.match-file-format-cbr{background-color:var(--pb-purple)}@supports (color:color-mix(in lab, red, red)){.match-file-format-cbr{background-color:color-mix(in srgb, var(--pb-purple) 12%, transparent)}}.match-file-format-pdf{color:var(--pb-warning);border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.match-file-format-pdf{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.match-file-format-pdf{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.match-file-format-pdf{background-color:color-mix(in srgb, var(--pb-warning) 12%, transparent)}}.match-file-format-other{color:var(--pb-text-secondary);border-color:var(--pb-border);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.match-file-format-other{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 82%, transparent)}}.match-file-card-actions{gap:calc(var(--spacing) * 2);flex-direction:column;width:100%;display:flex}@media (min-width:64rem){.match-file-card-actions{align-items:flex-end;width:auto;min-width:11rem}}.library-series-card{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);min-width:0;height:100%;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1);flex-direction:column;display:flex;overflow:hidden}.library-series-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.library-series-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 56%, var(--pb-bg-card))}}.library-series-card:hover{box-shadow:var(--pb-shadow-2)}.library-series-card-active{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-series-card-active{border-color:color-mix(in srgb, var(--pb-interactive) 30%, transparent)}}.library-series-card-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-series-card-active{background-color:color-mix(in srgb, var(--pb-interactive) 10%, var(--pb-bg-card))}}.library-series-cover-frame{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);background:linear-gradient(160deg, var(--pb-bg-card-hover) 0%, var(--pb-bg-card) 100%);position:relative;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.library-series-cover-frame{background:linear-gradient(160deg, color-mix(in srgb, var(--pb-bg-card-hover) 88%, transparent) 0%, color-mix(in srgb, var(--pb-bg-card) 82%, transparent) 100%)}}.library-series-cover-frame-compact{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border)}.library-series-placeholder{justify-content:center;align-items:center;gap:calc(var(--spacing) * 2);width:100%;height:100%;color:var(--pb-text-dim);flex-direction:column;display:flex}.library-series-meta{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.library-series-progress-track{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.library-series-progress-track{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 82%, transparent)}}.series-grid-card-meta{container-type:inline-size}.series-grid-card-publisher-year{grid-template-columns:8rem minmax(0,1fr);align-items:start;gap:.25rem .5rem;display:grid}.series-grid-card-publisher{overflow-wrap:anywhere;min-width:0}.series-grid-card-year{text-align:right;justify-self:end}.series-grid-card-progress-track{background-color:var(--pb-text-dim)}@supports (color:color-mix(in lab, red, red)){.series-grid-card-progress-track{background-color:color-mix(in srgb, var(--pb-text-dim) 72%, var(--pb-bg-card) 28%)}}@container (max-width:11rem){.series-grid-card-publisher-year{grid-template-columns:8rem}.series-grid-card-year{text-align:left;justify-self:start}}.admin-workspace-page{gap:calc(var(--spacing) * 4);flex-direction:column;min-height:0;display:flex}.admin-workspace-header{padding-block:0}.admin-workspace-header-copy{max-width:var(--container-3xl);min-width:0}.admin-workspace-tag-row{gap:calc(var(--spacing) * 2);padding-top:var(--spacing);flex-wrap:wrap;display:flex}.admin-workspace-header-aside{min-width:0}.admin-workspace-summary-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);height:100%;padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1)}.admin-workspace-body{gap:calc(var(--spacing) * 7);display:grid}@media (min-width:80rem){.admin-workspace-body{grid-template-columns:10.75rem minmax(0,1fr);align-items:flex-start}}.admin-workspace-body{padding-bottom:var(--pb-page-footer-clearance)}.admin-workspace-rail{gap:calc(var(--spacing) * 3);flex-direction:column;display:flex}@media (min-width:80rem){.admin-workspace-rail{top:calc(var(--spacing) * 24);position:sticky}}:where(.admin-workspace-rail-copy>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}.admin-workspace-rail-links{gap:var(--spacing);flex-direction:column;display:flex}.admin-workspace-content{flex:1;min-width:0}:where(.admin-workspace-content>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.admin-workspace-content{width:min(100%,58rem)}.section-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);overflow:hidden}.admin-workspace-content .space-y-6>.section-card:not(:has(~.section-card)){margin-bottom:0!important}.section-card-visible{overflow:visible}.system-version-banner{gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);flex-direction:column;padding-block:1.125rem;display:flex;position:relative;overflow:hidden}@media (min-width:64rem){.system-version-banner{flex-direction:row;justify-content:space-between;align-items:center}}.system-version-banner{box-shadow:var(--pb-shadow-1)}.system-version-banner:before{content:"";background:var(--pb-brand);width:3px;position:absolute;top:0;bottom:0;left:0}.system-version-banner-main{align-items:flex-start;gap:calc(var(--spacing) * 4);min-width:0;display:flex}.system-version-banner-icon{height:calc(var(--spacing) * 11);width:calc(var(--spacing) * 11);background:var(--pb-brand-muted);color:var(--pb-brand);border-radius:10px;flex-shrink:0;justify-content:center;align-items:center;display:flex}.system-version-banner-title{letter-spacing:-.02em;color:var(--pb-text-primary);flex-wrap:wrap;align-items:center;gap:.625rem;font-family:Bricolage Grotesque,sans-serif;font-size:1.125rem;font-weight:800;display:flex}.system-version-banner-meta{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.system-version-banner-meta span{font-family:JetBrains Mono,monospace;font-size:.6875rem}.system-version-banner-actions{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;flex-shrink:0;display:flex}:where(.system-detail-list>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)))}.system-detail-list{border-color:var(--pb-border-subtle)}.system-detail-row{justify-content:space-between;align-items:baseline;gap:calc(var(--spacing) * 5);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 2.5);display:flex}.system-detail-label{color:var(--pb-text-secondary);flex-shrink:0;font-size:13px}.system-detail-value{color:var(--pb-text-primary);text-align:right;word-break:break-word;font-family:JetBrains Mono,monospace;font-size:.78125rem}.system-detail-value-brand{color:var(--pb-brand)}.system-registry-filename{--tw-leading:calc(var(--spacing) * 5);font-family:JetBrains Mono,Fira Code,monospace;font-size:.8125rem;line-height:calc(var(--spacing) * 5);--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.system-links-grid{gap:calc(var(--spacing) * 2);padding:calc(var(--spacing) * 5);grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));display:grid}.system-link-card{align-items:center;gap:calc(var(--spacing) * 2.5);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 3.5);padding-block:calc(var(--spacing) * 2.5);--tw-font-weight:var(--font-weight-medium);font-size:12.5px;font-weight:var(--font-weight-medium);color:var(--pb-text-primary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background:var(--pb-bg-card-hover);display:flex}@supports (color:color-mix(in lab, red, red)){.system-link-card{background:color-mix(in srgb, var(--pb-bg-card-hover) 20%, transparent)}}.system-link-card:hover{border-color:var(--pb-border-hover);background:var(--pb-bg-card-hover)}.system-link-card svg{height:calc(var(--spacing) * 3.5);width:calc(var(--spacing) * 3.5);color:var(--pb-text-dim);flex-shrink:0}.system-link-card-arrow{height:calc(var(--spacing) * 3);width:calc(var(--spacing) * 3);color:var(--pb-text-dim);margin-left:auto}.stat-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.stat-card-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.stat-card-value{margin-top:calc(var(--spacing) * 3);font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.stat-card-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5);color:var(--pb-text-secondary)}.info-panel{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1)}.info-panel-muted{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.info-panel-muted{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.dashboard-briefing-shell{border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-2);background:radial-gradient(circle at top right, var(--pb-interactive) 0%, transparent 38%), radial-gradient(circle at left bottom, var(--pb-brand) 0%, transparent 34%), var(--pb-bg-card);position:relative;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.dashboard-briefing-shell{background:radial-gradient(circle at top right, color-mix(in srgb, var(--pb-interactive) 14%, transparent) 0%, transparent 38%), radial-gradient(circle at left bottom, color-mix(in srgb, var(--pb-brand) 10%, transparent) 0%, transparent 34%), var(--pb-bg-card)}}.dashboard-briefing-shell:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), var(--pb-interactive), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.dashboard-briefing-shell:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 44%, transparent), color-mix(in srgb, var(--pb-interactive) 44%, transparent), transparent)}}.dashboard-briefing-meta{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);width:100%;padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}@media (min-width:64rem){.dashboard-briefing-meta{max-width:13rem}}.dashboard-briefing-meta{box-shadow:var(--pb-shadow-1);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.dashboard-briefing-meta{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 42%, transparent)}}.dashboard-status-pill,.dashboard-priority-badge,.dashboard-watch-tag,.dashboard-scorecard-delta{align-items:center;gap:var(--spacing);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.16em;letter-spacing:.16em;text-transform:uppercase;border-width:1px;border-color:currentColor;border-radius:3.40282e38px;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.dashboard-status-pill,.dashboard-priority-badge,.dashboard-watch-tag,.dashboard-scorecard-delta{border-color:color-mix(in srgb, currentColor 20%, transparent)}}.dashboard-priority-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.dashboard-priority-grid{grid-template-columns:repeat(3,minmax(0,1fr))}}.dashboard-priority-card-v2{align-items:flex-start;gap:calc(var(--spacing) * 4);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:14px;grid-template-columns:1fr auto;transition:box-shadow .14s;display:grid;position:relative;overflow:hidden}.dashboard-priority-card-v2:before{content:"";background-color:#0000;width:6px;position:absolute;inset:0 auto 0 0}.dashboard-priority-card-v2:hover{box-shadow:var(--pb-shadow-2)}.dashboard-priority-card-v2-tone-pill-error:before{background-color:var(--pb-error)}.dashboard-priority-card-v2-tone-pill-warning:before{background-color:var(--pb-warning)}.dashboard-priority-card-v2-tone-pill-info:before{background-color:var(--pb-info)}.dashboard-priority-card-v2-tone-pill-success:before{background-color:var(--pb-success)}.dashboard-priority-card-v2-tone-pill-neutral:before,.dashboard-priority-card-v2-tone-pill-muted:before{background-color:var(--pb-text-dim)}.dashboard-priority-fact{border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-base);padding-inline:calc(var(--spacing) * 2.5);padding-block:calc(var(--spacing) * .5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary);display:inline-flex}.dashboard-priority-card,.dashboard-decision-card{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-width:1px}.dashboard-priority-card-critical,.dashboard-decision-card-critical{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-critical,.dashboard-decision-card-critical{border-color:color-mix(in srgb, var(--pb-error) 24%, transparent)}}.dashboard-priority-card-critical,.dashboard-decision-card-critical{background:linear-gradient(135deg, var(--pb-error) 0%, var(--pb-bg-card-hover) 100%)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-critical,.dashboard-decision-card-critical{background:linear-gradient(135deg, color-mix(in srgb, var(--pb-error) 10%, var(--pb-bg-card)) 0%, color-mix(in srgb, var(--pb-bg-card-hover) 92%, transparent) 100%)}}.dashboard-priority-card-high,.dashboard-decision-card-high{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-high,.dashboard-decision-card-high{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.dashboard-priority-card-high,.dashboard-decision-card-high{background:linear-gradient(135deg, var(--pb-warning) 0%, var(--pb-bg-card-hover) 100%)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-high,.dashboard-decision-card-high{background:linear-gradient(135deg, color-mix(in srgb, var(--pb-warning) 10%, var(--pb-bg-card)) 0%, color-mix(in srgb, var(--pb-bg-card-hover) 92%, transparent) 100%)}}.dashboard-priority-card-watch,.dashboard-decision-card-watch{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-watch,.dashboard-decision-card-watch{border-color:color-mix(in srgb, var(--pb-warning) 18%, transparent)}}.dashboard-priority-card-watch,.dashboard-decision-card-watch{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-watch,.dashboard-decision-card-watch{background-color:color-mix(in srgb, var(--pb-warning) 7%, var(--pb-bg-card))}}.dashboard-priority-card-healthy,.dashboard-decision-card-healthy{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-healthy,.dashboard-decision-card-healthy{border-color:color-mix(in srgb, var(--pb-success) 20%, transparent)}}.dashboard-priority-card-healthy,.dashboard-decision-card-healthy{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-healthy,.dashboard-decision-card-healthy{background-color:color-mix(in srgb, var(--pb-success) 7%, var(--pb-bg-card))}}.dashboard-priority-card-info,.dashboard-decision-card-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-info,.dashboard-decision-card-info{border-color:color-mix(in srgb, var(--pb-info) 18%, transparent)}}.dashboard-priority-card-info,.dashboard-decision-card-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-info,.dashboard-decision-card-info{background-color:color-mix(in srgb, var(--pb-info) 7%, var(--pb-bg-card))}}.dashboard-priority-facts{margin-top:calc(var(--spacing) * 5);gap:calc(var(--spacing) * 3);border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-top:calc(var(--spacing) * 4);display:grid}@media (min-width:40rem){.dashboard-priority-facts{grid-template-columns:repeat(3,minmax(0,1fr))}}.dashboard-priority-facts dt{--tw-font-weight:var(--font-weight-medium);font-size:10px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.dashboard-priority-facts dd{margin-top:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-primary)}.dashboard-quiet-note{align-items:center;gap:calc(var(--spacing) * 3);border-radius:var(--radius-xl);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-success);background:var(--pb-success);display:flex}@supports (color:color-mix(in lab, red, red)){.dashboard-quiet-note{background:color-mix(in srgb, var(--pb-success) 10%, var(--pb-bg-card))}}.dashboard-quiet-note{border:1px solid var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-quiet-note{border:1px solid color-mix(in srgb, var(--pb-success) 18%, transparent)}}.dashboard-pulse-tile{padding-block:calc(var(--spacing) * 2.5);flex-direction:column;align-items:flex-start;display:flex}.dashboard-pulse-value{margin-top:var(--spacing);font-size:var(--text-xl);line-height:var(--tw-leading,var(--text-xl--line-height));--tw-font-weight:var(--font-weight-bold);font-weight:var(--font-weight-bold)}.dashboard-pulse-tile-right{border-left-style:var(--tw-border-style);border-left-width:1px;border-color:var(--pb-border);padding-left:calc(var(--spacing) * 3)}.dashboard-pulse-tile-bottom{border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-top:calc(var(--spacing) * 2.5)}.dashboard-scoreboard-attention-grid{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.dashboard-scoreboard-attention-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.dashboard-scoreboard-attention-grid{grid-template-columns:repeat(4,minmax(0,1fr))}}.dashboard-scorecard{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);min-width:0;height:100%;padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-width:1px;flex-direction:column;justify-content:space-between;display:flex}.dashboard-scorecard-critical{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-critical{border-color:color-mix(in srgb, var(--pb-error) 22%, transparent)}}.dashboard-scorecard-critical{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-critical{background-color:color-mix(in srgb, var(--pb-error) 8%, var(--pb-bg-card))}}.dashboard-scorecard-high{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-high{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.dashboard-scorecard-high{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-high{background-color:color-mix(in srgb, var(--pb-warning) 9%, var(--pb-bg-card))}}.dashboard-scorecard-watch{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-watch{border-color:color-mix(in srgb, var(--pb-warning) 18%, transparent)}}.dashboard-scorecard-watch{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-watch{background-color:color-mix(in srgb, var(--pb-warning) 7%, var(--pb-bg-card))}}.dashboard-scorecard-healthy{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-healthy{border-color:color-mix(in srgb, var(--pb-success) 20%, transparent)}}.dashboard-scorecard-healthy{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-healthy{background-color:color-mix(in srgb, var(--pb-success) 7%, var(--pb-bg-card))}}.dashboard-scorecard-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-info{border-color:color-mix(in srgb, var(--pb-info) 18%, transparent)}}.dashboard-scorecard-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-info{background-color:color-mix(in srgb, var(--pb-info) 7%, var(--pb-bg-card))}}.dashboard-scorecard-title{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.dashboard-scorecard-value{font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.dashboard-scorecard-value-watch{color:var(--pb-status-warning)}.dashboard-scorecard-value-critical{color:var(--pb-status-danger)}.dashboard-scorecard-copy{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.dashboard-healthy-strip{align-items:center;gap:calc(var(--spacing) * 2);column-gap:calc(var(--spacing) * 6);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 3.5);flex-wrap:wrap;display:flex}.dashboard-healthy-strip-label{align-items:center;gap:calc(var(--spacing) * 1.5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-success);display:flex}.dashboard-healthy-metric{align-items:baseline;gap:calc(var(--spacing) * 1.5);color:var(--pb-text-secondary);font-size:.82rem;display:flex}.dashboard-healthy-metric strong{--tw-font-weight:var(--font-weight-bold);font-weight:var(--font-weight-bold);color:var(--pb-text-primary)}.dashboard-inline-link{align-items:center;gap:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.dashboard-inline-link:hover{color:var(--pb-interactive-hover)}.dashboard-download-row{align-items:center;gap:calc(var(--spacing) * 3);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3.5);grid-template-columns:1fr 100px 80px;display:grid}.dashboard-watch-item,.dashboard-exception-row{border-radius:var(--radius-2xl)}.dashboard-watch-item-critical,.dashboard-exception-row-critical{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-critical,.dashboard-exception-row-critical{border-color:color-mix(in srgb, var(--pb-error) 20%, transparent)}}.dashboard-watch-item-critical,.dashboard-exception-row-critical{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-critical,.dashboard-exception-row-critical{background-color:color-mix(in srgb, var(--pb-error) 7%, var(--pb-bg-card))}}.dashboard-watch-item-high,.dashboard-exception-row-high{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-high,.dashboard-exception-row-high{border-color:color-mix(in srgb, var(--pb-warning) 22%, transparent)}}.dashboard-watch-item-high,.dashboard-exception-row-high{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-high,.dashboard-exception-row-high{background-color:color-mix(in srgb, var(--pb-warning) 8%, var(--pb-bg-card))}}.dashboard-watch-item-watch,.dashboard-exception-row-watch{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-watch,.dashboard-exception-row-watch{border-color:color-mix(in srgb, var(--pb-warning) 16%, transparent)}}.dashboard-watch-item-watch,.dashboard-exception-row-watch{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-watch,.dashboard-exception-row-watch{background-color:color-mix(in srgb, var(--pb-warning) 6%, var(--pb-bg-card))}}.dashboard-watch-item-healthy,.dashboard-exception-row-healthy{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-healthy,.dashboard-exception-row-healthy{border-color:color-mix(in srgb, var(--pb-success) 16%, transparent)}}.dashboard-watch-item-healthy,.dashboard-exception-row-healthy{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-healthy,.dashboard-exception-row-healthy{background-color:color-mix(in srgb, var(--pb-success) 6%, var(--pb-bg-card))}}.dashboard-watch-item-info,.dashboard-exception-row-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-info,.dashboard-exception-row-info{border-color:color-mix(in srgb, var(--pb-info) 16%, transparent)}}.dashboard-watch-item-info,.dashboard-exception-row-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-info,.dashboard-exception-row-info{background-color:color-mix(in srgb, var(--pb-info) 6%, var(--pb-bg-card))}}.dashboard-mission-page{gap:calc(var(--spacing) * 4);width:100%;padding-bottom:var(--pb-page-footer-clearance);flex-direction:column;display:flex}.dashboard-mission-control{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 5);flex-wrap:wrap;display:flex}.dashboard-mission-control__summary{align-items:flex-start;gap:calc(var(--spacing) * 6);flex-wrap:wrap;flex:1;min-width:0;display:flex}.dashboard-mission-control__title-block{min-width:0}.dashboard-mission-control__title{letter-spacing:.04em;text-transform:uppercase;color:var(--pb-text);font-family:Syne,sans-serif;font-size:1.5rem;font-weight:800;line-height:1}.dashboard-mission-control__title span{color:var(--pb-brand)}.dashboard-mission-control__subtitle{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.dashboard-mission-control__gauges{align-items:flex-end;gap:calc(var(--spacing) * 4);flex-wrap:wrap;display:flex}.dashboard-mission-control__gauges-spacer{flex:none;width:224px;min-width:224px;min-height:72px}.dashboard-gauge{text-align:center;min-width:64px}.dashboard-gauge__ring{width:56px;height:56px;margin-inline:auto;position:relative}.dashboard-gauge__ring svg{width:56px;height:56px;transform:rotate(-90deg)}.dashboard-gauge__track{fill:none;stroke:var(--pb-text-dim)}@supports (color:color-mix(in lab, red, red)){.dashboard-gauge__track{stroke:color-mix(in srgb, var(--pb-text-dim) 14%, transparent)}}.dashboard-gauge__track{stroke-width:4.5px}.dashboard-gauge__fill{fill:none;stroke-width:4.5px;stroke-linecap:round}.dashboard-gauge__value{justify-content:center;align-items:center;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:700;display:flex;position:absolute;inset:0}.dashboard-gauge__label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);margin-top:.2rem;font-size:.52rem;font-weight:600;display:block}.dashboard-gauge--success .dashboard-gauge__fill,.dashboard-gauge--success .dashboard-gauge__value{stroke:var(--pb-success);color:var(--pb-success)}.dashboard-gauge--info .dashboard-gauge__fill,.dashboard-gauge--info .dashboard-gauge__value{stroke:var(--pb-info);color:var(--pb-info)}.dashboard-gauge--warning .dashboard-gauge__fill,.dashboard-gauge--warning .dashboard-gauge__value{stroke:var(--pb-warning);color:var(--pb-warning)}.dashboard-gauge--danger .dashboard-gauge__fill,.dashboard-gauge--danger .dashboard-gauge__value{stroke:var(--pb-error);color:var(--pb-error)}.dashboard-gauge--neutral .dashboard-gauge__fill,.dashboard-gauge--neutral .dashboard-gauge__value{stroke:var(--pb-text-dim);color:var(--pb-text-dim)}.dashboard-scoreboard{grid-template-columns:repeat(auto-fit,minmax(140px,1fr));gap:.5rem;display:grid}.dashboard-scoreboard__item{gap:calc(var(--spacing) * .5);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);min-width:0;padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);box-shadow:var(--pb-shadow-1);border-radius:10px;flex-direction:column;display:flex}.dashboard-scoreboard__label,.dashboard-section__label{letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);font-family:Syne,sans-serif;font-size:.54rem;font-weight:700}.dashboard-scoreboard__value{color:var(--pb-text);font-family:JetBrains Mono,monospace;font-size:1.05rem;font-weight:700}.dashboard-scoreboard__delta{color:var(--pb-text-dim);font-family:JetBrains Mono,monospace;font-size:.66rem}:where(.dashboard-section>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2.5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2.5) * calc(1 - var(--tw-space-y-reverse)))}.dashboard-table-wrap,.dashboard-activity-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:12px;overflow-x:auto}.dashboard-table{border-collapse:collapse;width:100%}.dashboard-table th{text-align:left;letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);background:var(--pb-bg-shell);border-bottom:1px solid var(--pb-border);padding:.45rem .9rem;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700}.dashboard-table th.c,.dashboard-table td.c{text-align:center}.dashboard-table th.r,.dashboard-table td.r{text-align:right}.dashboard-table td{border-bottom:1px solid var(--pb-border-subtle);padding:.6rem .9rem;font-size:.82rem;transition:background-color .12s}.dashboard-table tbody tr:last-child td{border-bottom:0}.dashboard-table tbody tr:hover td{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.dashboard-table tbody tr:hover td{background:color-mix(in srgb, var(--pb-interactive) 8%, var(--pb-bg-card))}}.dashboard-table__status-col{width:40px}.dashboard-table__name{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.dashboard-table__detail{margin-top:calc(var(--spacing) * .5);color:var(--pb-text-dim);font-size:.75rem}.table-mono-value,td.table-mono-value,.dashboard-table__mono,td.dashboard-table__mono,.health-table__mono,td.health-table__mono,.series-mission-control-owned,.series-mission-control-type,.utility-tool-table-mono,td.utility-tool-table-mono,.downloads-mono-cell,td.downloads-mono-cell{color:var(--pb-text-secondary);font-variant-numeric:tabular-nums;font-family:JetBrains Mono,monospace;font-size:.75rem;font-weight:400;line-height:1.2}.series-mission-control-reading{color:var(--pb-interactive);white-space:nowrap;margin-top:.22rem;font-family:JetBrains Mono,monospace;font-size:.58rem;font-weight:600;line-height:1.2;display:block}.dashboard-table__link{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.dashboard-table__link:hover{color:var(--pb-interactive-hover)}.dashboard-led{vertical-align:middle;border-radius:999px;flex-shrink:0;width:8px;height:8px;display:inline-block}.dashboard-led--sys{border:1px solid var(--pb-text);width:10px;height:10px}@supports (color:color-mix(in lab, red, red)){.dashboard-led--sys{border:1px solid color-mix(in srgb, var(--pb-text) 10%, transparent)}}.dashboard-led--green{background:var(--pb-success);box-shadow:0 0 5px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-led--green{box-shadow:0 0 5px color-mix(in srgb, var(--pb-success) 35%, transparent)}}.dashboard-led--amber{background:var(--pb-warning);box-shadow:0 0 5px var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-led--amber{box-shadow:0 0 5px color-mix(in srgb, var(--pb-warning) 35%, transparent)}}.dashboard-led--red{background:var(--pb-error);box-shadow:0 0 5px var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-led--red{box-shadow:0 0 5px color-mix(in srgb, var(--pb-error) 35%, transparent)}}.dashboard-led--blue{background:var(--pb-info);box-shadow:0 0 5px var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-led--blue{box-shadow:0 0 5px color-mix(in srgb, var(--pb-info) 35%, transparent)}}.dashboard-led--off{background:var(--pb-text-dim);opacity:.4}.dashboard-progress{align-items:center;gap:calc(var(--spacing) * 2);display:flex}.dashboard-progress__track{height:calc(var(--spacing) * 1.5);border-radius:var(--radius-xs);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background:var(--pb-text-dim);flex:1;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.dashboard-progress__track{background:color-mix(in srgb, var(--pb-text-dim) 10%, transparent)}}.dashboard-progress__fill{height:100%;position:relative}.dashboard-progress__fill:after{content:"";background:repeating-linear-gradient(90deg,#0000 0 2px,#ffffff1f 2px 4px);position:absolute;inset:0}.dashboard-progress__fill--green{background:linear-gradient(90deg, var(--pb-success), var(--pb-success))}@supports (color:color-mix(in lab, red, red)){.dashboard-progress__fill--green{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-success) 45%, transparent), var(--pb-success))}}.dashboard-progress__fill--blue{background:linear-gradient(90deg, var(--pb-info), var(--pb-info))}@supports (color:color-mix(in lab, red, red)){.dashboard-progress__fill--blue{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-info) 45%, transparent), var(--pb-info))}}.dashboard-progress__fill--pulse{opacity:.6;width:100%;animation:1.6s ease-in-out infinite pulse}.dashboard-progress__percent,.dashboard-activity-row__time,.dashboard-footer-strip{font-family:JetBrains Mono,monospace}.dashboard-progress__percent{text-align:right;min-width:32px;color:var(--pb-text-sec);font-size:.66rem;font-weight:600}.dashboard-activity-card{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2)}.dashboard-activity-row{align-items:flex-start;gap:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);border-bottom:1px solid var(--pb-border-subtle);display:flex}.dashboard-activity-row:last-child{border-bottom:0}.dashboard-activity-row__dot{margin-top:calc(var(--spacing) * 1.5);height:calc(var(--spacing) * 2);width:calc(var(--spacing) * 2);border-radius:3.40282e38px;flex-shrink:0}.dashboard-activity-row__dot--acquired{background:var(--pb-success)}.dashboard-activity-row__dot--imported{background:var(--pb-brand)}.dashboard-activity-row__dot--failed{background:var(--pb-error)}.dashboard-activity-row__body{flex:1}.dashboard-activity-row__summary{--tw-leading:calc(var(--spacing) * 6);font-size:.82rem;line-height:calc(var(--spacing) * 6);--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-primary)}.dashboard-activity-row__detail{--tw-leading:calc(var(--spacing) * 5);font-size:.78rem;line-height:calc(var(--spacing) * 5);color:var(--pb-text-secondary)}.dashboard-activity-row__meta{margin-top:var(--spacing);justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);display:flex}.dashboard-activity-row__time{color:var(--pb-text-dim);font-size:.66rem}.dashboard-activity-row__link{--tw-font-weight:var(--font-weight-semibold);font-size:.68rem;font-weight:var(--font-weight-semibold);color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.dashboard-activity-row__link:hover{color:var(--pb-interactive-hover)}.dashboard-activity-card__empty{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-dim)}.dashboard-footer-strip{align-items:center;column-gap:calc(var(--spacing) * 6);row-gap:calc(var(--spacing) * 2);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2.5);color:var(--pb-text-dim);box-shadow:var(--pb-shadow-1);border-radius:10px;flex-wrap:wrap;font-size:.72rem;display:flex}.dashboard-footer-strip strong{color:var(--pb-text);font-weight:600}.library-page-shell{flex:1;width:100%;min-height:0;overflow:hidden}.library-format-pills{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;display:flex}.library-scoreboard__item{text-decoration:none;transition:background-color .12s}.library-scoreboard__item--link:hover{background-color:var(--pb-bg-card-hover)}.library-scoreboard__value--info{color:var(--pb-info)}.library-scoreboard__value--success{color:var(--pb-success)}.library-scoreboard__value--warning{color:var(--pb-warning)}.library-matching-banner{justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));border-width:1px;border-color:var(--pb-warning);border-radius:12px;display:flex}@supports (color:color-mix(in lab, red, red)){.library-matching-banner{border-color:color-mix(in srgb, var(--pb-warning) 26%, transparent)}}.library-matching-banner{background:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.library-matching-banner{background:color-mix(in srgb, var(--pb-warning) 8%, var(--pb-bg-card))}}.library-matching-banner{box-shadow:var(--pb-shadow-1)}.library-matching-banner__body{align-items:center;column-gap:calc(var(--spacing) * 2);row-gap:var(--spacing);flex-wrap:wrap;min-width:0;display:flex}.library-matching-banner__title{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.library-matching-banner__copy{color:var(--pb-text-secondary)}.library-browser{--library-browser-header-height:calc(1.5rem + .75rem + 1px);border:1px solid var(--pb-border);background:var(--pb-bg-card);min-height:0;box-shadow:var(--pb-shadow-1);border-radius:14px;flex:1 1 0;grid-template-columns:minmax(220px,260px) minmax(0,1fr);display:grid;overflow:hidden}.library-browser__tree{overscroll-behavior:contain;z-index:2;border-right:1px solid var(--pb-border);background:var(--pb-bg-surface);flex-direction:column;min-height:0;display:flex;position:relative;overflow:hidden}.library-browser__tree-list{min-height:0;padding-block:var(--spacing);overscroll-behavior:contain;flex:1;overflow-y:auto}.library-browser__tree-header,.library-browser__toolbar{min-height:var(--library-browser-header-height);height:var(--library-browser-header-height)}.library-browser__tree-header{padding-inline:calc(var(--spacing) * 3);z-index:3;letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);border-bottom:1px solid var(--pb-border);background:var(--pb-bg-surface);align-items:center;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700;display:flex;position:sticky;top:0}@supports (color:color-mix(in lab, red, red)){.library-browser__tree-header{background:color-mix(in srgb, var(--pb-bg-surface) 72%, transparent)}}.library-browser__tree-node{min-height:calc(var(--spacing) * 6);border-radius:var(--radius-md);min-width:0;padding-inline:calc(var(--spacing) * 1.5);padding-block:var(--spacing);color:var(--pb-text-secondary);flex:1;align-items:center;font-size:.73rem;font-weight:400;line-height:1.2;text-decoration:none;display:flex}.library-browser__tree-node--active{background:var(--pb-interactive-dim);color:var(--pb-interactive);font-weight:500}.library-browser__tree-icon{height:calc(var(--spacing) * 3.5);width:calc(var(--spacing) * 3.5);color:var(--pb-text-dim);flex-shrink:0;transition:transform .14s,color .14s}.library-browser__tree-icon--folder{color:var(--pb-brand)}.library-browser__tree-icon--expanded{color:var(--pb-interactive);transform:rotate(90deg)}.library-browser__tree-item{padding-left:calc(var(--library-tree-level,0) * .8rem)}.library-browser__tree-row{min-height:calc(var(--spacing) * 6);align-items:center;gap:var(--spacing);min-width:0;padding-inline:calc(var(--spacing) * 2);display:flex}.library-browser__tree-row--context .library-browser__tree-node,.library-browser__tree-row--context .library-browser__tree-toggle,.library-browser__tree-row--context .library-browser__tree-leaf{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-browser__tree-row--context .library-browser__tree-node,.library-browser__tree-row--context .library-browser__tree-toggle,.library-browser__tree-row--context .library-browser__tree-leaf{background:color-mix(in srgb, var(--pb-interactive) 10%, transparent)}}.library-browser__tree-row--context .library-browser__tree-node,.library-browser__tree-row--context .library-browser__tree-toggle,.library-browser__tree-row--context .library-browser__tree-leaf{color:var(--pb-interactive)}.library-browser__tree-toggle,.library-browser__tree-leaf{height:calc(var(--spacing) * 6);width:calc(var(--spacing) * 6);color:var(--pb-text-dim);flex-shrink:0;justify-content:center;align-items:center;display:inline-flex}.library-browser__tree-toggle{border-radius:4px;transition:background-color .12s,color .12s}.library-browser__tree-toggle:hover{background:var(--pb-bg-card-hover);color:var(--pb-text)}:where(.library-browser__tree-children>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * .5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * .5) * calc(1 - var(--tw-space-y-reverse)))}.library-browser__tree-name{flex:1;min-width:0;display:block}.library-browser__panel{z-index:1;flex-direction:column;min-width:0;min-height:0;display:flex;position:relative;overflow:hidden}.library-browser__toolbar{align-items:center;gap:calc(var(--spacing) * 2);padding-inline:calc(var(--spacing) * 3);border-bottom:1px solid var(--pb-border);background:var(--pb-bg-surface);flex-shrink:0;display:flex}@supports (color:color-mix(in lab, red, red)){.library-browser__toolbar{background:color-mix(in srgb, var(--pb-bg-surface) 72%, transparent)}}.library-browser__toolbar-main{align-items:center;gap:calc(var(--spacing) * 1.5);flex:1;min-width:0;display:flex}.library-browser__up-btn{height:calc(var(--spacing) * 6);width:calc(var(--spacing) * 6);border-radius:var(--radius-xs);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-shrink:0;justify-content:center;align-items:center;text-decoration:none;display:inline-flex}.library-browser__up-btn:hover{background:var(--pb-bg-card-hover);color:var(--pb-text)}.library-browser__up-btn.is-disabled{pointer-events:none;opacity:.4}.library-browser__path-block{flex:1;align-items:center;min-width:0;display:flex}.library-browser__path{color:var(--pb-text-sec);white-space:nowrap;text-overflow:ellipsis;font-family:JetBrains Mono,monospace;font-size:.56rem;line-height:1.15;overflow:hidden}.library-browser__table-wrap{overscroll-behavior:contain;min-height:0;box-shadow:none;background:0 0;border:0;border-radius:0;flex:1;position:relative;overflow:auto}.library-browser-table{min-width:760px}.library-browser-table th{padding:.4rem .75rem;font-size:.54rem;font-weight:600}.library-browser-table td{padding:.5rem .75rem;font-size:.76rem;font-weight:400;line-height:1.3}.library-browser__table-row--file,.library-browser__table-row--file td,.library-browser__table-row--file .library-browser__name-label{cursor:pointer}.library-browser__table-row--context{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-browser__table-row--context{background:color-mix(in srgb, var(--pb-interactive) 8%, transparent)}}.library-context-menu{border-radius:var(--radius-xl);border-style:var(--tw-border-style);min-width:15.5rem;padding-block:calc(var(--spacing) * 1.5);z-index:80;border-width:1px;border-color:var(--pb-border);position:fixed;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.library-context-menu{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-context-menu{background:linear-gradient(180deg, var(--pb-bg-card) 0%, var(--pb-bg-surface) 100%)}@supports (color:color-mix(in lab, red, red)){.library-context-menu{background:linear-gradient(180deg, color-mix(in srgb, var(--pb-bg-card) 94%, transparent) 0%, color-mix(in srgb, var(--pb-bg-surface) 97%, transparent) 100%)}}.library-context-menu{-webkit-backdrop-filter:blur(12px);backdrop-filter:blur(12px);box-shadow:0 18px 48px #0f172a42,0 6px 18px #0f172a29}.library-context-menu__group{display:contents}.library-context-menu__item{align-items:center;gap:calc(var(--spacing) * 3);width:100%;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);text-align:left;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));color:var(--pb-text);display:flex}.library-context-menu__item:hover{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-context-menu__item:hover{background:color-mix(in srgb, var(--pb-interactive) 10%, transparent)}}.library-context-menu__item:hover{color:var(--pb-interactive)}.library-context-menu__item--danger:hover{background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-context-menu__item--danger:hover{background:color-mix(in srgb, var(--pb-error) 11%, transparent)}}.library-context-menu__item--danger:hover{color:var(--pb-error)}.library-context-menu__icon{height:calc(var(--spacing) * 4);width:calc(var(--spacing) * 4);color:currentColor;flex-shrink:0}.library-context-menu__sep{margin-block:var(--spacing);background:var(--pb-border-subtle);height:1px}@supports (color:color-mix(in lab, red, red)){.library-context-menu__sep{background:color-mix(in srgb, var(--pb-border-subtle) 80%, transparent)}}.modal-panel.library-browser-modal{border-radius:18px;width:min(92vw,48rem);max-height:min(86vh,52rem);overflow:hidden}.modal-panel.library-browser-modal--properties{width:min(84vw,28rem)}.modal-panel.library-browser-modal--compact,.modal-panel.library-delete-modal{width:min(92vw,34rem)}.library-delete-modal__header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 4);border-bottom-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);border-bottom-width:1px;border-color:var(--pb-border);display:flex}.library-delete-modal__eyebrow{letter-spacing:.14em;text-transform:uppercase;color:var(--pb-text-dim);font-size:11px;font-weight:500}.library-delete-modal__title{margin-top:var(--spacing);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.library-delete-modal__subtitle{margin-top:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}:where(.library-delete-modal__body>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.library-delete-modal__body{padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4)}.library-delete-modal__copy{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}.library-delete-modal__impact{border-radius:var(--radius-lg);border-style:var(--tw-border-style);padding:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border-hover)}@supports (color:color-mix(in lab, red, red)){.library-delete-modal__impact{border-color:color-mix(in srgb, var(--pb-border-hover) 88%, transparent)}}.library-delete-modal__impact{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-delete-modal__impact{background:color-mix(in srgb, var(--pb-bg-surface) 40%, transparent)}}.library-delete-modal__impact .library-browser-modal__details-table{margin-top:0}.series-delete-modal__options{gap:calc(var(--spacing) * 3);flex-direction:column;display:flex}.series-delete-modal__option{align-items:flex-start;gap:calc(var(--spacing) * 3);border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border);display:flex}@supports (color:color-mix(in lab, red, red)){.series-delete-modal__option{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.series-delete-modal__option{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.series-delete-modal__option{background:color-mix(in srgb, var(--pb-bg-surface) 82%, transparent)}}.series-delete-modal__checkbox{margin-top:calc(var(--spacing) * .5);border-color:var(--pb-border-hover);background:var(--pb-bg-card-hover);color:var(--pb-error);border-radius:.25rem}.series-delete-modal__checkbox:focus{--tw-ring-color:var(--pb-error);--tw-ring-offset-width:0px}.series-delete-modal__option-copy{min-width:0}.series-delete-modal__option-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-primary)}.series-delete-modal__option-text{margin-top:calc(var(--spacing) * .5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary);line-height:1.5}.library-delete-modal__footer{justify-content:flex-end;align-items:center;gap:calc(var(--spacing) * 2);border-top-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 3);border-top-width:1px;border-color:var(--pb-border);background:var(--pb-bg-card);display:flex}@supports (color:color-mix(in lab, red, red)){.library-delete-modal__footer{background:color-mix(in srgb, var(--pb-bg-card) 80%, transparent)}}.library-browser-modal__header{align-items:flex-start;gap:calc(var(--spacing) * 4);background:linear-gradient(180deg, var(--pb-bg-surface) 0%, var(--pb-bg-card) 100%)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__header{background:linear-gradient(180deg, color-mix(in srgb, var(--pb-bg-surface) 95%, transparent) 0%, color-mix(in srgb, var(--pb-bg-card) 96%, transparent) 100%)}}.library-browser-modal--properties .library-browser-modal__header,.library-browser-modal--properties .library-browser-modal__body{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}.library-browser-modal--properties .modal-footer{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);justify-content:flex-end}.library-browser-modal--form .library-browser-modal__body{padding-block:0;padding-inline:0}.library-browser-modal--form .modal-footer{padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 3)}.library-browser-modal .modal-footer{justify-content:flex-end;gap:calc(var(--spacing) * 2);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 3);background:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal .modal-footer{background:color-mix(in srgb, var(--pb-bg-card) 80%, transparent)}}.library-browser-modal__title-block{align-items:flex-start;gap:calc(var(--spacing) * 3);min-width:0;display:flex}.library-browser-modal__icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);color:var(--pb-interactive);border-width:1px;border-color:var(--pb-interactive);flex-shrink:0;justify-content:center;align-items:center;display:flex}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon{border-color:color-mix(in srgb, var(--pb-interactive) 24%, transparent)}}.library-browser-modal__icon{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon{background:color-mix(in srgb, var(--pb-interactive) 10%, transparent)}}.library-browser-modal__icon svg{height:calc(var(--spacing) * 5);width:calc(var(--spacing) * 5)}.library-browser-modal__icon--warning{color:var(--pb-warning);border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon--warning{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.library-browser-modal__icon--warning{background:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon--warning{background:color-mix(in srgb, var(--pb-warning) 10%, transparent)}}.library-browser-modal__icon--danger{color:var(--pb-error);border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon--danger{border-color:color-mix(in srgb, var(--pb-error) 24%, transparent)}}.library-browser-modal__icon--danger{background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon--danger{background:color-mix(in srgb, var(--pb-error) 10%, transparent)}}.library-browser-modal--properties .library-browser-modal__icon{height:calc(var(--spacing) * 9);width:calc(var(--spacing) * 9);border-radius:var(--radius-xl)}.library-browser-modal--properties .library-browser-modal__icon svg{width:1.125rem;height:1.125rem}.library-browser-modal__title{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.library-browser-modal__subtitle{margin-top:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}.library-browser-modal__close{border-radius:var(--radius-lg);padding:calc(var(--spacing) * 1.5);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.library-browser-modal__close:hover{background:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__close:hover{background:color-mix(in srgb, var(--pb-bg-card-hover) 88%, transparent)}}.library-browser-modal__close:hover{color:var(--pb-text)}.library-browser-modal__close:focus-visible{outline:2px solid var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__close:focus-visible{outline:2px solid color-mix(in srgb, var(--pb-interactive) 42%, transparent)}}.library-browser-modal__close:focus-visible{outline-offset:2px}.library-browser-modal__body{min-height:0}.library-browser-modal__form{flex-direction:column;flex:1;min-height:0;display:flex}.library-browser-modal__content{gap:calc(var(--spacing) * 4);flex-direction:column;display:flex}.library-browser-modal__section-label{margin-bottom:calc(var(--spacing) * 2);padding-bottom:calc(var(--spacing) * 2);letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);border-bottom:1px solid var(--pb-border-subtle);font-family:Syne,sans-serif;font-size:.64rem;font-weight:700;display:block}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__section-label{border-bottom:1px solid color-mix(in srgb, var(--pb-border-subtle) 80%, transparent)}}.library-browser-modal__section{padding-bottom:var(--spacing)}.library-browser-modal__details-table{border-collapse:collapse;table-layout:fixed;width:100%;margin-top:.625rem;font-size:.8rem}.library-browser-modal__details-table--flush{margin-top:0}.library-browser-modal__detail-label{vertical-align:top;width:6.875rem;color:var(--pb-text-dim);font-size:inherit;text-align:left;padding:.25rem .75rem .25rem 0;font-weight:500;line-height:1.35}.library-browser-modal__detail-value{vertical-align:top;min-width:0;color:var(--pb-text);font-size:inherit;overflow-wrap:anywhere;padding:.25rem 0;font-weight:500;line-height:1.4}.library-browser-modal__detail-value--mono{color:var(--pb-text-sec);font-family:JetBrains Mono,monospace;font-size:.76rem;line-height:1.45}.library-browser-modal__loading{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 6);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);border-width:1px;border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__loading{border-color:color-mix(in srgb, var(--pb-border) 86%, transparent)}}.library-browser-modal__loading{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__loading{background:color-mix(in srgb, var(--pb-bg-surface) 94%, transparent)}}.library-browser-modal__stats{gap:calc(var(--spacing) * 3);grid-template-columns:repeat(auto-fit,minmax(8.5rem,1fr));display:grid}.library-browser-modal__stat{justify-content:space-between;gap:calc(var(--spacing) * 2);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);min-height:5rem;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border);flex-direction:column;display:flex}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__stat{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__stat{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__stat{background:color-mix(in srgb, var(--pb-bg-surface) 94%, transparent)}}.library-browser-modal__stat-label,.library-browser-modal__field-label{letter-spacing:.11em;text-transform:uppercase;color:var(--pb-text-dim);font-family:Syne,sans-serif;font-size:.62rem;font-weight:700}.library-browser-modal__stat-value,.library-browser-modal__preview-value{min-width:0;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-primary);overflow-wrap:anywhere;font-weight:500;line-height:1.35}.library-browser-modal__meta{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__meta{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__meta{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__meta{background:color-mix(in srgb, var(--pb-bg-surface) 93%, transparent)}}.library-browser-modal__meta-row{gap:calc(var(--spacing) * 2);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);grid-template-columns:minmax(8rem,10rem) minmax(0,1fr);display:grid}.library-browser-modal__meta-row+.library-browser-modal__meta-row{border-top:1px solid var(--pb-border-subtle)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__meta-row+.library-browser-modal__meta-row{border-top:1px solid color-mix(in srgb, var(--pb-border-subtle) 84%, transparent)}}.library-browser-modal__meta-label{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-dim);letter-spacing:.08em;text-transform:uppercase}.library-browser-modal__meta-value{min-width:0;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);overflow-wrap:anywhere;font-family:JetBrains Mono,monospace;font-size:.76rem;line-height:1.45}.library-browser-modal__storage{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__storage{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__storage{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__storage{background:color-mix(in srgb, var(--pb-bg-surface) 93%, transparent)}}.library-browser-modal__storage-head{margin-bottom:calc(var(--spacing) * 2);justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);display:flex}.library-browser-modal__storage-bar{height:calc(var(--spacing) * 2.5);background:var(--pb-bg-card-hover);border-radius:3.40282e38px;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__storage-bar{background:color-mix(in srgb, var(--pb-bg-card-hover) 72%, transparent)}}.library-browser-modal__storage-fill{background:linear-gradient(90deg, var(--pb-interactive) 0%, var(--pb-brand) 100%);border-radius:3.40282e38px;height:100%;display:block}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__storage-fill{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-interactive) 82%, white) 0%, color-mix(in srgb, var(--pb-brand) 74%, white) 100%)}}.library-browser-modal__field{gap:calc(var(--spacing) * 2);flex-direction:column;display:flex}.library-browser-modal__settings-rows{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border)}.library-browser-modal__note{padding-inline:calc(var(--spacing) * 6)}.library-browser-modal__body-inset{padding-inline:calc(var(--spacing) * 6);padding-bottom:calc(var(--spacing) * 4)}.library-browser-modal__body-inset--section{padding-bottom:0}.library-browser-modal__body-inset--compact-top{padding-top:calc(var(--spacing) * 3)}.library-browser-modal__body-inset--roomy-bottom{padding-bottom:calc(var(--spacing) * 4)}.library-browser-modal__section-stack{gap:calc(var(--spacing) * 4);padding-top:calc(var(--spacing) * 4);flex-direction:column;display:flex}.library-browser-modal__section-stack--delete{gap:calc(var(--spacing) * 4)}.library-browser-modal__section-panel{flex-direction:column;display:flex}.library-browser-modal__section-heading-row{padding-inline:calc(var(--spacing) * 6);padding-bottom:calc(var(--spacing) * 3);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-bottom:1px solid var(--pb-border-subtle);padding-top:0}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__section-heading-row{border-bottom:1px solid color-mix(in srgb, var(--pb-border-subtle) 80%, transparent)}}.library-browser-modal__preview-block{min-height:calc(var(--spacing) * 10);border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);border-width:1px;border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__preview-block{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__preview-block{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__preview-block{background:color-mix(in srgb, var(--pb-bg-surface) 94%, transparent)}}.library-browser-modal__preview-block{overflow-wrap:anywhere;line-height:1.45}.library-browser-modal__preview-block--mono{font-family:JetBrains Mono,monospace;font-size:.76rem}.library-browser-modal__detail-input{width:100%;min-height:2.5rem;font-size:.88rem;display:block}.library-browser-modal__helper,.library-browser-modal__context-value{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.library-browser-modal__context-value--mono{color:var(--pb-text-sec);overflow-wrap:anywhere;font-family:JetBrains Mono,monospace;font-size:.76rem;line-height:1.45}.library-browser-modal__error{border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-error);border-width:1px;border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__error{border-color:color-mix(in srgb, var(--pb-error) 24%, transparent)}}.library-browser-modal__error{background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__error{background:color-mix(in srgb, var(--pb-error) 8%, transparent)}}.library-browser-modal__actions{align-items:center;gap:calc(var(--spacing) * 2);display:flex}.library-browser-modal__preview-grid{gap:calc(var(--spacing) * 3);grid-template-columns:repeat(auto-fit,minmax(10rem,1fr));display:grid}.library-browser-modal__preview-card{justify-content:space-between;gap:calc(var(--spacing) * 2);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);min-height:5.25rem;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border);flex-direction:column;display:flex}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__preview-card{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__preview-card{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__preview-card{background:color-mix(in srgb, var(--pb-bg-surface) 94%, transparent)}}.library-browser__name{align-items:center;gap:calc(var(--spacing) * 2);display:flex}.library-browser__name-icon{height:calc(var(--spacing) * 3.5);width:calc(var(--spacing) * 3.5);flex-shrink:0}.library-browser__name-icon--folder{color:var(--pb-brand)}.library-browser__name-icon--file{color:var(--pb-text-dim)}.library-browser__name-label{text-overflow:ellipsis;white-space:nowrap;min-width:0;font-size:.76rem;font-weight:400;overflow:hidden}.library-browser__name-label--folder{color:var(--pb-text-primary);font-weight:500;text-decoration:none}.library-browser__name-label--folder:hover{color:var(--pb-interactive)}.library-browser__empty{height:100%;min-height:280px;padding-inline:calc(var(--spacing) * 6);text-align:center;flex-direction:column;justify-content:center;align-items:center;display:flex}.library-browser__empty-icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);color:var(--pb-text-dim)}.library-browser__empty-title{margin-top:calc(var(--spacing) * 4);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.library-browser__empty-copy{margin-top:calc(var(--spacing) * 2);max-width:var(--container-xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.library-empty-shell{align-items:flex-start;gap:calc(var(--spacing) * 4);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 8);box-shadow:var(--pb-shadow-1);border-radius:14px;flex-direction:column;display:flex}.library-empty-shell__icon-wrap{height:calc(var(--spacing) * 16);width:calc(var(--spacing) * 16);background-color:var(--pb-bg-card-hover);color:var(--pb-text-dim);border-radius:3.40282e38px;justify-content:center;align-items:center;display:flex}.library-empty-shell__icon{height:calc(var(--spacing) * 8);width:calc(var(--spacing) * 8)}:where(.library-empty-shell__body>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}.library-empty-shell__copy{max-width:var(--container-2xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 7);line-height:calc(var(--spacing) * 7);color:var(--pb-text-secondary)}.library-empty-shell__actions{align-items:center;gap:calc(var(--spacing) * 3);flex-wrap:wrap;display:flex}@media (max-width:1024px){.library-browser{flex:none;grid-template-columns:1fr;height:auto;min-height:520px}.library-browser__tree{border-right:0;border-bottom:1px solid var(--pb-border);max-height:220px}}.health-section__label{letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);margin-block:.625rem;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700;display:block}.health-component-card__sub,.health-detail-card__stat-value,.health-history-table td{font-family:JetBrains Mono,monospace}:where(.health-section>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2.5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2.5) * calc(1 - var(--tw-space-y-reverse)))}.health-component-grid{gap:calc(var(--spacing) * 3);grid-template-columns:repeat(auto-fit,minmax(280px,1fr));display:grid}.health-component-card{cursor:pointer;gap:calc(var(--spacing) * 3);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-top:calc(var(--spacing) * 4);padding-bottom:calc(var(--spacing) * 3);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-width:1px;border-radius:14px;flex-direction:column;height:100%;display:flex}.health-component-card:hover{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.health-component-card:hover{background:color-mix(in srgb, var(--pb-interactive) 4%, var(--pb-bg-card))}}.health-component-card:hover{border-color:var(--pb-border-strong)}.health-component-card--success{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.health-component-card--success{border-color:color-mix(in srgb, var(--pb-success) 22%, var(--pb-border))}}.health-component-card--warning{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.health-component-card--warning{border-color:color-mix(in srgb, var(--pb-warning) 22%, var(--pb-border))}}.health-component-card--danger{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.health-component-card--danger{border-color:color-mix(in srgb, var(--pb-error) 22%, var(--pb-border))}}.health-component-card--neutral{border-color:var(--pb-border)}.health-component-card__header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 3);display:flex}.health-component-card__name{text-transform:uppercase;letter-spacing:.04em;color:var(--pb-text);font-family:Syne,sans-serif;font-size:.84rem;font-weight:700}.health-component-card__stats{align-items:flex-start;gap:calc(var(--spacing) * 2);grid-template-columns:repeat(2,minmax(0,1fr));display:grid}.health-detail-card__stats{gap:calc(var(--spacing) * 2);display:grid}@media (min-width:40rem){.health-detail-card__stats{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.health-detail-card__stats{grid-template-columns:repeat(4,minmax(0,1fr))}}.health-component-card__stat,.health-detail-card__stat{border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);border-width:1px;border-color:var(--pb-border-subtle);border-radius:10px;flex-direction:column;gap:0;display:flex}.health-component-card__stat{background:var(--pb-bg-base)}.health-detail-card__stat{padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 1.5);background:var(--pb-bg-base)}.health-component-card__stat-label,.health-detail-card__stat-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);font-family:Syne,sans-serif;font-size:.46rem;font-weight:700}.health-component-card__stat-value,.health-detail-card__stat-value{--tw-font-weight:var(--font-weight-semibold);font-size:.82rem;font-weight:var(--font-weight-semibold);color:var(--pb-text-primary);display:block}.health-component-card__stat-value{white-space:pre-line;line-height:1.25}.health-component-card__stat-value--danger,.health-detail-card__stat-value--danger{color:var(--pb-error)}.health-component-card__stat-value--warning,.health-detail-card__stat-value--warning{color:var(--pb-warning)}.health-component-card__message{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary);min-height:2.5rem}.health-component-card__sub{color:var(--pb-text-dim);margin-top:auto;font-size:.68rem}:where(.health-detail-shell>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}.health-detail-back{align-items:center;gap:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.health-detail-back:hover{color:var(--pb-interactive-hover)}:where(.health-detail-card>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.health-detail-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:14px}.health-detail-card__header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 3);flex-wrap:wrap;display:flex}.health-detail-card__title-row{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;min-width:0;display:flex}.health-detail-card__title{letter-spacing:.03em;text-transform:uppercase;color:var(--pb-text);font-family:Syne,sans-serif;font-size:1rem;font-weight:800}:where(.health-check-list>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}.health-check-row{align-items:center;gap:calc(var(--spacing) * 3);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);border-radius:10px;grid-template-columns:8px minmax(120px,140px) minmax(0,1fr) auto;display:grid}.health-check-row--database-metric{grid-template-columns:8px minmax(160px,180px) minmax(0,1fr) auto}.health-check-row__name{text-align:left;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-primary);justify-self:flex-start}.health-check-row__message{min-width:0;font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5);color:var(--pb-text-dim);flex:1}.health-check-list--database-metrics .health-check-row__name,.health-check-list--database-metrics .health-check-row__message{text-align:left;justify-self:flex-start}.health-check-list--database-metrics .health-led{justify-self:center}.health-table-wrap{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);border-radius:12px;overflow-x:auto}.health-table{border-collapse:collapse;width:100%}.health-table th{text-align:left;letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);background:var(--pb-bg-shell);border-bottom:1px solid var(--pb-border);padding:.45rem .85rem;font-family:Syne,sans-serif;font-size:.52rem;font-weight:700}.health-table td{border-bottom:1px solid var(--pb-border-subtle);color:var(--pb-text-sec);padding:.55rem .85rem;font-size:.76rem}.health-table tbody tr:last-child td{border-bottom:0}.health-table tbody tr:hover td{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.health-table tbody tr:hover td{background:color-mix(in srgb, var(--pb-interactive) 8%, var(--pb-bg-card))}}.health-table th.c,.health-table td.c{text-align:center}.health-table th.r,.health-table td.r{text-align:right}.health-table__status-col{width:40px}.health-table__name{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.health-table__detail{margin-top:calc(var(--spacing) * .5);color:var(--pb-text-dim);font-size:.72rem}.health-led{border-radius:999px;flex-shrink:0;width:8px;height:8px;display:inline-block}.health-led--header{width:10px;height:10px}.health-led--green{background:var(--pb-success);box-shadow:0 0 5px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.health-led--green{box-shadow:0 0 5px color-mix(in srgb, var(--pb-success) 35%, transparent)}}.health-led--amber{background:var(--pb-warning);box-shadow:0 0 5px var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.health-led--amber{box-shadow:0 0 5px color-mix(in srgb, var(--pb-warning) 35%, transparent)}}.health-led--red{background:var(--pb-error);box-shadow:0 0 5px var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.health-led--red{box-shadow:0 0 5px color-mix(in srgb, var(--pb-error) 35%, transparent)}}.health-led--off{background:var(--pb-text-dim);opacity:.4}.health-footer-strip{align-items:center;column-gap:calc(var(--spacing) * 6);row-gap:var(--spacing);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2);color:var(--pb-text-dim);border-radius:10px;flex-wrap:wrap;font-family:JetBrains Mono,monospace;font-size:.72rem;display:flex}.health-footer-strip strong{color:var(--pb-text);font-weight:600}.step-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1)}.step-badge{height:calc(var(--spacing) * 8);width:calc(var(--spacing) * 8);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-info);background-color:var(--pb-info);border-radius:3.40282e38px;justify-content:center;align-items:center;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.step-badge{background-color:color-mix(in srgb, var(--pb-info) 14%, transparent)}}.step-badge{border:1px solid var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.step-badge{border:1px solid color-mix(in srgb, var(--pb-info) 28%, transparent)}}.action-card{align-items:flex-start;gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);padding:calc(var(--spacing) * 4);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background-color:var(--pb-bg-card-hover);display:flex}@supports (color:color-mix(in lab, red, red)){.action-card{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 40%, transparent)}}.action-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.action-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 82%, transparent)}}.action-card-icon{height:calc(var(--spacing) * 11);width:calc(var(--spacing) * 11);border-radius:var(--radius-xl);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-shrink:0;justify-content:center;align-items:center;display:flex}.action-card-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.action-card-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5);color:var(--pb-text-dim)}.utility-launch-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);height:100%;padding:calc(var(--spacing) * 5);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;box-shadow:var(--pb-shadow-1);flex-direction:column;transition-duration:.15s;display:flex}.utility-launch-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.utility-launch-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 82%, transparent)}}.utility-launch-card:hover{box-shadow:var(--pb-shadow-2)}.utility-launch-icon-brand{background-color:var(--pb-brand-dim);color:var(--pb-brand)}.utility-launch-icon-interactive{background-color:var(--pb-interactive-dim);color:var(--pb-interactive)}.utility-launch-icon-info{background-color:var(--pb-info-dim);color:var(--pb-info)}.utility-launch-icon-success{background-color:var(--pb-success-dim);color:var(--pb-success)}.utility-launch-icon-warning{background-color:var(--pb-warning-dim);color:var(--pb-warning)}.utility-launch-kicker{margin-top:calc(var(--spacing) * 4);--tw-font-weight:var(--font-weight-medium);font-size:10px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.utility-launch-title{margin-top:calc(var(--spacing) * 2);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.utility-launch-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utility-launch-footer{justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);padding-top:calc(var(--spacing) * 4);margin-top:auto;display:flex}.utility-launch-tag{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.utility-launch-link{align-items:center;gap:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.utility-launch-link:hover{color:var(--pb-interactive-hover)}.utility-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.utility-hero-card{padding-inline:calc(var(--spacing) * 7)}}.utility-hero-card{box-shadow:var(--pb-shadow-2)}.utility-top-link{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 1.5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1);border-radius:3.40282e38px;align-items:center;display:inline-flex}.utility-top-link:hover{border-color:var(--pb-border-hover);color:var(--pb-text);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.utility-top-link:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 58%, transparent)}}.utility-top-link-muted{color:var(--pb-text-dim)}.utility-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.utility-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 42%, transparent), transparent)}}.utility-hero-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.utility-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.95fr);align-items:flex-start}}.utility-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.utility-hero-title{margin-top:calc(var(--spacing) * 2);font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.utility-hero-copy{margin-top:calc(var(--spacing) * 2);max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utility-hero-highlights{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:48rem){.utility-hero-highlights{grid-template-columns:repeat(3,minmax(0,1fr))}}.utility-hero-highlight{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.utility-hero-highlight-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.utility-hero-highlight-value{margin-top:calc(var(--spacing) * 2);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.utility-hero-highlight-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utility-hero-actions{gap:calc(var(--spacing) * 3);display:grid}.utility-hero-action{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1)}.utility-hero-action:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.utility-hero-action:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 58%, transparent)}}.utility-hero-action-primary{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.utility-hero-action-primary{border-color:color-mix(in srgb, var(--pb-interactive) 26%, transparent)}}.utility-hero-action-primary{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.utility-hero-action-primary{background-color:color-mix(in srgb, var(--pb-interactive) 8%, var(--pb-bg-card))}}.utility-hero-action-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.utility-hero-action-title{margin-top:calc(var(--spacing) * 2);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.utility-hero-action-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utility-hero-notes{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.utility-hero-notes{grid-template-columns:repeat(3,minmax(0,1fr))}}@media (min-width:80rem){.utility-hero-notes{grid-template-columns:repeat(1,minmax(0,1fr))}}.utility-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.utility-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.utility-hero-note-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utilities-page{width:100%;max-width:1100px;padding-bottom:var(--pb-page-footer-clearance);margin-inline:auto}.utilities-shell{width:100%}:where(.utilities-content>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.utilities-header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 5);flex-wrap:wrap;display:flex}.utilities-header-copy{align-items:flex-start;gap:calc(var(--spacing) * 5);flex-wrap:wrap;display:flex}.utilities-header-title{letter-spacing:.04em;text-transform:uppercase;font-family:Syne,sans-serif;font-size:1.6rem;font-weight:800;line-height:1}.utilities-header-title span{color:var(--pb-brand)}.utilities-header-subtitle{color:var(--pb-text-dim);margin-top:4px;font-size:.78rem;font-weight:400;line-height:1.5}.utilities-gauges{align-items:flex-end;gap:calc(var(--spacing) * 3.5);display:flex}.utilities-gauge{text-align:center}.utilities-gauge-ring{height:calc(var(--spacing) * 12);width:calc(var(--spacing) * 12);border-style:var(--tw-border-style);background-color:var(--pb-bg-card);border-width:4px;border-radius:3.40282e38px;justify-content:center;align-items:center;margin-inline:auto;display:flex;position:relative}@supports (color:color-mix(in lab, red, red)){.utilities-gauge-ring{background-color:color-mix(in srgb, var(--pb-bg-card) 70%, var(--pb-bg-base))}}.utilities-gauge-ring-success{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.utilities-gauge-ring-success{border-color:color-mix(in srgb, var(--pb-success) 60%, var(--pb-border))}}.utilities-gauge-ring-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.utilities-gauge-ring-info{border-color:color-mix(in srgb, var(--pb-info) 60%, var(--pb-border))}}.utilities-gauge-ring-ink{border-color:var(--pb-text)}@supports (color:color-mix(in lab, red, red)){.utilities-gauge-ring-ink{border-color:color-mix(in srgb, var(--pb-text) 22%, var(--pb-border))}}.utilities-gauge-value{font-family:JetBrains Mono,monospace;font-size:.68rem;font-weight:700}.utilities-gauge-label{--tw-font-weight:var(--font-weight-medium);font-size:.52rem;font-weight:var(--font-weight-medium);color:var(--pb-text-dim);text-transform:uppercase;letter-spacing:.1em;margin-top:2px;display:block}.utilities-tabs{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);border-radius:10px;align-self:center;display:inline-flex;overflow:hidden}.utilities-tab{padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 2);--tw-font-weight:var(--font-weight-bold);font-family:Syne,sans-serif;font-size:.78rem;font-weight:var(--font-weight-bold);--tw-tracking:.08em;letter-spacing:.08em;color:var(--pb-text-dim);text-transform:uppercase;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.utilities-tab+.utilities-tab{border-left:1px solid var(--pb-border)}.utilities-tab:hover{color:var(--pb-text);background-color:var(--pb-text)}@supports (color:color-mix(in lab, red, red)){.utilities-tab:hover{background-color:color-mix(in srgb, var(--pb-text) 3%, transparent)}}.utilities-tab.is-active{background-color:var(--pb-interactive);color:var(--pb-text-inverse)}.utilities-section-label{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-bottom:calc(var(--spacing) * 1.5);--tw-font-weight:var(--font-weight-bold);font-family:Syne,sans-serif;font-size:.62rem;font-weight:var(--font-weight-bold);--tw-tracking:.12em;letter-spacing:.12em;color:var(--pb-text-dim);text-transform:uppercase}.utilities-tool-grid{grid-template-columns:repeat(auto-fill,minmax(240px,1fr));gap:.75rem;display:grid}.utilities-tool-grid-wide{grid-template-columns:repeat(auto-fill,minmax(280px,1fr))}.utility-launch-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);height:100%;padding:calc(var(--spacing) * 5);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;box-shadow:var(--pb-shadow-1);border-radius:14px;flex-direction:column;transition-duration:.15s;display:flex;overflow:hidden}.utility-launch-card:hover{border-color:var(--pb-border-strong);box-shadow:var(--pb-shadow-2);background-color:var(--pb-bg-card);transform:translateY(-2px)}.utility-launch-header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 3);display:flex}.utility-launch-arrow{color:var(--pb-text-dim);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;transition-duration:.15s}.utility-launch-card:hover .utility-launch-arrow{color:var(--pb-interactive);transform:translate(2px)}.utility-launch-icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:10px;justify-content:center;align-items:center;display:flex}.utility-launch-title{margin-top:calc(var(--spacing) * 3);--tw-font-weight:var(--font-weight-extrabold);font-family:Syne,sans-serif;font-size:.9rem;font-weight:var(--font-weight-extrabold);--tw-tracking:.04em;letter-spacing:.04em;color:var(--pb-text-primary);text-transform:uppercase}.utility-launch-copy{margin-top:calc(var(--spacing) * 1.5);color:var(--pb-text-secondary);flex:1;font-size:.78rem;font-weight:400;line-height:1.5}.utility-launch-footer{margin-top:calc(var(--spacing) * 4);justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);display:flex}.utility-launch-tag{border-radius:var(--radius-md);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 2);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-family:JetBrains Mono,monospace;font-weight:var(--font-weight-medium);color:var(--pb-text-dim);text-transform:uppercase;letter-spacing:.06em;align-items:center;font-size:.6rem;display:inline-flex}.utility-tool-page{width:100%;max-width:1100px;margin-inline:auto}:where(.utility-tool-header-block>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}.utility-tool-back-link{align-items:center;gap:var(--spacing);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));font-size:.82rem;display:inline-flex}@media (hover:hover){.utility-tool-back-link:hover{color:var(--pb-text-primary)}}.utility-tool-header-shell{justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 4);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:14px;flex-wrap:wrap;display:flex}.utility-tool-header-left{align-items:center;gap:calc(var(--spacing) * 3.5);display:flex}.utility-tool-header-icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:10px;justify-content:center;align-items:center;display:flex}.utility-tool-header-icon-converter{background-color:var(--pb-interactive-dim);color:var(--pb-interactive)}.utility-tool-header-icon-mass-convert{background-color:var(--pb-success-dim);color:var(--pb-success)}.utility-tool-header-icon-integrity{background-color:var(--pb-info-dim);color:var(--pb-info)}.utility-tool-header-icon-rename,.utility-tool-header-icon-db-check{background-color:var(--pb-warning-dim);color:var(--pb-warning)}.utility-tool-header-icon-export{background-color:var(--pb-brand-dim);color:var(--pb-brand)}.utility-tool-header-title{--tw-font-weight:var(--font-weight-extrabold);font-family:Syne,sans-serif;font-size:1.1rem;font-weight:var(--font-weight-extrabold);--tw-tracking:.03em;letter-spacing:.03em;color:var(--pb-text-primary);text-transform:uppercase}.utility-tool-header-title span{color:var(--pb-brand)}.utility-tool-header-subtitle{margin-top:calc(var(--spacing) * .5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.utility-tool-header-tag{border-radius:var(--radius-md);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-semibold);font-family:JetBrains Mono,monospace;font-size:.62rem;font-weight:var(--font-weight-semibold);--tw-tracking:.08em;letter-spacing:.08em;color:var(--pb-text-dim);text-transform:uppercase;align-items:center;display:inline-flex}.system-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.system-hero-card{padding-inline:calc(var(--spacing) * 7)}}.system-hero-card{box-shadow:var(--pb-shadow-2)}.system-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.system-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 42%, transparent), transparent)}}.system-hero-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.system-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.95fr);align-items:flex-start}}.system-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.system-hero-title{font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.system-hero-copy{max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.system-hero-aside{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.system-hero-aside{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.system-hero-aside{grid-template-columns:repeat(1,minmax(0,1fr))}}.system-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.system-hero-note-emphasis{border-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.system-hero-note-emphasis{border-color:color-mix(in srgb, var(--pb-brand) 20%, transparent)}}.system-hero-note-emphasis{background-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.system-hero-note-emphasis{background-color:color-mix(in srgb, var(--pb-brand) 8%, var(--pb-bg-card))}}.system-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.system-hero-note-title{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.system-hero-note-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.system-shell-body{gap:calc(var(--spacing) * 6);display:grid}@media (min-width:80rem){.system-shell-body{grid-template-columns:280px minmax(0,1fr)}}@media (min-width:96rem){.system-shell-body{grid-template-columns:300px minmax(0,1fr)}}.system-rail-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;overflow:hidden}.system-rail-header{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 5)}.system-rail-eyebrow{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.system-rail-title{margin-top:var(--spacing);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.system-rail-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}:where(.system-rail-links>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}.system-rail-links{padding:calc(var(--spacing) * 3)}.system-rail-footer{border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.system-rail-footer{background-color:color-mix(in srgb, var(--pb-bg-card) 55%, transparent)}}.system-tab-card{align-items:flex-start;gap:calc(var(--spacing) * 3);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;border-width:1px;transition-duration:.15s;display:flex}.system-tab-card-inactive{border-color:var(--pb-border);background-color:var(--pb-bg-surface)}.system-tab-card-inactive:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.system-tab-card-inactive:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 58%, transparent)}}.system-tab-card-inactive:hover{box-shadow:var(--pb-shadow-1)}.system-tab-card-active{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.system-tab-card-active{border-color:color-mix(in srgb, var(--pb-interactive) 30%, transparent)}}.system-tab-card-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.system-tab-card-active{background-color:color-mix(in srgb, var(--pb-interactive) 10%, var(--pb-bg-card))}}.system-tab-card-active{box-shadow:var(--pb-shadow-1)}.system-tab-icon{margin-top:calc(var(--spacing) * .5);height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:var(--radius-2xl);flex-shrink:0;justify-content:center;align-items:center;display:flex}.system-tab-copy{flex:1;min-width:0}.system-tab-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.system-tab-description{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5);color:var(--pb-text-secondary);display:block}.system-tab-arrow{margin-top:var(--spacing);color:var(--pb-text-dim);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;transition-duration:.15s}.system-tab-card:hover .system-tab-arrow,.system-tab-card-active .system-tab-arrow{color:var(--pb-interactive);transform:translate(.125rem)}.system-focus-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;position:relative;overflow:hidden}.system-focus-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-interactive), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.system-focus-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-interactive) 34%, transparent), transparent)}}.system-focus-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.system-focus-grid{grid-template-columns:minmax(0,1.35fr) minmax(320px,.95fr);align-items:flex-start}}.system-focus-meta{align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim);flex-wrap:wrap;display:flex}.system-focus-chip{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.system-focus-actions{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.system-focus-actions{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.system-focus-actions{grid-template-columns:repeat(1,minmax(0,1fr))}}.review-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.review-hero-card{padding-inline:calc(var(--spacing) * 7)}}.review-hero-card{box-shadow:var(--pb-shadow-2)}.review-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-info), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.review-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-info) 42%, transparent), transparent)}}.review-hero-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.review-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.95fr);align-items:flex-start}}.review-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.review-hero-title{font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.review-hero-copy{max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.review-hero-notes{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.review-hero-notes{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.review-hero-notes{grid-template-columns:repeat(1,minmax(0,1fr))}}.review-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.review-hero-note-emphasis{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.review-hero-note-emphasis{border-color:color-mix(in srgb, var(--pb-info) 20%, transparent)}}.review-hero-note-emphasis{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.review-hero-note-emphasis{background-color:color-mix(in srgb, var(--pb-info) 10%, var(--pb-bg-card))}}.review-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.review-hero-note-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.review-workspace-tabs{align-items:center;gap:calc(var(--spacing) * 2);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);border-radius:1.25rem;flex-wrap:wrap;display:flex}@media (min-width:40rem){.review-workspace-tabs{padding-inline:calc(var(--spacing) * 5)}}.review-workspace-tabs{box-shadow:var(--pb-shadow-1)}.review-tab-chip-active{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary);border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.review-tab-chip-active{border-color:color-mix(in srgb, var(--pb-info) 34%, transparent)}}.review-tab-chip-active{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.review-tab-chip-active{background-color:color-mix(in srgb, var(--pb-info) 12%, var(--pb-bg-card))}}.review-tab-chip-active{box-shadow:var(--pb-shadow-1)}.review-tab-chip-inactive{color:var(--pb-text-dim)}.review-focus-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;position:relative;overflow:hidden}.review-focus-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.review-focus-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 34%, transparent), transparent)}}.review-focus-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.review-focus-grid{grid-template-columns:minmax(0,1.35fr) minmax(320px,.95fr);align-items:flex-start}}.review-focus-chip{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.review-focus-actions{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.review-focus-actions{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.review-focus-actions{grid-template-columns:repeat(1,minmax(0,1fr))}}.review-note-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:1.25rem}.workflow-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.workflow-hero-card{padding-inline:calc(var(--spacing) * 7)}}.workflow-hero-card{box-shadow:var(--pb-shadow-2)}.workflow-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-interactive), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.workflow-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-interactive) 40%, transparent), transparent)}}.workflow-hero-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.workflow-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.95fr);align-items:flex-start}}.workflow-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.workflow-hero-title{font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.workflow-hero-copy{max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.workflow-hero-notes{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.workflow-hero-notes{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.workflow-hero-notes{grid-template-columns:repeat(1,minmax(0,1fr))}}.workflow-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.workflow-hero-note-emphasis{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-hero-note-emphasis{border-color:color-mix(in srgb, var(--pb-interactive) 20%, transparent)}}.workflow-hero-note-emphasis{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-hero-note-emphasis{background-color:color-mix(in srgb, var(--pb-interactive) 10%, var(--pb-bg-card))}}.workflow-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.workflow-hero-note-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.series-page-shell{gap:calc(var(--spacing) * 6);flex-direction:column;min-height:100%;display:flex}.series-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.series-hero-card{padding-inline:calc(var(--spacing) * 7)}}.series-hero-card{box-shadow:var(--pb-shadow-2)}.series-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), var(--pb-interactive), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.series-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 34%, transparent), color-mix(in srgb, var(--pb-interactive) 34%, transparent), transparent)}}.series-hero-grid{gap:calc(var(--spacing) * 5);display:grid}@media (min-width:96rem){.series-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.8fr);align-items:flex-start}}.series-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.series-hero-title{font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.series-hero-copy{max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 7);line-height:calc(var(--spacing) * 7);color:var(--pb-text-secondary)}.series-hero-notes{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.series-hero-notes{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:96rem){.series-hero-notes{grid-template-columns:repeat(1,minmax(0,1fr))}}.series-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.series-hero-note-emphasis{border-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.series-hero-note-emphasis{border-color:color-mix(in srgb, var(--pb-brand) 22%, transparent)}}.series-hero-note-emphasis{background-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.series-hero-note-emphasis{background-color:color-mix(in srgb, var(--pb-brand) 8%, var(--pb-bg-card))}}.series-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.series-hero-note-title{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.series-hero-note-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.page-context-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:1.25rem}.page-context-back-link{align-items:center;gap:calc(var(--spacing) * 1.5);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.page-context-back-link:hover{color:var(--pb-text-primary)}.page-context-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.page-context-grid{grid-template-columns:minmax(0,1.35fr) minmax(280px,.9fr);align-items:flex-start}}.page-context-breadcrumbs{align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);flex-wrap:wrap;display:flex}.page-context-link{align-items:center;gap:calc(var(--spacing) * 1.5);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.page-context-link:hover{color:var(--pb-text-primary)}.page-context-separator{height:calc(var(--spacing) * 3.5);width:calc(var(--spacing) * 3.5);color:var(--pb-text-dim)}.page-context-summary{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.page-context-chip-row{gap:calc(var(--spacing) * 2);flex-wrap:wrap;display:flex}.page-context-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);padding:calc(var(--spacing) * 4);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.page-context-note{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 40%, transparent)}}.page-context-note-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.page-context-note-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.detail-hero-shell{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.detail-hero-shell{padding-inline:calc(var(--spacing) * 7)}}.detail-hero-shell{box-shadow:var(--pb-shadow-2)}.detail-hero-shell:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.detail-hero-shell:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 40%, transparent), transparent)}}.detail-hero-grid{gap:calc(var(--spacing) * 6);display:grid}@media (min-width:80rem){.detail-hero-grid{grid-template-columns:220px minmax(0,1fr) 320px}}.detail-cover-panel{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card-hover);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:1.5rem}.detail-summary-panel{min-width:0}:where(.detail-summary-panel>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 5) * calc(1 - var(--tw-space-y-reverse)))}:where(.detail-header-stack>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.detail-status-row{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;display:flex}.detail-body-copy{max-width:56rem}.detail-stat-grid{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.detail-stat-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.detail-stat-grid{grid-template-columns:repeat(4,minmax(0,1fr))}}.detail-info-grid{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.detail-info-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.detail-info-grid{grid-template-columns:repeat(3,minmax(0,1fr))}}:where(.detail-aside>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.detail-action-panel{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:1.25rem}.series-domain-page{width:100%;max-width:1400px;padding-bottom:var(--pb-page-footer-clearance);margin-inline:auto}.series-domain-breadcrumb-row{flex-wrap:wrap;align-items:center;gap:.75rem;display:flex}.series-domain-back-link{color:var(--pb-interactive);align-items:center;gap:.25rem;font-size:.82rem;transition:color .14s;display:inline-flex}.series-domain-back-link:hover{color:var(--pb-interactive-hover)}.series-domain-breadcrumbs{color:var(--pb-text-tertiary);flex-wrap:wrap;align-items:center;gap:.35rem;font-size:.82rem;display:flex}.series-domain-breadcrumbs a{color:inherit}.series-domain-breadcrumbs a:hover{color:var(--pb-interactive)}.series-domain-breadcrumb-sep{font-size:.7rem}.series-domain-breadcrumbs .current{color:var(--pb-text-primary);font-weight:600}.detail-hero-shell{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);box-shadow:var(--pb-shadow-1);border-radius:14px;overflow:hidden}.detail-hero-shell:before{display:none}.series-domain-hero-inner{grid-template-columns:130px minmax(0,1fr) minmax(13rem,16rem);align-items:start;gap:1.25rem;display:grid}.issue-domain-hero-inner{grid-template-columns:120px minmax(0,1fr) minmax(13rem,16rem)}.series-domain-cover-column{flex-shrink:0;width:130px}.issue-domain-cover-column{width:120px}.series-domain-cover-frame{aspect-ratio:2/3;border:1px solid var(--pb-border-default);background:linear-gradient(155deg, var(--pb-surface-card) 0%, var(--pb-surface-app) 100%);color:var(--pb-text-tertiary);border-radius:12px;justify-content:center;align-items:center;transition:box-shadow .2s;display:flex;overflow:hidden}.series-domain-cover-frame:hover{box-shadow:var(--pb-shadow-2)}.issue-domain-cover-frame{width:120px}.series-domain-cover-image{object-fit:cover;object-position:top;cursor:pointer;width:100%;height:100%}.series-domain-cover-placeholder{justify-content:center;align-items:center;width:100%;height:100%;display:flex}.series-domain-hero-title{text-transform:uppercase;letter-spacing:.04em;color:var(--pb-text-primary);overflow-wrap:anywhere;font-family:Syne,sans-serif;font-size:1.6rem;font-weight:800;line-height:1.05}.series-domain-hero-title span{color:var(--pb-brand)}.series-domain-hero-title-link{color:inherit;text-underline-offset:.18em;text-decoration:none;text-decoration-thickness:2px;transition:color .14s,text-decoration-color .14s;display:inline}.series-domain-hero-title-link:hover,.series-domain-hero-title-link:focus-visible{color:var(--pb-interactive);text-decoration-line:underline;-webkit-text-decoration-color:var(--pb-interactive);-webkit-text-decoration-color:var(--pb-interactive);text-decoration-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-domain-hero-title-link:hover,.series-domain-hero-title-link:focus-visible{-webkit-text-decoration-color:color-mix(in srgb, var(--pb-interactive) 58%, transparent);-webkit-text-decoration-color:color-mix(in srgb, var(--pb-interactive) 58%, transparent);text-decoration-color:color-mix(in srgb, var(--pb-interactive) 58%, transparent)}}.series-domain-hero-title-link:hover,.series-domain-hero-title-link:focus-visible{outline:none}.series-domain-hero-title-sm{font-size:1.4rem}.series-domain-hero-description{max-width:35rem;color:var(--pb-text-secondary);margin-top:.5rem;font-size:.85rem;line-height:1.7}.series-domain-hero-subtitle{color:var(--pb-text-secondary);margin-top:.25rem;font-size:.88rem}.series-domain-status-row{flex-wrap:wrap;align-items:center;gap:.375rem;margin-top:.75rem;display:flex}.series-domain-led{border-radius:999px;width:8px;height:8px;display:inline-block}.series-domain-led-on{background:var(--pb-status-success);box-shadow:0 0 5px var(--pb-status-success), 0 0 10px var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.series-domain-led-on{box-shadow:0 0 5px color-mix(in srgb, var(--pb-status-success) 35%, transparent), 0 0 10px color-mix(in srgb, var(--pb-status-success) 12%, transparent)}}.series-domain-led-off{background:var(--pb-text-tertiary);opacity:.35}.app-progress{align-items:center;gap:.625rem;display:flex}.app-progress-track{background:var(--pb-text-secondary);border-radius:2px;flex:1;height:8px}@supports (color:color-mix(in lab, red, red)){.app-progress-track{background:color-mix(in srgb, var(--pb-text-secondary) 6%, transparent)}}.app-progress-track{border:1px solid var(--pb-border-subtle);position:relative;overflow:hidden}.app-progress-fill{background:linear-gradient(90deg, var(--pb-status-success), var(--pb-status-success));height:100%}@supports (color:color-mix(in lab, red, red)){.app-progress-fill{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-success) 45%, transparent), var(--pb-status-success))}}.app-progress-fill{position:relative}.app-progress-fill:after{content:"";background:repeating-linear-gradient(90deg,#0000 0 2px,#ffffff1f 2px 4px);position:absolute;inset:0}.app-progress-fill--interactive{background:linear-gradient(90deg, var(--pb-interactive), var(--pb-interactive))}@supports (color:color-mix(in lab, red, red)){.app-progress-fill--interactive{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-interactive) 45%, transparent), var(--pb-interactive))}}.app-progress-fill--error{background:linear-gradient(90deg, var(--pb-error), var(--pb-error))}@supports (color:color-mix(in lab, red, red)){.app-progress-fill--error{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-error) 45%, transparent), var(--pb-error))}}.app-progress-value{text-align:right;min-width:40px;color:var(--pb-status-success);font-family:JetBrains Mono,monospace;font-size:.82rem;font-weight:700}.app-progress-value--interactive{color:var(--pb-interactive)}.app-progress-value--error{color:var(--pb-error)}.app-progress-value-stack{flex-direction:column;align-items:flex-end;line-height:1.05;display:flex}.app-progress-value-secondary{letter-spacing:.01em;color:var(--pb-text-dim);white-space:nowrap;font-family:DM Sans,sans-serif;font-size:.62rem;font-weight:600}.series-domain-info-grid{grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:.625rem;margin-top:1rem;display:grid}.series-domain-info-box{background:var(--pb-surface-app);border:1px solid var(--pb-border-subtle);border-radius:10px;padding:.75rem .875rem}.series-domain-info-box-wide{grid-column:span 2}.series-domain-info-label{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.series-domain-info-value{color:var(--pb-text-secondary);margin-top:.25rem;font-size:.85rem}.series-domain-path{word-break:break-all;font-size:.75rem}.series-domain-actions-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);border-radius:14px;padding:1.125rem 1.5rem}.series-domain-actions-inner{flex-wrap:wrap;align-items:center;gap:1rem;display:flex}.series-domain-actions-panel{border-left:1px solid var(--pb-border-subtle);flex-direction:column;gap:.75rem;min-width:0;padding-left:1.25rem;display:flex}.series-domain-actions-title{text-transform:uppercase;letter-spacing:.08em;color:var(--pb-text-primary);white-space:nowrap;font-family:Syne,sans-serif;font-size:.88rem;font-weight:800}.series-domain-actions-title span{color:var(--pb-brand)}.series-domain-actions-divider{background:var(--pb-border-default);align-self:stretch;width:1px}.series-domain-actions-buttons{flex-wrap:wrap;flex:1;align-items:center;gap:.5rem;display:flex}.series-domain-actions-panel .series-domain-actions-buttons{flex-direction:column;flex:initial;align-items:center;gap:.55rem}.series-domain-actions-panel .series-domain-actions-buttons>a,.series-domain-actions-panel .series-domain-actions-buttons>button,.series-domain-actions-panel .series-domain-actions-buttons .series-domain-inline-toggle{width:100%}.series-domain-actions-panel .series-domain-actions-buttons>a,.series-domain-actions-panel .series-domain-actions-buttons>button{gap:var(--pb-control-gap);justify-content:center}.series-domain-actions-panel .series-domain-actions-buttons .series-domain-inline-toggle{justify-content:space-between}.series-domain-inline-toggle{border:1px solid var(--pb-border-default);background:var(--pb-surface-card);border-radius:10px;align-items:center;gap:.75rem;height:40px;padding:0 .75rem;display:inline-flex}.series-domain-inline-toggle-label{color:var(--pb-text-primary);font-size:.82rem;font-weight:600}.series-domain-section-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);border-radius:14px;padding:1rem 1.25rem}.series-domain-section-title{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:.82rem;font-weight:700}:where(.series-domain-alt-panel>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}.series-domain-alt-title{text-transform:uppercase;letter-spacing:.12em;color:var(--pb-text-dim);font-family:Syne,sans-serif;font-size:.72rem;font-weight:700}.series-domain-alt-strip{flex-direction:column;align-items:flex-start;gap:.5rem;display:flex}.series-domain-alt-list{flex-wrap:wrap;align-items:flex-start;gap:.35rem;min-height:0;display:flex}.series-domain-alt-pill{gap:.35rem}.series-domain-alt-pill-remove{color:var(--pb-text-tertiary);margin-left:.1rem;transition:color .14s}.series-domain-alt-pill-remove:hover{color:var(--pb-status-danger)}.series-domain-alt-form{flex-wrap:wrap;align-items:center;gap:.5rem;width:100%;display:flex}.series-domain-alt-input{border:1px solid var(--pb-border-default);background:var(--pb-surface-input);width:180px;color:var(--pb-text-primary);border-radius:8px;padding:.45rem .75rem;font-size:.78rem}.series-domain-alt-input:focus{border-color:var(--pb-interactive);outline:none}.series-domain-alt-add{padding:.3rem .75rem;font-size:.78rem}.series-domain-issues-wrap{overflow:visible}.series-domain-issues-card{padding-top:.875rem}.series-domain-issues-toolbar{flex-wrap:wrap;justify-content:space-between;align-items:flex-start;gap:.75rem;display:flex}.series-domain-issues-toolbar-left{flex-wrap:wrap;align-items:center;gap:.5rem;display:flex}.series-domain-issues-title{color:var(--pb-text-primary);font-size:.88rem;font-weight:700}.series-domain-issues-summary{flex-wrap:wrap;align-items:center;gap:.35rem;display:flex}.series-domain-issues-summary .pill{font-size:.62rem}.series-domain-issues-select.dropdown-select{--pb-control-min-height:34px;--pb-control-font-size:.78rem;width:auto}.series-domain-issues-select .dropdown-select-trigger{background:var(--pb-surface-input);padding-block:.4rem;padding-inline:.75rem .65rem}.series-domain-issues-progress{align-items:center;gap:.625rem;margin-top:.9rem;display:flex}.series-domain-issues-progress-track{background:var(--pb-text-secondary);border-radius:2px;flex:1;height:8px}@supports (color:color-mix(in lab, red, red)){.series-domain-issues-progress-track{background:color-mix(in srgb, var(--pb-text-secondary) 6%, transparent)}}.series-domain-issues-progress-track{border:1px solid var(--pb-border-subtle);overflow:hidden}.series-domain-issues-progress-fill{background:linear-gradient(90deg, var(--pb-status-success), var(--pb-status-success));height:100%}@supports (color:color-mix(in lab, red, red)){.series-domain-issues-progress-fill{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-success) 45%, transparent), var(--pb-status-success))}}.series-domain-issues-progress-label{text-align:right;min-width:36px;color:var(--pb-status-success);font-family:JetBrains Mono,monospace;font-size:.78rem;font-weight:700}.series-domain-issues-table-wrap{margin-top:.875rem;overflow-x:auto}.series-domain-issues-table{border-collapse:collapse;width:100%}.series-domain-issues-table th,.series-domain-issues-table td{border-bottom:1px solid var(--pb-border-subtle);padding:.75rem .875rem}.series-domain-issues-table thead th{text-align:left;text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);background:var(--pb-surface-shell);border-bottom:2px solid var(--pb-border-default);font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.series-domain-issues-table tbody tr:hover td{background:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.series-domain-issues-table tbody tr:hover td{background:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.series-domain-table-head.r{text-align:right}.series-domain-issue-num{white-space:nowrap;color:var(--pb-text-secondary);font-family:JetBrains Mono,monospace;font-size:.84rem}.series-domain-issue-title-cell{min-width:240px;color:var(--pb-text-secondary);font-size:.84rem}.series-domain-issue-title-link{color:var(--pb-text-primary);transition:color .14s;display:block}.series-domain-issue-title-link:hover{color:var(--pb-interactive)}.series-domain-issue-date{white-space:nowrap;color:var(--pb-text-tertiary);font-size:.8rem}.series-domain-issue-reading{min-width:9.5rem;color:var(--pb-text-secondary);font-family:JetBrains Mono,monospace;font-size:.7rem}.series-domain-issue-reading-mobile{color:var(--pb-interactive);margin-top:.3rem;font-family:JetBrains Mono,monospace;font-size:.65rem}.series-domain-reading-progress{gap:.28rem;max-width:8.5rem;display:grid}.series-domain-reading-progress-track{background:var(--pb-border-default);border-radius:999px;height:3px;display:block;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.series-domain-reading-progress-track{background:color-mix(in srgb, var(--pb-border-default) 68%, transparent)}}.series-domain-reading-progress-fill{border-radius:inherit;background:var(--pb-interactive);height:100%;display:block}.series-domain-reading-queue{color:var(--pb-text-tertiary);margin-top:.3rem;font-family:DM Sans,sans-serif;font-size:.62rem;display:block}.series-domain-issue-status{white-space:nowrap}.series-domain-issue-actions-cell{text-align:right}.series-domain-issue-actions{align-items:center;gap:.25rem;display:inline-flex}.series-domain-issue-action-btn{width:28px;height:28px;color:var(--pb-text-tertiary);background:0 0;border:1px solid #0000;border-radius:8px;justify-content:center;align-items:center;transition:all .14s;display:inline-flex}.series-domain-issue-action-btn:hover{color:var(--pb-interactive);background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-domain-issue-action-btn:hover{background:color-mix(in srgb, var(--pb-interactive) 6%, transparent)}}.series-domain-reading-menu{z-index:60;border:1px solid var(--pb-border-default);background:var(--pb-surface-raised);min-width:10.5rem;box-shadow:var(--pb-shadow-2);border-radius:10px;display:grid;position:absolute;top:calc(100% + .3rem);right:0;overflow:hidden}.series-domain-reading-menu button{text-align:left;min-height:2.5rem;color:var(--pb-text-secondary);padding:.55rem .75rem;font-size:.72rem;font-weight:600}.series-domain-reading-menu button:hover,.series-domain-reading-menu button:focus-visible{color:var(--pb-text-primary);background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-domain-reading-menu button:hover,.series-domain-reading-menu button:focus-visible{background:color-mix(in srgb, var(--pb-interactive) 9%, transparent)}}.series-domain-table-footer{margin-top:.75rem}.series-domain-copy-block{color:var(--pb-text-secondary);margin-top:.75rem}.issue-domain-stat-strip{grid-template-columns:repeat(4,minmax(0,1fr));gap:.625rem;margin-top:1rem;display:grid}.issue-domain-stat-box{background:var(--pb-surface-shell);border:1px solid var(--pb-border-subtle);border-radius:12px;padding:.8rem .9rem}.issue-domain-stat-label{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.issue-domain-stat-value{color:var(--pb-text-primary);margin-top:.3rem;font-family:JetBrains Mono,monospace;font-size:.85rem}.issue-domain-stat-value a{color:var(--pb-interactive)}.issue-domain-creators-grid{grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:.75rem;margin-top:.75rem;display:grid}.issue-domain-creator-card{background:var(--pb-surface-app);border:1px solid var(--pb-border-subtle);border-radius:10px;padding:.8rem .9rem}.issue-domain-creator-name{color:var(--pb-text-primary);font-size:.85rem;font-weight:600}.issue-domain-creator-role{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);margin-top:.2rem;font-size:.68rem}.issue-domain-file-grid{grid-template-columns:repeat(3,minmax(0,1fr));gap:.75rem;margin-top:.75rem;display:grid}.issue-domain-file-name{font-size:.78rem}.issue-domain-file-path-box{background:var(--pb-surface-app);border:1px solid var(--pb-border-subtle);border-radius:12px;justify-content:space-between;align-items:flex-start;gap:.75rem;margin-top:.75rem;padding:.875rem 1rem;display:flex}.issue-domain-file-path{word-break:break-all;color:var(--pb-text-secondary);margin-top:.3rem;font-family:JetBrains Mono,monospace;font-size:.75rem;line-height:1.7}.issue-domain-copy-btn{border:1px solid var(--pb-border-default);background:var(--pb-surface-card);width:30px;height:30px;color:var(--pb-text-tertiary);border-radius:8px;justify-content:center;align-items:center;transition:all .14s;display:inline-flex}.issue-domain-copy-btn:hover{border-color:var(--pb-interactive);color:var(--pb-interactive)}.series-domain-telemetry-strip{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);color:var(--pb-text-tertiary);box-shadow:var(--pb-shadow-1);border-radius:12px;flex-wrap:wrap;align-items:center;gap:.35rem 1rem;padding:.65rem 1rem;font-family:JetBrains Mono,monospace;font-size:.7rem;display:flex}.series-domain-telemetry-strip strong{color:var(--pb-text-primary);font-weight:700}.series-page-shell{gap:calc(var(--spacing) * 4);flex-direction:column;display:flex}.series-registry-header{flex-wrap:wrap;justify-content:space-between;align-items:flex-start;gap:1.25rem;display:flex}.series-registry-header-left{flex-wrap:wrap;align-items:flex-start;gap:1.5rem;display:flex}.series-registry-title{text-transform:uppercase;letter-spacing:.04em;color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:1.6rem;font-weight:800;line-height:1}.series-registry-title span{color:var(--pb-brand)}.series-registry-subtitle{color:var(--pb-text-tertiary);letter-spacing:.01em;margin-top:.25rem;font-size:.78rem}.series-registry-gauges{align-items:flex-end;gap:1rem;display:flex}.series-registry-gauge{text-align:center}.series-registry-gauge-ring{width:56px;height:56px;margin:0 auto;position:relative}.series-registry-gauge-ring svg{transform:rotate(-90deg)}.series-registry-gauge-bg{fill:none;stroke:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.series-registry-gauge-bg{stroke:color-mix(in srgb, var(--pb-text-secondary) 10%, transparent)}}.series-registry-gauge-bg{stroke-width:4.5px}.series-registry-gauge-fill{fill:none;stroke-width:4.5px;stroke-linecap:round;transition:stroke-dashoffset .6s}.series-registry-gauge-fill-success{stroke:var(--pb-status-success)}.series-registry-gauge-fill-info{stroke:var(--pb-status-info)}.series-registry-gauge-fill-warning{stroke:var(--pb-status-warning)}.series-registry-gauge-fill-danger{stroke:var(--pb-status-danger)}.series-registry-gauge-fill-default{stroke:var(--pb-text-secondary)}.series-registry-gauge-value{justify-content:center;align-items:center;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:700;display:flex;position:absolute;inset:0}.series-registry-gauge-label{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);margin-top:.25rem;font-size:.6rem;font-weight:600}.series-registry-actions{flex-wrap:wrap;align-items:center;gap:.5rem;display:flex}.series-toolbar-shell{z-index:20;position:sticky;top:0}.series-toolbar-frame{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);overflow:visible}.series-toolbar-body{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}.series-toolbar-browse{flex-wrap:wrap;align-items:flex-end;gap:.5rem;display:flex}.series-toolbar-primary{flex-wrap:wrap;flex:1;align-items:flex-end;gap:.5rem;min-width:0;display:flex}.series-toolbar-actions{flex-wrap:wrap;align-items:flex-end;gap:.5rem;display:flex}.series-toolbar-field{flex-shrink:0;display:block}.series-toolbar-label{text-transform:uppercase;letter-spacing:.08em;color:var(--pb-text-tertiary);margin-bottom:.25rem;font-size:.68rem;font-weight:700;display:block}.series-registry-search{width:220px;max-width:100%}.add-series-search-field{width:275px}.series-registry-search.search-field{border-color:var(--pb-border-default);background:var(--pb-surface-input);min-height:38px;box-shadow:none;border-radius:8px}.series-registry-search .search-field-input{font-size:.82rem}.series-registry-dropdown.dropdown-select{--pb-control-min-height:38px;--pb-control-font-size:.78rem;--pb-control-radius:8px;width:fit-content;max-width:100%}.series-registry-dropdown .dropdown-select-trigger{border-color:var(--pb-border-default);background:var(--pb-surface-input);box-shadow:none;padding-inline:.75rem}.series-registry-dropdown .dropdown-select-trigger-label{color:var(--pb-text-primary);font-weight:500}.series-registry-dropdown .dropdown-select-panel{border-radius:10px}.series-toolbar-view-block{flex-direction:column;display:inline-flex}.series-view-toggle{border:1px solid var(--pb-border-default);background:0 0;border-radius:8px;display:inline-flex;overflow:visible}.series-view-toggle-item:first-child{border-top-left-radius:7px;border-bottom-left-radius:7px}.series-view-toggle-item:last-child{border-top-right-radius:7px;border-bottom-right-radius:7px}.series-view-toggle-item{width:38px;height:34px;color:var(--pb-text-tertiary);background:0 0;border:none;justify-content:center;align-items:center;transition:all .14s;display:inline-flex}.series-view-toggle-item+.series-view-toggle-item{border-left:1px solid var(--pb-border-default)}.series-view-toggle-item:hover{background:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.series-view-toggle-item:hover{background:color-mix(in srgb, var(--pb-text-secondary) 4%, transparent)}}.series-view-toggle-item:hover{color:var(--pb-text-primary)}html[data-series-view=list] .series-view-toggle-item[data-series-view-option=list],html[data-series-view=grid] .series-view-toggle-item[data-series-view-option=grid],html[data-whats-new-view=list] .whats-new-view-toggle-item[data-whats-new-view-option=list],html[data-whats-new-view=compact] .whats-new-view-toggle-item[data-whats-new-view-option=compact]{background:var(--pb-interactive);color:var(--pb-text-inverse)}.whats-new-pane-toggle{background:var(--pb-surface-input)}.whats-new-pane-toggle-item{min-height:34px;color:var(--pb-text-tertiary);letter-spacing:.06em;text-transform:uppercase;background:0 0;border:none;justify-content:center;align-items:center;padding:0 .875rem;font-size:.78rem;font-weight:700;transition:all .14s;display:inline-flex}.whats-new-pane-toggle-item:first-child{border-top-left-radius:7px;border-bottom-left-radius:7px}.whats-new-pane-toggle-item:last-child{border-top-right-radius:7px;border-bottom-right-radius:7px}.whats-new-pane-toggle-item+.whats-new-pane-toggle-item{border-left:1px solid var(--pb-border-default)}.whats-new-pane-toggle-item:hover{background:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.whats-new-pane-toggle-item:hover{background:color-mix(in srgb, var(--pb-text-secondary) 4%, transparent)}}.whats-new-pane-toggle-item:hover{color:var(--pb-text-primary)}.whats-new-pane-toggle-item.is-active,.whats-new-pane-toggle-item[aria-current=page]{background:var(--pb-interactive);color:var(--pb-text-inverse)}.whats-new-title-block{min-height:4.55rem}.whats-new-cache-badge{min-height:34px;color:var(--pb-text-inverse);align-items:center;display:inline-flex}.whats-new-cache-badge-success{background:#1f6f4a}.whats-new-cache-badge-error{background:#9f2f24}.whats-new-week-nav{border:1px solid var(--pb-border-default);background:var(--pb-surface-input);border-radius:10px;justify-content:center;justify-self:center;align-items:center;gap:.375rem;width:fit-content;max-width:min(100%,34rem);margin-inline:auto;padding:.375rem;display:inline-flex}.whats-new-week-nav-button{border:1px solid var(--pb-border-default);background:var(--pb-surface-card);min-width:32px;min-height:32px;color:var(--pb-text-secondary);border-radius:8px;justify-content:center;align-items:center;font-size:.9rem;font-weight:800;transition:all .14s;display:inline-flex}.whats-new-week-nav-button:hover{border-color:var(--pb-border-hover);background:var(--pb-interactive-selected);color:var(--pb-text-primary)}.whats-new-week-nav-button.is-disabled{opacity:.45;cursor:not-allowed}.whats-new-week-dropdown.dropdown-select{--pb-control-min-height:32px;--pb-control-font-size:.78rem;--pb-control-radius:8px;flex:0 auto;width:fit-content;max-width:min(52vw,14rem)}.whats-new-week-dropdown .dropdown-select-trigger{border-color:var(--pb-border-default);background:var(--pb-surface-card);min-width:9.5rem;max-width:14rem;box-shadow:none;padding-inline:.625rem}.whats-new-week-dropdown .dropdown-select-panel{border-radius:10px;min-width:11.5rem;max-width:min(90vw,16rem)}.whats-new-week-dropdown .dropdown-select-trigger-label,.whats-new-week-dropdown .dropdown-select-option-label{font-weight:700}.whats-new-week-position{color:var(--pb-text-tertiary);letter-spacing:.08em;text-transform:uppercase;flex:none;padding-inline:.25rem .375rem;font-size:.68rem;font-weight:700}@media (max-width:640px){.whats-new-week-nav{max-width:100%}.whats-new-week-dropdown.dropdown-select{max-width:min(58vw,13rem)}.whats-new-week-position{text-align:center;width:100%}}.whats-new-results-stack,.whats-new-panel{gap:1rem;display:grid}.whats-new-stale-banner{border:1px solid var(--pb-status-warning);justify-content:space-between;align-items:center;gap:1rem;display:flex}@supports (color:color-mix(in lab, red, red)){.whats-new-stale-banner{border:1px solid color-mix(in srgb, var(--pb-status-warning) 40%, transparent)}}.whats-new-stale-banner{background:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.whats-new-stale-banner{background:color-mix(in srgb, var(--pb-status-warning) 10%, transparent)}}.whats-new-stale-banner{color:var(--pb-text-primary);border-radius:12px;margin-bottom:1rem;padding:.75rem 1rem;font-size:.82rem}.whats-new-stale-copy{min-width:0}.whats-new-stale-actions{flex-direction:column;flex:none;align-items:flex-end;gap:.375rem;display:flex}.whats-new-stale-message{max-width:24rem;color:var(--pb-text-secondary);text-align:right;font-size:.72rem;line-height:1.35}@media (max-width:640px){.whats-new-stale-banner{flex-direction:column;align-items:stretch}.whats-new-stale-actions{align-items:flex-start}.whats-new-stale-message{text-align:left}}.whats-new-release-summary{margin-bottom:0}.whats-new-release-table-slot{min-width:0}html[data-whats-new-view=compact] .whats-new-release-cover,html[data-whats-new-view=compact] .whats-new-pulls-col{display:none}html[data-whats-new-view=compact] .whats-new-release-title-cell{gap:0}.select-btn{border:1px solid var(--pb-border-default);min-height:38px;color:var(--pb-text-secondary);background:0 0;border-radius:8px;justify-content:center;align-items:center;gap:.375rem;padding:.375rem .875rem;font-family:DM Sans,sans-serif;font-size:.78rem;font-weight:600;transition:all .14s;display:inline-flex}.select-btn:hover{border-color:var(--pb-interactive);color:var(--pb-interactive)}.select-btn:disabled{cursor:not-allowed;opacity:.45}.select-btn-success{color:var(--pb-status-success);border-color:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.select-btn-success{border-color:color-mix(in srgb, var(--pb-status-success) 30%, transparent)}}.select-btn-success{background:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.select-btn-success{background:color-mix(in srgb, var(--pb-status-success) 8%, transparent)}}.select-btn-warning{color:var(--pb-status-warning);border-color:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.select-btn-warning{border-color:color-mix(in srgb, var(--pb-status-warning) 30%, transparent)}}.select-btn-warning{background:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.select-btn-warning{background:color-mix(in srgb, var(--pb-status-warning) 8%, transparent)}}.select-btn-danger{color:var(--pb-status-danger);border-color:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){.select-btn-danger{border-color:color-mix(in srgb, var(--pb-status-danger) 30%, transparent)}}.select-btn-danger{background:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){.select-btn-danger{background:color-mix(in srgb, var(--pb-status-danger) 8%, transparent)}}.series-selection-shell{flex-direction:column;gap:.625rem;display:flex}.series-selection-inline{align-items:center;display:inline-flex}.series-selection-count{color:var(--pb-text-secondary);font-family:JetBrains Mono,monospace;font-size:.78rem;font-weight:600}.series-selection-controls-row{flex-wrap:wrap;justify-content:space-between;gap:.75rem;display:flex}.series-selection-bulk,.series-selection-actions{flex-wrap:wrap;align-items:center;gap:.5rem;display:flex}.series-results-shell{gap:calc(var(--spacing) * 4);min-height:16rem;padding-bottom:var(--pb-page-footer-clearance);flex-direction:column;flex:none;display:flex}.series-results-body-shell{min-height:14rem}:where(:is(.series-mission-control-shell,.series-collector-wall-shell)>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.series-mission-control-table-wrap{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);border-radius:14px;overflow:auto hidden}.series-mission-control-table{border-collapse:collapse;width:100%}.series-mission-control-table th{text-align:left;text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);background:var(--pb-surface-shell);border-bottom:2px solid var(--pb-border-default);padding:.5625rem .875rem;font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.series-mission-control-table td{border-bottom:1px solid var(--pb-border-subtle);padding:.6875rem .875rem;font-size:.82rem;transition:background .1s}.series-mission-control-table tbody tr:last-child td{border-bottom:none}.series-mission-control-table tbody tr:not(.table-detail-row):hover td{background:var(--pb-surface-selected)}.series-mission-control-table th.c,.series-mission-control-table td.c{text-align:center}.series-mission-control-table th.r,.series-mission-control-table td.r{text-align:right}.series-mission-control-row-selected td{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-mission-control-row-selected td{background:color-mix(in srgb, var(--pb-interactive) 10%, transparent)}}.series-led{border-radius:50%;width:8px;height:8px;display:inline-block}.series-led-green{background:var(--pb-status-success);box-shadow:0 0 5px var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.series-led-green{box-shadow:0 0 5px color-mix(in srgb, var(--pb-status-success) 35%, transparent)}}.series-led-amber{background:var(--pb-status-warning);box-shadow:0 0 5px var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.series-led-amber{box-shadow:0 0 5px color-mix(in srgb, var(--pb-status-warning) 35%, transparent)}}.series-led-off{background:var(--pb-text-tertiary);opacity:.35}.series-mission-control-name{color:var(--pb-text-primary);font-weight:700;transition:color .14s}.series-mission-control-name:hover{color:var(--pb-interactive)}.series-mission-control-publisher{color:var(--pb-text-tertiary);margin-top:1px;font-size:.72rem}.series-mission-control-year{color:var(--pb-text-tertiary);font-size:.78rem}.series-mission-control-bar-cell{width:220px}.series-mission-control-bar{align-items:center;gap:.5rem;display:flex}.series-mission-control-bar-track{background:var(--pb-text-secondary);border-radius:2px;flex:1;height:8px}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-track{background:color-mix(in srgb, var(--pb-text-secondary) 6%, transparent)}}.series-mission-control-bar-track{border:1px solid var(--pb-border-subtle);position:relative;overflow:hidden}.series-mission-control-bar-fill{height:100%;position:relative}.series-mission-control-bar-fill:after{content:"";background:repeating-linear-gradient(90deg,#0000 0 2px,#ffffff1f 2px 4px);position:absolute;inset:0}.series-mission-control-bar-fill-green{background:linear-gradient(90deg, var(--pb-status-success), var(--pb-status-success))}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-fill-green{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-success) 45%, transparent), var(--pb-status-success))}}.series-mission-control-bar-fill-amber{background:linear-gradient(90deg, var(--pb-status-warning), var(--pb-status-warning))}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-fill-amber{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-warning) 45%, transparent), var(--pb-status-warning))}}.series-mission-control-bar-fill-red{background:linear-gradient(90deg, var(--pb-status-danger), var(--pb-status-danger))}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-fill-red{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-danger) 45%, transparent), var(--pb-status-danger))}}.series-mission-control-bar-fill-blue{background:linear-gradient(90deg, var(--pb-status-info), var(--pb-status-info))}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-fill-blue{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-info) 45%, transparent), var(--pb-status-info))}}.series-mission-control-bar-pct{text-align:right;font-variant-numeric:tabular-nums;min-width:36px;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:700}.series-mission-control-bar-pct-green{color:var(--pb-status-success)}.series-mission-control-bar-pct-amber{color:var(--pb-status-warning)}.series-mission-control-bar-pct-red{color:var(--pb-status-danger)}.series-mission-control-bar-pct-blue{color:var(--pb-status-info)}.series-mission-control-actions{justify-content:flex-end;gap:.25rem;display:flex}.series-mission-control-action-btn{border:1px solid var(--pb-border-subtle);width:30px;height:30px;color:var(--pb-text-tertiary);background:0 0;border-radius:8px;justify-content:center;align-items:center;transition:all .14s;display:inline-flex}.series-mission-control-action-btn:hover{border-color:var(--pb-interactive);color:var(--pb-interactive);background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-mission-control-action-btn:hover{background:color-mix(in srgb, var(--pb-interactive) 4%, transparent)}}.series-mission-control-action-btn.disabled{opacity:.25;pointer-events:none}.series-mission-control-action-btn svg{width:14px;height:14px}.series-mission-control-footer{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);color:var(--pb-text-tertiary);border-radius:10px;flex-wrap:wrap;align-items:center;gap:.375rem 1.5rem;padding:.625rem 1rem;font-family:JetBrains Mono,monospace;font-size:.7rem;display:flex}.series-mission-control-footer strong{color:var(--pb-text-primary);font-weight:600}.series-collector-wall-grid{grid-template-columns:repeat(auto-fill,minmax(172px,1fr));gap:24px 18px;display:grid}.series-wall-card{min-width:0;transition:transform .22s;position:relative}.series-wall-card:hover{transform:translateY(-5px)}.series-wall-card-selected .series-wall-cover-wrap{outline:2px solid var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-wall-card-selected .series-wall-cover-wrap{outline:2px solid color-mix(in srgb, var(--pb-interactive) 28%, transparent)}}.series-wall-card-selected .series-wall-cover-wrap{outline-offset:2px}.series-wall-cover-wrap{aspect-ratio:2/3;border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);border-radius:14px;transition:box-shadow .22s;position:relative;overflow:hidden}.series-wall-card:hover .series-wall-cover-wrap{box-shadow:var(--pb-shadow-2), 0 0 20px var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-wall-card:hover .series-wall-cover-wrap{box-shadow:var(--pb-shadow-2), 0 0 20px color-mix(in srgb, var(--pb-interactive) 6%, transparent)}}.series-wall-cover-placeholder{width:100%;height:100%;color:var(--pb-text-tertiary);background:linear-gradient(155deg, var(--pb-surface-card) 0%, var(--pb-surface-app) 100%);justify-content:center;align-items:center;display:flex}.series-monitor-badge{width:1.5rem;height:1.5rem;color:var(--pb-text-inverse);background:var(--pb-brand-signal);border:2px solid var(--pb-surface-card);border-radius:999px;flex:none;justify-content:center;align-items:center;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.series-monitor-badge{border:2px solid color-mix(in srgb, var(--pb-surface-card) 92%, white)}}.series-monitor-badge{box-shadow:0 0 0 2px #0000006b,0 10px 22px #00000057}.series-monitor-badge svg{flex:none;width:.9rem;height:.9rem}.series-monitor-badge.is-paused{color:var(--pb-text-secondary);background:var(--pb-surface-raised);border-color:var(--pb-border-default);box-shadow:0 0 0 1px var(--pb-border-subtle)}.series-wall-monitor-dot{z-index:3;position:absolute;top:9px;left:9px}.series-wall-catalog-state-badge{z-index:4;border:1px solid var(--pb-warning);background:var(--pb-warning);border-radius:999px;justify-content:center;align-items:center;max-width:calc(100% - 18px);display:inline-flex;position:absolute;top:46px;right:9px}@supports (color:color-mix(in lab, red, red)){.series-wall-catalog-state-badge{background:color-mix(in srgb, var(--pb-warning) 92%, black)}}.series-wall-catalog-state-badge{color:var(--pb-text-inverse);letter-spacing:.04em;text-align:center;text-transform:uppercase;padding:.2rem .5rem;font-size:.58rem;font-weight:800;line-height:1.15;box-shadow:0 8px 18px #00000057}.series-wall-catalog-state-badge.is-failed{border-color:var(--pb-error);background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.series-wall-catalog-state-badge.is-failed{background:color-mix(in srgb, var(--pb-error) 92%, black)}}.pull-list-monitor-toggle{width:28px;height:28px;box-shadow:none;cursor:pointer;border-width:1px;border-radius:6px;transition:background-color .14s,border-color .14s,color .14s,box-shadow .14s}.pull-list-monitor-toggle svg{width:12px;height:12px}.pull-list-monitor-toggle:hover{border-color:var(--pb-brand-signal)}@supports (color:color-mix(in lab, red, red)){.pull-list-monitor-toggle:hover{border-color:color-mix(in srgb, var(--pb-brand-signal) 56%, transparent)}}.pull-list-monitor-toggle:hover{background:var(--pb-brand-signal);color:var(--pb-text-inverse);box-shadow:0 0 0 2px var(--pb-brand-signal)}@supports (color:color-mix(in lab, red, red)){.pull-list-monitor-toggle:hover{box-shadow:0 0 0 2px color-mix(in srgb, var(--pb-brand-signal) 14%, transparent)}}.pull-list-monitor-toggle:focus-visible{outline:2px solid var(--pb-focus-outline);outline-offset:3px}.series-wall-selection-control{z-index:4;border:1px solid var(--pb-border-default);background:var(--pb-surface-card);border-radius:10px;position:absolute;top:10px;right:10px}@supports (color:color-mix(in lab, red, red)){.series-wall-selection-control{background:color-mix(in srgb, var(--pb-surface-card) 92%, transparent)}}.series-wall-selection-control{box-shadow:var(--pb-shadow-1);padding:.25rem}.series-wall-ring{z-index:2;width:44px;height:44px;position:absolute;bottom:-6px;right:-6px}.series-wall-ring svg{transform:rotate(-90deg)}.series-wall-ring-bg{fill:none;stroke:var(--pb-surface-card);stroke-width:3.5px}.series-wall-ring-fill{fill:none;stroke-width:3.5px;stroke-linecap:round;transition:stroke-dashoffset .6s}.series-wall-ring-fill-green{stroke:var(--pb-status-success)}.series-wall-ring-fill-amber{stroke:var(--pb-status-warning)}.series-wall-ring-fill-red{stroke:var(--pb-status-danger)}.series-wall-ring-center{background:var(--pb-surface-card);color:var(--pb-text-primary);border-radius:999px;justify-content:center;align-items:center;margin:5px;font-family:JetBrains Mono,monospace;font-size:.58rem;font-weight:700;display:flex;position:absolute;inset:0}.series-wall-overlay{--series-wall-overlay-edge:#fff;z-index:3;-webkit-backdrop-filter:blur(6px);backdrop-filter:blur(6px);box-shadow:inset 0 0 0 4.5px var(--series-wall-overlay-edge);opacity:0;pointer-events:none;background:#1e1a17e0;border-radius:18px;flex-direction:column;justify-content:flex-end;padding:.875rem;transition:opacity .2s;display:flex;position:absolute;inset:-4px}[data-theme=light] .series-wall-overlay{--series-wall-overlay-edge:#1e1a17}@media (prefers-color-scheme:light){:root:not([data-theme]) .series-wall-overlay{--series-wall-overlay-edge:#1e1a17}}.series-wall-card:hover .series-wall-overlay{opacity:1}.series-wall-overlay-title{color:var(--pb-text-primary);font-size:.82rem;font-weight:700;line-height:1.3}.series-wall-overlay-meta{color:var(--pb-text-tertiary);gap:.25rem;margin-top:.45rem;font-size:.68rem;display:grid}.series-wall-overlay-meta-line,.series-wall-overlay-stat{grid-template-columns:minmax(4.25rem,max-content) minmax(0,1fr);align-items:baseline;column-gap:.5rem;line-height:1.25;display:grid}.series-wall-overlay-meta-label{color:var(--pb-text-tertiary);letter-spacing:.08em;text-transform:uppercase;font-family:JetBrains Mono,monospace;font-size:.56rem;font-weight:700}.series-wall-overlay-meta-value{overflow-wrap:anywhere;min-width:0;color:var(--pb-text-secondary)}.series-wall-overlay-stats{color:var(--pb-text-secondary);gap:.25rem;margin-top:.35rem;font-size:.68rem;display:grid}.series-wall-overlay-stats strong{color:var(--pb-text-primary);font-weight:600}.series-wall-overlay-actions{pointer-events:auto;gap:6px;margin-top:10px;display:flex}.series-wall-overlay-btn{color:var(--pb-text-secondary);text-align:center;background:#ffffff0f;border:1px solid #cbd5e140;border-radius:8px;flex:1;justify-content:center;padding:7px;font-family:DM Sans,sans-serif;font-size:.72rem;font-weight:600;transition:all .14s;display:inline-flex}.series-wall-overlay-btn:hover{background:var(--pb-interactive);color:var(--pb-text-primary);border-color:var(--pb-interactive)}.series-wall-card-title{text-overflow:ellipsis;white-space:nowrap;color:var(--pb-text-primary);margin-top:10px;font-size:.82rem;font-weight:600;line-height:1.3;display:block;overflow:hidden}.series-empty-state{justify-content:center;align-items:center;gap:calc(var(--spacing) * 3);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 12);text-align:center;box-shadow:var(--pb-shadow-1);border-radius:1.25rem;flex-direction:column;display:flex}.series-empty-state-title{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.series-empty-state-copy{max-width:var(--container-xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}@media (max-width:1180px){.series-domain-hero-inner{grid-template-columns:130px minmax(0,1fr)}.issue-domain-hero-inner{grid-template-columns:120px minmax(0,1fr)}.series-domain-actions-panel{border-top:1px solid var(--pb-border-subtle);border-left:0;grid-column:1/-1;padding-top:1rem;padding-left:0}.series-domain-actions-panel .series-domain-actions-buttons{grid-template-columns:repeat(auto-fit,minmax(12rem,1fr));align-items:stretch;display:grid}}@media (max-width:900px){.issue-domain-stat-strip{grid-template-columns:repeat(2,minmax(0,1fr))}.issue-domain-file-grid{grid-template-columns:1fr}}@media (max-width:640px){.detail-hero-shell,.series-domain-actions-card,.series-domain-section-card{padding:1rem}.series-domain-cover-column,.issue-domain-cover-column,.issue-domain-cover-frame{width:104px}.series-domain-hero-inner,.issue-domain-hero-inner{grid-template-columns:104px minmax(0,1fr)}.series-domain-hero-title{font-size:1.2rem}.series-domain-actions-inner{align-items:flex-start}.series-domain-actions-divider{display:none}.series-domain-actions-panel .series-domain-actions-buttons,.issue-domain-stat-strip{grid-template-columns:1fr}.series-domain-info-box-wide{grid-column:auto}.series-domain-issues-toolbar{flex-direction:column;align-items:stretch}}[data-series-page][data-series-toolbar-mode=browse] [data-testid=series-select-toolbar],[data-series-page][data-series-toolbar-mode=select] [data-testid=series-browse-toolbar]{display:none}[data-series-page][data-series-toolbar-mode=select] [data-testid=series-select-toolbar]{display:flex}[data-select-toolbar-page][data-toolbar-mode=browse] [data-select-toolbar],[data-select-toolbar-page][data-toolbar-mode=select] [data-browse-toolbar]{display:none}[data-select-toolbar-page][data-toolbar-mode=select] [data-select-toolbar]{display:flex}[data-series-page][data-series-toolbar-mode=browse] [data-series-selection-cell]{display:none}[data-series-page][data-series-toolbar-mode=select] [data-series-selection-cell]{display:table-cell}[data-series-page][data-series-toolbar-mode=browse] [data-series-selection-control]{display:none}[data-series-page][data-series-toolbar-mode=select] [data-series-selection-control]{display:block}[data-series-page][data-series-toolbar-mode=select] [data-series-selection-control=grid]{display:inline-flex}#page-footer-dock:empty{display:none}body:has(#page-footer-dock:not(:empty)) #content{padding-bottom:var(--pb-page-footer-clearance)}body:has(#page-footer-dock:not(:empty)) #content:has(.admin-workspace-page,.dashboard-mission-page,.downloads-view,.series-domain-page,.series-results-shell,.utilities-page){padding-bottom:0}#content:has(.admin-workspace-page){overflow-anchor:none}.page-dock-inner{--page-dock-height-status:2rem;--page-dock-height-pagination:2.5rem;max-width:85rem;height:var(--page-dock-height-status);flex-wrap:nowrap;justify-content:space-between;align-items:center;gap:.75rem;margin:0 auto;padding:0 1rem;display:flex;overflow:hidden}.page-dock-inner:has(.page-dock-pagination){height:var(--page-dock-height-pagination)}.page-footer-clearance{height:var(--pb-page-footer-clearance);flex:0 0 var(--pb-page-footer-clearance);pointer-events:none}.page-dock-inner-status-only{justify-content:flex-end}.page-dock-pagination{flex:none}.page-dock-pagination nav{gap:.125rem}.page-dock-pagination nav>a,.page-dock-pagination nav>button,.page-dock-pagination nav>span{text-align:center;border-radius:.375rem;min-width:1.75rem;padding:.25rem .625rem;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:600;line-height:1}.page-dock-pagination nav>a,.page-dock-pagination nav>button{appearance:none;color:var(--pb-text-secondary);cursor:pointer;background:0 0;border:0;margin:0}.page-dock-pagination nav>span.bg-pb-interactive{border-radius:.375rem}.page-dock-status{min-width:0;color:var(--pb-text-tertiary);scrollbar-width:none;flex-wrap:nowrap;align-items:center;gap:1rem;font-family:JetBrains Mono,monospace;font-size:.65rem;display:flex;overflow-x:auto}.page-dock-status::-webkit-scrollbar{display:none}.page-dock-status-item{white-space:nowrap;align-items:center;gap:.25rem;display:inline-flex}.page-dock-status-value{color:var(--pb-text-primary);font-weight:600}.page-dock-status-label{text-transform:uppercase;letter-spacing:.04em}.page-dock-led{border-radius:999px;width:6px;height:6px;margin-right:2px;display:inline-block}.page-dock-led-green{background:var(--pb-status-success);box-shadow:0 0 4px var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.page-dock-led-green{box-shadow:0 0 4px color-mix(in srgb, var(--pb-status-success) 35%, transparent)}}.page-dock-led-amber{background:var(--pb-status-warning);box-shadow:0 0 4px var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.page-dock-led-amber{box-shadow:0 0 4px color-mix(in srgb, var(--pb-status-warning) 35%, transparent)}}.page-dock-led-off{background:var(--pb-text-tertiary);opacity:.35}@media (max-width:767px){.page-dock-inner{padding-inline:0}.page-dock-status{gap:.75rem}}.workflow-tabs-shell{align-items:center;gap:calc(var(--spacing) * 2);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);border-radius:1.25rem;flex-wrap:wrap;display:flex}@media (min-width:40rem){.workflow-tabs-shell{padding-inline:calc(var(--spacing) * 5)}}.workflow-tabs-shell{box-shadow:var(--pb-shadow-1)}.workflow-tab-chip-active{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary);border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-tab-chip-active{border-color:color-mix(in srgb, var(--pb-interactive) 34%, transparent)}}.workflow-tab-chip-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-tab-chip-active{background-color:color-mix(in srgb, var(--pb-interactive) 12%, var(--pb-bg-card))}}.workflow-tab-chip-active{box-shadow:var(--pb-shadow-1)}.workflow-tab-chip-inactive{color:var(--pb-text-dim)}.workflow-focus-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;position:relative;overflow:hidden}.workflow-focus-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.workflow-focus-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 34%, transparent), transparent)}}.workflow-focus-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.workflow-focus-grid{grid-template-columns:minmax(0,1.35fr) minmax(320px,.95fr);align-items:flex-start}}.workflow-focus-chip{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.workflow-focus-actions{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.workflow-focus-actions{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.workflow-focus-actions{grid-template-columns:repeat(1,minmax(0,1fr))}}:where(.workflow-shell-region>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.workflow-shell-region{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 4);border-radius:1.5rem;overflow:visible}@media (min-width:40rem){.workflow-shell-region{padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 5)}}.workflow-shell-region{box-shadow:var(--pb-shadow-1)}.utility-workspace-shell{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;overflow:hidden}.utility-workspace-shell-visible{overflow:visible}:where(.utility-workspace-body>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.utility-workspace-body{padding:calc(var(--spacing) * 6)}@media (min-width:40rem){.utility-workspace-body{padding:calc(var(--spacing) * 7)}}:where(.utility-tool-body>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.utility-tool-body{padding:calc(var(--spacing) * 5)}@media (min-width:40rem){.utility-tool-body{padding:calc(var(--spacing) * 5)}}.utility-workspace-shell .info-panel{border-color:var(--pb-border-subtle);background-color:var(--pb-bg-base);box-shadow:none}.utility-workspace-shell .info-panel-muted{background-color:var(--pb-bg-base)}.utility-empty-state{border-radius:var(--radius-xl);border-style:var(--tw-border-style);--tw-border-style:dashed;border-style:dashed;border-width:1px;border-color:var(--pb-border-subtle);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 8);text-align:center;background-color:var(--pb-bg-base)}.utility-path-pill{align-items:center;gap:calc(var(--spacing) * 2);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);display:flex}.utility-token{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.utility-step-grid{gap:calc(var(--spacing) * 3);display:grid}.utility-step-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);width:100%;padding:calc(var(--spacing) * 4);display:block}.utility-step-card-inline{align-items:center;gap:.75rem;display:flex}.utility-step-card-locked{opacity:.78}.utility-step-num{color:var(--pb-text-tertiary);min-width:1.25rem;font-family:JetBrains Mono,monospace;font-size:.62rem;font-weight:700}.utility-step-name{color:var(--pb-text-primary);flex:auto;min-width:0;font-size:.82rem;font-weight:600}.utility-step-tag{color:var(--pb-text-tertiary);white-space:nowrap;font-family:JetBrains Mono,monospace;font-size:.62rem}.utility-step-card-inline .utility-step-num,.utility-step-card-inline .utility-step-tag{color:var(--pb-text-secondary)}.utility-step-check{flex-shrink:0;margin-top:0}.utility-scan-mode-grid{gap:calc(var(--spacing) * 2);display:grid}@media (min-width:40rem){.utility-scan-mode-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}.utility-scan-mode{justify-content:center;align-items:flex-start;gap:var(--spacing);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);min-height:72px;padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);text-align:left;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-direction:column;display:flex}.utility-scan-mode:hover{border-color:var(--pb-border-hover)}.utility-scan-mode-active{border-color:var(--pb-interactive);background-color:var(--pb-surface-selected)}.utility-scan-mode-label{--tw-font-weight:var(--font-weight-semibold);font-size:.82rem;font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.utility-scan-mode-copy{--tw-leading:calc(var(--spacing) * 5);font-size:.72rem;line-height:calc(var(--spacing) * 5);color:var(--pb-text-dim)}.utility-scope-chip-row{flex-wrap:wrap;gap:.375rem;display:flex}.utility-scope-chip{border:1px solid var(--pb-border-default);color:var(--pb-text-secondary);background:0 0;border-radius:.5rem;justify-content:center;align-items:center;padding:.4375rem 1rem;font-size:.78rem;font-weight:600;transition:border-color .14s,background-color .14s,color .14s;display:inline-flex}.utility-scope-chip:hover{border-color:var(--pb-interactive);color:var(--pb-interactive)}.utility-scope-chip-active{background:var(--pb-interactive);color:var(--pb-text-inverse);border-color:var(--pb-interactive)}.utility-scope-chip-active:hover{color:var(--pb-text-inverse)}.utility-export-chip-row{flex-wrap:wrap;gap:.375rem;display:flex}.utility-export-chip{border:1px solid var(--pb-border-default);color:var(--pb-text-secondary);background:0 0;border-radius:999px;justify-content:center;align-items:center;padding:.4375rem .875rem;font-family:DM Sans,sans-serif;font-size:.76rem;font-weight:500;line-height:1.2;transition:border-color .14s,background-color .14s,color .14s;display:inline-flex}.utility-export-chip:hover{border-color:var(--pb-interactive);color:var(--pb-interactive)}.utility-export-chip-active{border-color:var(--pb-interactive);background:var(--pb-interactive);color:var(--pb-text-inverse)}.utility-export-chip-active:hover{color:var(--pb-text-inverse)}.utility-tool-section-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);margin-bottom:.5rem;font-family:Syne,sans-serif;font-size:.62rem;font-weight:700}.utility-tool-section-row{flex-wrap:wrap;justify-content:space-between;align-items:center;gap:.75rem;display:flex}.utility-tool-field-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);font-family:Syne,sans-serif;font-size:.58rem;font-weight:700;display:block}.utility-export-group-grid{gap:.75rem;display:grid}@media (min-width:1024px){.utility-export-group-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}.utility-export-group-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-app);border-radius:12px;flex-direction:column;gap:.5rem;padding:.875rem;display:flex}.utility-export-group-title{color:var(--pb-text-primary);font-size:.72rem;font-weight:600}.utility-export-group-fields{flex-direction:column;gap:.375rem;display:flex}.utility-export-group-fields .utility-step-card{padding:.5rem .75rem}.utility-export-field-label{color:var(--pb-text-primary);font-size:.8rem;font-weight:500;line-height:1.25}.utility-export-field-meta{justify-content:space-between;align-items:baseline;gap:.875rem;width:100%;min-width:0;display:flex}.utility-export-field-sub{color:var(--pb-text-tertiary);text-align:right;white-space:nowrap;text-overflow:ellipsis;flex:0 auto;min-width:0;font-family:JetBrains Mono,monospace;font-size:.66rem;line-height:1.2;overflow:hidden}.utility-export-options-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-app);border-radius:10px;padding:1rem}.utility-export-inline-label{color:var(--pb-text-primary);font-size:.72rem;font-weight:600}.utility-export-multi-grid{gap:.5rem;display:grid}@media (min-width:640px){.utility-export-multi-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}.utility-export-summary-grid{gap:.625rem;display:grid}@media (min-width:640px){.utility-export-summary-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}.utility-export-summary-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-app);border-radius:10px;padding:.875rem 1rem}.utility-export-summary-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);font-family:Syne,sans-serif;font-size:.58rem;font-weight:700}.utility-export-summary-value{color:var(--pb-text-primary);margin-top:.35rem;font-family:JetBrains Mono,monospace;font-size:.92rem;font-weight:600}.utility-tool-select.dropdown-select{--pb-control-font-size:.82rem;--pb-control-line-height:1.25rem;--pb-control-radius:8px;width:100%}.utility-tool-select .dropdown-select-trigger{border-color:var(--pb-border-default);background:var(--pb-surface-input);min-height:0;box-shadow:none;padding:.5rem .75rem}.utility-tool-select .dropdown-select-trigger-label{color:var(--pb-text-primary);font-weight:400}.utility-tool-select .dropdown-select-chevron{color:var(--pb-text-secondary)}.utility-tool-output-field{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-app);color:var(--pb-text-secondary);border-radius:8px;padding:.5rem .75rem;font-family:DM Sans,sans-serif;font-size:.82rem;line-height:1.25rem}.utility-tool-browse-button{gap:.375rem;min-height:0;padding:.375rem .875rem;font-size:.78rem;font-weight:600;line-height:1.5}.utility-tool-browse-button svg{width:.875rem;height:.875rem}.utility-tool-table-wrap{border:1px solid var(--pb-border-subtle);border-radius:10px;overflow:hidden}.utility-tool-table{border-collapse:collapse;width:100%}.utility-tool-table th{text-align:left;letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);background:var(--pb-surface-shell);border-bottom:1px solid var(--pb-border-default);padding:.4375rem .75rem;font-family:Syne,sans-serif;font-size:.52rem;font-weight:700}.utility-tool-table th.r,.utility-tool-table td.r{text-align:right}.utility-tool-table td{color:var(--pb-text-secondary);border-bottom:1px solid var(--pb-border-subtle);padding:.5rem .75rem;font-size:.78rem}.utility-tool-table tbody tr:last-child td{border-bottom:none}.utility-tool-table tbody tr:hover td{background:var(--pb-surface-selected)}.utility-tool-table-output{color:var(--pb-status-success)}.utility-tool-action-footer{border-top:1px solid var(--pb-border-subtle);justify-content:flex-end;align-items:center;gap:.625rem;padding:.75rem 0 0;display:flex}.utility-tool-action-footer .settings-footer-actions{gap:.625rem;margin-left:auto}.utility-tool-action-footer .btn-ghost,.utility-tool-action-footer .btn-primary{min-height:0;padding:.5625rem 1.125rem;font-size:.82rem;line-height:1.25rem}.utility-tool-action-footer-card{margin-top:.25rem}.utility-step-card-active{border-color:var(--pb-interactive);background-color:var(--pb-surface-selected)}.utility-step-card-static{border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card)}.utility-empty-state-compact{padding-top:1rem;padding-bottom:1rem}.utility-template-table-type{color:var(--pb-text-primary);font-size:.78rem;font-weight:600}.utility-template-meta-token{color:var(--pb-text-tertiary);font-family:JetBrains Mono,monospace;font-size:.72rem}.utility-template-meta-token strong{color:var(--pb-text-secondary);font-weight:600}.utilities-queue-table-wrap{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:14px;overflow:hidden}.utilities-queue-table{border-collapse:collapse;width:100%}.utilities-queue-table th{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);text-align:left;--tw-font-weight:var(--font-weight-bold);font-family:Syne,sans-serif;font-size:.58rem;font-weight:var(--font-weight-bold);--tw-tracking:.1em;letter-spacing:.1em;color:var(--pb-text-dim);text-transform:uppercase}.utilities-queue-table th.c,.utilities-queue-table td.c{text-align:center}.utilities-queue-table th.r,.utilities-queue-table td.r{text-align:right}.utilities-queue-table td{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border-subtle);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);font-size:.82rem}.utilities-queue-table tbody:last-child tr:last-child td{border-bottom:0}.utilities-queue-row:hover td{background-color:var(--pb-surface-selected)}.utilities-queue-row>td{vertical-align:middle}.utilities-queue-led{height:calc(var(--spacing) * 2);width:calc(var(--spacing) * 2);background-color:var(--pb-text-dim);border-radius:3.40282e38px;display:inline-block}@supports (color:color-mix(in lab, red, red)){.utilities-queue-led{background-color:color-mix(in srgb, var(--pb-text-dim) 45%, transparent)}}.utilities-queue-led-green{background-color:var(--pb-success);box-shadow:0 0 8px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-led-green{box-shadow:0 0 8px color-mix(in srgb, var(--pb-success) 30%, transparent)}}.utilities-queue-led-amber{background-color:var(--pb-warning);box-shadow:0 0 8px var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-led-amber{box-shadow:0 0 8px color-mix(in srgb, var(--pb-warning) 28%, transparent)}}.utilities-queue-led-blue{background-color:var(--pb-info);box-shadow:0 0 8px var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-led-blue{box-shadow:0 0 8px color-mix(in srgb, var(--pb-info) 28%, transparent)}}.utilities-queue-job-name{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-primary)}.utilities-queue-job-type{margin-top:calc(var(--spacing) * .5);color:var(--pb-text-dim);font-size:.72rem}.utilities-queue-job-meta{margin-top:var(--spacing);color:var(--pb-text-secondary);font-size:.68rem}.utilities-queue-items{--tw-font-weight:var(--font-weight-semibold);font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:var(--font-weight-semibold);color:var(--pb-text-secondary)}.utilities-queue-progress-cell .series-mission-control-bar{gap:.625rem}.utilities-queue-progress-cell .series-mission-control-bar-track{min-width:120px}.utilities-queue-time{color:var(--pb-text-secondary);font-family:JetBrains Mono,monospace;font-size:.72rem}.utilities-queue-actions{justify-content:flex-end;align-items:center;gap:calc(var(--spacing) * 1.5);display:flex}.utilities-queue-act-btn{height:calc(var(--spacing) * 8);width:calc(var(--spacing) * 8);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:8px;justify-content:center;align-items:center;display:inline-flex}.utilities-queue-act-btn:hover{border-color:var(--pb-border-strong);color:var(--pb-text);background-color:var(--pb-bg-card-hover)}.utilities-queue-act-btn-danger{color:var(--pb-error)}.utilities-queue-act-btn-danger:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-act-btn-danger:hover{border-color:color-mix(in srgb, var(--pb-error) 36%, transparent)}}.utilities-queue-act-btn-danger:hover{background-color:var(--pb-error-dim);color:var(--pb-error)}.utilities-queue-detail-cell{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-detail-cell{background-color:color-mix(in srgb, var(--pb-bg-surface) 72%, transparent)}}.utilities-queue-detail-summary{align-items:center;column-gap:calc(var(--spacing) * 6);row-gap:var(--spacing);border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);color:var(--pb-text-dim);flex-wrap:wrap;font-size:.72rem;display:flex}.utilities-queue-detail-summary strong{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.utilities-queue-detail-label{--tw-tracking:.08em;letter-spacing:.08em;text-transform:uppercase}.utilities-queue-empty{justify-content:center;align-items:center;gap:calc(var(--spacing) * 2);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 12);text-align:center;flex-direction:column;display:flex}.utilities-queue-empty-compact{padding-block:calc(var(--spacing) * 8)}.utilities-queue-empty-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}.utilities-queue-empty-copy{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.queue-job-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1);overflow:hidden}.queue-job-card-expanded{border-color:var(--pb-border-hover);box-shadow:var(--pb-shadow-2)}.queue-progress-track{height:calc(var(--spacing) * 1.5);background-color:var(--pb-bg-card-hover);border-radius:3.40282e38px;width:100%;overflow:hidden}.queue-progress-fill{height:100%;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.5s;background-color:var(--pb-success);border-radius:3.40282e38px;transition-duration:.5s}.queue-progress-fill-paused{background-color:var(--pb-warning)}.selection-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);text-align:left;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1)}.selection-card:not(.selection-card-active):hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.selection-card:not(.selection-card-active):hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 55%, transparent)}}.selection-card-active{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.selection-card-active{border-color:color-mix(in srgb, var(--pb-interactive) 42%, transparent)}}.selection-card-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.selection-card-active{background-color:color-mix(in srgb, var(--pb-interactive) 14%, var(--pb-bg-card))}}.selection-card-active{box-shadow:var(--pb-shadow-2)}.selection-card-icon{height:calc(var(--spacing) * 11);width:calc(var(--spacing) * 11);border-radius:var(--radius-xl);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-shrink:0;justify-content:center;align-items:center;display:flex}.selection-card-icon-active{color:var(--pb-interactive);background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.selection-card-icon-active{background-color:color-mix(in srgb, var(--pb-interactive) 14%, transparent)}}.settings-theme-choice{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}.settings-theme-choice-icon{height:calc(var(--spacing) * 9);width:calc(var(--spacing) * 9);border-radius:var(--radius-lg)}.settings-media-preview-panel{background-color:var(--pb-surface-selected);border-color:var(--pb-border-hover)}@supports (color:color-mix(in lab, red, red)){.settings-media-preview-panel{border-color:color-mix(in srgb, var(--pb-border-hover) 85%, transparent)}}.settings-media-preview-panel{overflow-wrap:anywhere;min-height:4.625rem;transition:opacity .16s,border-color .16s}.settings-media-preview-panel.is-loading{opacity:.68;border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.settings-media-preview-panel.is-loading{border-color:color-mix(in srgb, var(--pb-interactive) 30%, var(--pb-border-hover))}}.workflow-step{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);box-shadow:var(--pb-shadow-1)}.workflow-step-active{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-step-active{border-color:color-mix(in srgb, var(--pb-interactive) 32%, transparent)}}.workflow-step-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-step-active{background-color:color-mix(in srgb, var(--pb-interactive) 10%, var(--pb-bg-card))}}.workflow-step-complete{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.workflow-step-complete{border-color:color-mix(in srgb, var(--pb-success) 28%, transparent)}}.workflow-step-complete{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.workflow-step-complete{background-color:color-mix(in srgb, var(--pb-success) 10%, var(--pb-bg-card))}}.workflow-step-badge{height:calc(var(--spacing) * 8);width:calc(var(--spacing) * 8);border-style:var(--tw-border-style);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);border-width:1px;border-color:var(--pb-border-hover);color:var(--pb-text-secondary);background-color:var(--pb-bg-card);border-radius:3.40282e38px;justify-content:center;align-items:center;display:flex}.workflow-step-active .workflow-step-badge{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-step-active .workflow-step-badge{border-color:color-mix(in srgb, var(--pb-interactive) 32%, transparent)}}.workflow-step-active .workflow-step-badge{color:var(--pb-interactive);background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-step-active .workflow-step-badge{background-color:color-mix(in srgb, var(--pb-interactive) 12%, transparent)}}.workflow-step-complete .workflow-step-badge{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.workflow-step-complete .workflow-step-badge{border-color:color-mix(in srgb, var(--pb-success) 28%, transparent)}}.workflow-step-complete .workflow-step-badge{color:var(--pb-success);background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.workflow-step-complete .workflow-step-badge{background-color:color-mix(in srgb, var(--pb-success) 12%, transparent)}}.workflow-step-label{--tw-font-weight:var(--font-weight-medium);font-size:10px;font-weight:var(--font-weight-medium);--tw-tracking:.12em;letter-spacing:.12em;color:var(--pb-text-dim);text-transform:uppercase}.workflow-step-title{margin-top:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.checklist-item{align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary);display:flex}.section-body{padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4)}.section-header{gap:calc(var(--spacing) * 2);border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4);flex-direction:column;display:flex}@media (min-width:40rem){.section-header{flex-direction:row;justify-content:space-between;align-items:flex-start}}.section-header-copy{min-width:0}:where(.section-header-copy>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}.section-header-actions{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;flex-shrink:0;display:flex}.section-eyebrow{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.section-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-primary);text-transform:uppercase}.section-title-plain{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.section-description{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.settings-connection-list{gap:calc(var(--spacing) * 3);flex-direction:column;display:flex}.settings-connection-card{cursor:pointer;border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);text-align:left;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.settings-connection-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.settings-connection-card-muted{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card-muted{background-color:color-mix(in srgb, var(--pb-bg-surface) 70%, transparent)}}.settings-connection-card-muted{opacity:.78}.settings-connection-card .downloads-action-btn.is-danger{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card .downloads-action-btn.is-danger{border-color:color-mix(in srgb, var(--pb-error) 28%, var(--pb-border-subtle))}}.settings-connection-card .downloads-action-btn.is-danger{background:var(--pb-error-dim);color:var(--pb-error)}.settings-connection-card .downloads-action-btn.is-danger:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card .downloads-action-btn.is-danger:hover{border-color:color-mix(in srgb, var(--pb-error) 44%, transparent)}}.settings-connection-card .downloads-action-btn.is-danger:hover{background:var(--pb-error-dim)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card .downloads-action-btn.is-danger:hover{background:color-mix(in srgb, var(--pb-error-dim) 86%, transparent)}}.settings-connection-card .downloads-action-btn.is-danger:hover{color:var(--pb-error)}.admin-nav-link{align-items:center;gap:calc(var(--spacing) * 2.5);border-radius:var(--radius-lg);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 2.5);padding-block:calc(var(--spacing) * 2);--tw-font-weight:var(--font-weight-medium);font-size:.8125rem;font-weight:var(--font-weight-medium);color:var(--pb-text-dim);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-width:1px;border-color:#0000;display:flex}.admin-nav-link:hover{color:var(--pb-text-secondary);background-color:var(--pb-surface-selected)}.admin-nav-link-active{color:var(--pb-interactive);background-color:var(--pb-surface-selected)}.admin-nav-link-icon{height:calc(var(--spacing) * 4);width:calc(var(--spacing) * 4);flex-shrink:0;justify-content:center;align-items:center;display:flex}.admin-nav-link-active .admin-nav-link-icon{color:var(--pb-interactive)}.admin-nav-link-copy{min-width:0}.admin-nav-link-title{text-overflow:ellipsis;white-space:nowrap;--tw-font-weight:var(--font-weight-medium);font-size:.8125rem;font-weight:var(--font-weight-medium);overflow:hidden}:where(.settings-rows>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)));border-color:var(--pb-border)}.settings-row{gap:calc(var(--spacing) * 4);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4);grid-template-columns:repeat(1,minmax(0,1fr));display:grid}@media (min-width:40rem){.settings-row{grid-template-columns:220px minmax(0,1fr);align-items:flex-start}}.settings-row-meta{padding-top:var(--spacing)}.settings-row-label{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);display:block}.settings-row-help{margin-top:calc(var(--spacing) * .5);--tw-leading:calc(var(--spacing) * 5);font-size:11px;line-height:calc(var(--spacing) * 5);color:var(--pb-text-dim)}.settings-row-content{min-width:0}.settings-rows-align-end .settings-row-content{flex-direction:column;align-items:flex-end;display:flex}.settings-footer{gap:calc(var(--spacing) * 3);border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 3);flex-direction:column;display:flex}@media (min-width:40rem){.settings-footer{flex-direction:row;justify-content:space-between;align-items:center}}.settings-footer{background-color:var(--pb-bg-surface);overflow-anchor:none}@media (min-width:40rem){.settings-footer-end{justify-content:flex-end}}.settings-footer-copy{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.settings-footer-actions{align-items:center;gap:calc(var(--spacing) * 3);flex-wrap:wrap;display:flex}.settings-footer-actions .btn-primary,.settings-footer-actions .btn-ghost,.settings-footer-actions .btn-danger,.settings-footer-actions .btn-warning{min-height:calc(var(--spacing) * 10);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.field-note{margin-top:calc(var(--spacing) * 1.5);align-items:flex-start;gap:calc(var(--spacing) * 1.5);--tw-leading:calc(var(--spacing) * 5);font-size:11px;line-height:calc(var(--spacing) * 5);display:flex}.field-note code{color:var(--pb-text-secondary)}.field-note-icon{margin-top:calc(var(--spacing) * .5);flex-shrink:0;width:.75rem;height:.75rem}.field-note-warning{color:var(--pb-warning)}.field-note-info{color:var(--pb-info)}.field-note-danger{color:var(--pb-error)}.field-note-success{color:var(--pb-success)}.alert-banner{gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4);border-width:1px;flex-direction:column;display:flex}@media (min-width:40rem){.alert-banner{flex-direction:row;justify-content:space-between;align-items:flex-start}}.alert-banner{box-shadow:var(--pb-shadow-1)}.alert-banner-main{align-items:flex-start;gap:calc(var(--spacing) * 4);min-width:0;display:flex}.alert-banner-icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:var(--radius-lg);flex-shrink:0;justify-content:center;align-items:center;display:flex}.alert-banner-copy{min-width:0}:where(.alert-banner-copy>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}.alert-banner-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.alert-banner-description{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.alert-banner-actions{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;flex-shrink:0;display:flex}.alert-banner-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.alert-banner-info{background-color:color-mix(in srgb, var(--pb-info) 10%, var(--pb-bg-card))}}.alert-banner-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.alert-banner-info{border-color:color-mix(in srgb, var(--pb-info) 18%, transparent)}}.alert-banner-info .alert-banner-icon{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.alert-banner-info .alert-banner-icon{background-color:color-mix(in srgb, var(--pb-info) 18%, transparent)}}.alert-banner-info .alert-banner-icon{color:var(--pb-info)}.alert-banner-warning{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.alert-banner-warning{background-color:color-mix(in srgb, var(--pb-warning) 10%, var(--pb-bg-card))}}.alert-banner-warning{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.alert-banner-warning{border-color:color-mix(in srgb, var(--pb-warning) 18%, transparent)}}.alert-banner-warning .alert-banner-icon{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.alert-banner-warning .alert-banner-icon{background-color:color-mix(in srgb, var(--pb-warning) 18%, transparent)}}.alert-banner-warning .alert-banner-icon{color:var(--pb-warning)}.alert-banner-danger{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.alert-banner-danger{background-color:color-mix(in srgb, var(--pb-error) 10%, var(--pb-bg-card))}}.alert-banner-danger{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.alert-banner-danger{border-color:color-mix(in srgb, var(--pb-error) 18%, transparent)}}.alert-banner-danger .alert-banner-icon{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.alert-banner-danger .alert-banner-icon{background-color:color-mix(in srgb, var(--pb-error) 18%, transparent)}}.alert-banner-danger .alert-banner-icon{color:var(--pb-error)}.alert-banner-success{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.alert-banner-success{background-color:color-mix(in srgb, var(--pb-success) 10%, var(--pb-bg-card))}}.alert-banner-success{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.alert-banner-success{border-color:color-mix(in srgb, var(--pb-success) 18%, transparent)}}.alert-banner-success .alert-banner-icon{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.alert-banner-success .alert-banner-icon{background-color:color-mix(in srgb, var(--pb-success) 18%, transparent)}}.alert-banner-success .alert-banner-icon{color:var(--pb-success)}.modal-shell{z-index:50;padding:calc(var(--spacing) * 4);padding-top:calc(var(--spacing) * 16);justify-content:center;align-items:flex-start;display:flex;position:fixed;inset:0}.modal-backdrop{z-index:0;background-color:var(--pb-bg-overlay);--tw-backdrop-blur:blur(var(--blur-xs));-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);position:fixed;inset:0}.modal-panel{z-index:10;border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);width:100%;box-shadow:var(--pb-shadow-overlay);flex-direction:column;display:flex;position:relative;overflow:hidden}.modal-header{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);justify-content:space-between;align-items:center;display:flex}.modal-body{flex:1;overflow-y:auto}.modal-footer{border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);flex-shrink:0;justify-content:space-between;align-items:center;display:flex}.donation-modal-shell{align-items:flex-start;padding-top:4.75rem}.donation-modal-panel{border-radius:1.5rem;width:min(100vw - 2rem,56rem);max-height:calc(100dvh - 6rem)}.donation-modal-header{border-bottom-color:var(--pb-border-subtle);background:radial-gradient(circle at top left, var(--pb-brand-dim), transparent 18rem), var(--pb-bg-card);align-items:flex-start;gap:1rem}.donation-modal-kicker{letter-spacing:.18em;text-transform:uppercase;color:var(--pb-brand);font-family:JetBrains Mono,monospace;font-size:.68rem;font-weight:700}.donation-modal-title{color:var(--pb-text);letter-spacing:-.03em;margin-top:.2rem;font-family:Syne,sans-serif;font-size:clamp(1.35rem,2.5vw,2rem);font-weight:800;line-height:1.05}.donation-modal-copy{max-width:38rem;color:var(--pb-text-sec);margin-top:.45rem;font-size:.9rem;line-height:1.55}.donation-modal-body{background:linear-gradient(135deg, transparent, var(--pb-surface-selected)), var(--pb-bg-surface);gap:1rem;padding:1rem;display:grid}.donation-option-card{border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:1.25rem;grid-template-columns:minmax(0,1fr) 10.5rem;align-items:stretch;gap:1rem;padding:1rem;display:grid}.donation-option-card--coffee{border-color:#fd0}@supports (color:color-mix(in lab, red, red)){.donation-option-card--coffee{border-color:color-mix(in srgb, #fd0 38%, var(--pb-border-subtle))}}.donation-option-card--liberapay{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.donation-option-card--liberapay{border-color:color-mix(in srgb, var(--pb-success) 32%, var(--pb-border-subtle))}}.donation-option-main{flex-direction:column;justify-content:space-between;gap:1rem;min-width:0;display:flex}.donation-option-eyebrow{letter-spacing:.16em;text-transform:uppercase;color:var(--pb-text-sec);font-family:JetBrains Mono,monospace;font-size:.64rem;font-weight:700}.donation-option-title{color:var(--pb-text);margin-top:.25rem;font-family:Syne,sans-serif;font-size:1.1rem;font-weight:800}.donation-option-copy{max-width:28rem;color:var(--pb-text-sec);margin-top:.35rem;font-size:.82rem;line-height:1.5}.donation-option-button{border-radius:999px;justify-content:center;align-items:center;width:fit-content;padding:.62rem 1rem;font-size:.82rem;font-weight:800;line-height:1;text-decoration:none;transition:transform .16s,box-shadow .16s,filter .16s;display:inline-flex}.donation-option-button:hover,.donation-option-button:focus-visible{filter:brightness(1.02);transform:translateY(-1px)}.donation-option-button:focus-visible{outline:2px solid var(--pb-focus-outline);outline-offset:3px}.donation-option-button--coffee{color:#16130a;background:#fd0;box-shadow:0 10px 20px #fd03}.donation-option-button--liberapay{background:var(--pb-success);color:#f7f1e8;box-shadow:0 10px 20px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.donation-option-button--liberapay{box-shadow:0 10px 20px color-mix(in srgb, var(--pb-success) 20%, transparent)}}[data-theme=dark] .donation-option-button--liberapay{color:#16130a}.donation-qr-frame{border:1px solid var(--pb-border-subtle);background:#fff;border-radius:1rem;place-items:center;min-height:10.5rem;padding:.625rem;display:grid}.donation-qr{object-fit:contain;width:9.25rem;height:9.25rem}@media (max-width:720px){.donation-modal-shell{padding-top:3.25rem}.donation-modal-panel{max-height:calc(100dvh - 4rem)}.donation-option-card{grid-template-columns:1fr}.donation-option-button{width:100%}}.issue-search-modal-panel{border-radius:1.25rem;width:min(100vw - 2rem,78rem);max-width:none;max-height:90vh}.issue-search-modal-header{border-bottom-color:var(--pb-border-subtle);align-items:flex-start;padding:1.125rem 1.5rem}.issue-search-modal-title{letter-spacing:.03em;text-transform:uppercase;color:var(--pb-text);font-family:Syne,sans-serif;font-size:1.05rem;font-weight:800}.issue-search-modal-title span{color:var(--pb-brand)}.issue-search-modal-subtitle{color:var(--pb-text-dim);margin-top:.1875rem;font-size:.78rem}.issue-search-modal-stats{flex-wrap:wrap;gap:.375rem;margin-top:.5rem;display:flex}.issue-search-modal-close{border-radius:var(--radius-lg);--tw-border-style:none;color:var(--pb-text-dim);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background-color:#0000;border-style:none;justify-content:center;align-items:center;display:inline-flex}@media (hover:hover){.issue-search-modal-close:hover{color:var(--pb-text-primary)}}.issue-search-modal-close{width:1.875rem;height:1.875rem}.issue-search-modal-close:hover{background:var(--pb-surface-selected)}.issue-search-modal-body{padding:0}.issue-search-loading-state{min-height:24rem}.issue-search-loading-title{color:var(--pb-text);font-size:.82rem;font-weight:600}.issue-search-loading-copy{color:var(--pb-text-dim);font-size:.72rem}.issue-search-dc-status{border-top:1px solid var(--pb-border-subtle);color:var(--pb-text-dim);align-items:center;gap:.625rem;padding:.75rem 1rem;font-size:.78rem;display:flex}.issue-search-dc-results{border-top:1px solid var(--pb-border-subtle)}.issue-search-dc-results-heading{align-items:center;gap:.5rem;padding:.75rem 1rem;display:flex}.issue-search-modal-footer{border-top-color:var(--pb-border-subtle);justify-content:flex-end;padding:.75rem 1.5rem}.issue-search-modal-footer-meta{color:var(--pb-text-dim);font-family:JetBrains Mono,monospace;font-size:.68rem}.issue-search-modal-footer-meta:empty{display:none}.issue-search-modal-footer-meta strong{color:var(--pb-text);font-weight:700}.issue-search-modal-footer-close{margin-left:auto}.inline-alert{align-items:flex-start;gap:calc(var(--spacing) * 2);border-radius:var(--radius-lg);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2.5);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));border-width:1px;display:flex}.inline-alert-icon{margin-top:calc(var(--spacing) * .5);flex-shrink:0}.inline-alert-danger{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.inline-alert-danger{background-color:color-mix(in srgb, var(--pb-error) 12%, var(--pb-bg-card))}}.inline-alert-danger{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.inline-alert-danger{border-color:color-mix(in srgb, var(--pb-error) 22%, transparent)}}.inline-alert-danger{color:var(--pb-error)}.inline-alert-warning{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.inline-alert-warning{background-color:color-mix(in srgb, var(--pb-warning) 12%, var(--pb-bg-card))}}.inline-alert-warning{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.inline-alert-warning{border-color:color-mix(in srgb, var(--pb-warning) 22%, transparent)}}.inline-alert-warning{color:var(--pb-warning)}.inline-alert-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.inline-alert-info{background-color:color-mix(in srgb, var(--pb-info) 12%, var(--pb-bg-card))}}.inline-alert-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.inline-alert-info{border-color:color-mix(in srgb, var(--pb-info) 22%, transparent)}}.inline-alert-info{color:var(--pb-info)}.inline-alert-success{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.inline-alert-success{background-color:color-mix(in srgb, var(--pb-success) 12%, var(--pb-bg-card))}}.inline-alert-success{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.inline-alert-success{border-color:color-mix(in srgb, var(--pb-success) 22%, transparent)}}.inline-alert-success{color:var(--pb-success)}.field-invalid{border-color:var(--pb-error)!important}@supports (color:color-mix(in lab, red, red)){.field-invalid{border-color:color-mix(in srgb, var(--pb-error) 52%, transparent)!important}}.icon-btn{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));justify-content:center;align-items:center;display:inline-flex}@media (hover:hover){.icon-btn:hover{border-color:var(--pb-border-strong);color:var(--pb-text-primary)}}.icon-btn:disabled{cursor:not-allowed;opacity:.5}.icon-btn{min-width:var(--pb-control-min-height);width:var(--pb-control-min-height);min-height:var(--pb-control-min-height);border-radius:var(--pb-control-radius);padding:0}.toggle-switch{cursor:pointer;background-color:var(--pb-bg-card-hover);width:2.25rem;height:1.25rem;box-shadow:inset 0 0 0 1px var(--pb-border-hover);border-radius:999px;flex-shrink:0;transition:background-color .18s,box-shadow .18s;display:inline-flex;position:relative}label:has(>.toggle-input+.toggle-switch){position:relative}.toggle-input{clip:rect(0, 0, 0, 0);clip-path:inset(50%);white-space:nowrap;border:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.toggle-input-overlay{opacity:0;clip:auto;clip-path:none;white-space:normal;z-index:1;width:100%;height:100%;margin:0;inset:0}.toggle-switch:after{content:"";background-color:var(--pb-brand);width:1rem;height:1rem;box-shadow:0 1px 2px var(--pb-text-primary);border-radius:999px;position:absolute;top:2px;left:2px}@supports (color:color-mix(in lab, red, red)){.toggle-switch:after{box-shadow:0 1px 2px color-mix(in srgb, var(--pb-text-primary) 14%, transparent)}}.toggle-switch:after{transition:transform .18s}.peer:checked+.toggle-switch{background-color:var(--pb-interactive);box-shadow:inset 0 0 0 1px var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.peer:checked+.toggle-switch{box-shadow:inset 0 0 0 1px color-mix(in srgb, var(--pb-interactive) 34%, transparent)}}.peer:checked+.toggle-switch:after{background-color:var(--pb-surface-shell);transform:translate(1rem)}.peer:focus-visible+.toggle-switch{outline:2px solid var(--pb-focus-outline);outline-offset:3px;box-shadow:0 0 0 4px var(--pb-focus-ring)}.strength-segment{height:calc(var(--spacing) * 1.5);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.3s;border-radius:3.40282e38px;flex:1;transition-duration:.3s}.strength-segment-neutral{background-color:var(--pb-bg-card-hover)}.strength-segment-danger{background-color:var(--pb-error)}.strength-segment-warning{background-color:var(--pb-warning)}.strength-segment-success{background-color:var(--pb-success)}.input-pb{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-input);width:100%;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);font-family:DM Sans,sans-serif;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-primary);border-radius:10px}.input-pb::placeholder{color:var(--pb-text-dim)}.input-pb:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;border-color:#0000;outline-style:none}@media (forced-colors:active){.input-pb:focus{outline-offset:2px;outline:2px solid #0000}}.checkbox-pb{height:calc(var(--spacing) * 4);width:calc(var(--spacing) * 4);border-color:var(--pb-border-hover);background-color:var(--pb-bg-input);color:var(--pb-interactive);border-radius:.25rem}.checkbox-pb:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-ring-offset-width:0px;--tw-ring-offset-shadow:var(--tw-ring-inset,) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color)}.select-pb{appearance:none;border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-input);width:100%;padding-block:calc(var(--spacing) * 2);padding-right:calc(var(--spacing) * 8);padding-left:calc(var(--spacing) * 3);font-family:DM Sans,sans-serif;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-primary);border-radius:10px}.select-pb:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;border-color:#0000;outline-style:none}@media (forced-colors:active){.select-pb:focus{outline-offset:2px;outline:2px solid #0000}}.select-pb{background-image:url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 20 20'%3E%3Cpath stroke='%237A8599' stroke-linecap='round' stroke-linejoin='round' stroke-width='1.5' d='M6 8l4 4 4-4'/%3E%3C/svg%3E");background-position:right .5rem center;background-repeat:no-repeat;background-size:1.25rem}.dropdown-select{min-width:0;position:relative}.dropdown-select-trigger{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);width:100%;color:var(--pb-text-primary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));justify-content:space-between;align-items:center;display:flex}@media (hover:hover){.dropdown-select-trigger:hover{border-color:var(--pb-border-strong)}}.dropdown-select-trigger:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;border-color:#0000;outline-style:none}@media (forced-colors:active){.dropdown-select-trigger:focus{outline-offset:2px;outline:2px solid #0000}}.dropdown-select-trigger:disabled{cursor:not-allowed;opacity:.6}.dropdown-select-trigger{gap:var(--pb-control-gap);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.dropdown-select-trigger-label{text-overflow:ellipsis;white-space:nowrap;text-align:left;flex:1;min-width:0;overflow:hidden}.dropdown-select-chevron{color:var(--pb-text-secondary);transition-property:transform,translate,scale,rotate;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));width:var(--pb-control-icon-size);height:var(--pb-control-icon-size);flex-shrink:0}.dropdown-select-panel{z-index:80;max-height:calc(var(--spacing) * 60);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card);padding-block:var(--spacing);--tw-shadow:0 25px 50px -12px var(--tw-shadow-color,#00000040);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-shadow-color:#0003;position:fixed;top:0;left:0;overflow-y:auto}@supports (color:color-mix(in lab, red, red)){.dropdown-select-panel{--tw-shadow-color:color-mix(in oklab, color-mix(in oklab, var(--color-black) 20%, transparent) var(--tw-shadow-alpha), transparent)}}.dropdown-select-panel{max-height:var(--pb-dropdown-panel-max-height,15rem);will-change:top, left}.dropdown-select-panel[data-ready=false]{visibility:hidden;pointer-events:none}.dropdown-select-option{text-align:left;width:100%;color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));justify-content:space-between;align-items:center;gap:var(--pb-control-gap);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-panel-item-px);padding-block:var(--pb-control-panel-item-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);display:flex}.dropdown-select-option-label{text-align:left;white-space:nowrap;overflow-wrap:normal;flex:1;min-width:0}.import-reconcile-issue-dropdown .dropdown-select-trigger{align-items:flex-start;height:auto}.import-reconcile-issue-dropdown .dropdown-select-trigger-label{white-space:normal;overflow-wrap:anywhere;text-overflow:clip;overflow:visible}.dropdown-select-panel-wrap .dropdown-select-option{align-items:flex-start}.dropdown-select-panel-wrap .dropdown-select-option-label{white-space:normal;overflow-wrap:anywhere;word-break:normal}.dropdown-select-option:hover,.dropdown-select-option-active{background-color:var(--pb-info-dim);color:var(--pb-text-primary)}.dropdown-select-option-active{transition-duration:0s}.dropdown-select-option-selected{background-color:var(--pb-info-dim);color:var(--pb-text-primary);font-weight:500}.dropdown-select-option-check{width:var(--pb-control-icon-size);height:var(--pb-control-icon-size);flex-shrink:0}.direct-provider-uri-control{position:relative}.direct-provider-uri-input{padding-right:calc(var(--spacing) * 10);font-family:JetBrains Mono,Fira Code,monospace}.direct-provider-uri-toggle{width:calc(var(--spacing) * 10);border-top-right-radius:var(--radius-lg);border-bottom-right-radius:var(--radius-lg);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));inset-block:0;justify-content:center;align-items:center;display:flex;position:absolute;right:0}@media (hover:hover){.direct-provider-uri-toggle:hover{color:var(--pb-text-primary)}}.direct-provider-uri-toggle:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;outline-style:none}@media (forced-colors:active){.direct-provider-uri-toggle:focus{outline-offset:2px;outline:2px solid #0000}}.direct-provider-uri-toggle:focus{--tw-ring-inset:inset}.direct-provider-uri-options{z-index:90;margin-top:var(--spacing);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card);padding-block:var(--spacing);--tw-shadow:0 25px 50px -12px var(--tw-shadow-color,#00000040);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-shadow-color:#0003;position:absolute;top:100%;left:0;right:0;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.direct-provider-uri-options{--tw-shadow-color:color-mix(in oklab, color-mix(in oklab, var(--color-black) 20%, transparent) var(--tw-shadow-alpha), transparent)}}.direct-provider-uri-option{justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 2);width:100%;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);text-align:left;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:flex}@media (hover:hover){.direct-provider-uri-option:hover{background-color:var(--pb-info-dim);color:var(--pb-text-primary)}}.direct-provider-uri-option:focus{background-color:var(--pb-info-dim);color:var(--pb-text-primary);--tw-outline-style:none;outline-style:none}@media (forced-colors:active){.direct-provider-uri-option:focus{outline-offset:2px;outline:2px solid #0000}}.direct-provider-uri-option-selected{background-color:var(--pb-info-dim);color:var(--pb-text-primary)}.search-field{min-width:0;position:relative}.search-field-icon{pointer-events:none;z-index:10;--tw-translate-y:calc(calc(1 / 2 * 100%) * -1);translate:var(--tw-translate-x) var(--tw-translate-y);color:var(--pb-text-secondary);top:50%;left:var(--pb-control-search-icon-left);width:var(--pb-control-icon-size);height:var(--pb-control-icon-size);position:absolute}.search-field-input{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);width:100%;color:var(--pb-text-primary)}.search-field-input::placeholder{color:var(--pb-text-secondary)}.search-field-input:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;border-color:#0000;outline-style:none}@media (forced-colors:active){.search-field-input:focus{outline-offset:2px;outline:2px solid #0000}}.search-field-input{min-height:var(--pb-control-min-height);padding-inline-start:var(--pb-control-search-padding-left);padding-inline-end:var(--pb-control-search-padding-right);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius);appearance:none;background-image:none}.search-field-editor{white-space:nowrap;min-height:42px;overflow:hidden}.search-field-editor[data-empty=true]:before{content:attr(data-placeholder);color:var(--pb-text-secondary)}.search-field-input::-webkit-search-decoration{appearance:none;display:none}.search-field-input::-webkit-search-cancel-button{appearance:none;display:none}.search-field-input::-webkit-search-results-button{appearance:none;display:none}.search-field-input::-webkit-search-results-decoration{appearance:none;display:none}.search-field-input::-ms-clear{width:0;height:0;display:none}.search-field-input::-ms-reveal{width:0;height:0;display:none}.search-field-clear{z-index:10;--tw-translate-y:calc(calc(1 / 2 * 100%) * -1);translate:var(--tw-translate-x) var(--tw-translate-y);padding:calc(var(--spacing) * .5);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:.25rem;position:absolute;top:50%}@media (hover:hover){.search-field-clear:hover{color:var(--pb-text-primary)}}.search-field-clear{right:var(--pb-control-clear-right)}.search-history-panel{z-index:30;margin-top:var(--spacing);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card);--tw-shadow:0 20px 25px -5px var(--tw-shadow-color,#0000001a), 0 8px 10px -6px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-shadow-color:#0006;position:absolute;top:100%;left:0;right:0;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.search-history-panel{--tw-shadow-color:color-mix(in oklab, color-mix(in oklab, var(--color-black) 40%, transparent) var(--tw-shadow-alpha), transparent)}}.search-history-panel-header{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);justify-content:space-between;align-items:center;display:flex}.search-history-panel-title{--tw-font-weight:var(--font-weight-semibold);font-size:10px;font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-wider);letter-spacing:var(--tracking-wider);color:var(--pb-text-dim);text-transform:uppercase}.search-history-panel-clear{color:var(--pb-text-dim);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));font-size:10px}@media (hover:hover){.search-history-panel-clear:hover{color:var(--pb-error)}}.search-history-list{max-height:280px;padding-block:var(--spacing);overflow-y:auto}.search-history-item{align-items:center;display:flex}.search-history-item-button{align-items:center;gap:calc(var(--spacing) * 2.5);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);text-align:left;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex:1;display:flex}@media (hover:hover){.search-history-item-button:hover{color:var(--pb-text-primary)}}.search-history-item-button:hover{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.search-history-item-button:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.search-history-item-remove{padding-inline:calc(var(--spacing) * 2);padding-block:calc(var(--spacing) * 2);color:var(--pb-text-dim);opacity:0;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}@media (hover:hover){.search-history-item-remove:is(:where(.group):hover *){opacity:1}.search-history-item-remove:hover{color:var(--pb-error)}}.touch-target{min-width:44px;min-height:44px}.progress-track{height:var(--spacing);background-color:var(--pb-border);border-radius:3.40282e38px}.progress-fill{background-color:var(--pb-success);height:100%;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:3.40282e38px}.progress-fill-partial{background-color:var(--pb-interactive);height:100%;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:3.40282e38px}.callout-brand{align-items:center;gap:calc(var(--spacing) * 3);border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding:calc(var(--spacing) * 4);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background-color:var(--pb-brand-signal-dim);border-width:1px;border-color:var(--pb-brand-signal-border);display:flex}.callout-brand:hover{background-color:var(--pb-brand-signal-hover);border-color:var(--pb-brand-signal-border)}.callout-brand-icon{border-radius:var(--radius-lg);padding:calc(var(--spacing) * 2);background-color:var(--pb-brand-signal-dim);color:var(--pb-brand-signal);flex-shrink:0}.callout-brand-title{color:var(--pb-brand-signal)}.callout-brand-body{color:var(--pb-brand-signal);opacity:.7}.callout-brand-chevron{color:var(--pb-brand-signal);opacity:.5}}@layer utilities{.pointer-events-none{pointer-events:none}.visible{visibility:visible}.sr-only{clip-path:inset(50%);white-space:nowrap;border-width:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.absolute{position:absolute}.fixed{position:fixed}.relative{position:relative}.sticky{position:sticky}.inset-0{inset:0}.inset-y-0{inset-block:0}.-top-3{top:calc(var(--spacing) * -3)}.top-0{top:0}.top-1\/2{top:50%}.top-2{top:calc(var(--spacing) * 2)}.top-4{top:calc(var(--spacing) * 4)}.-right-1{right:calc(var(--spacing) * -1)}.-right-3{right:calc(var(--spacing) * -3)}.right-0{right:0}.right-2{right:calc(var(--spacing) * 2)}.right-4{right:calc(var(--spacing) * 4)}.-bottom-1{bottom:calc(var(--spacing) * -1)}.left-0{left:0}.left-3{left:calc(var(--spacing) * 3)}.z-10{z-index:10}.z-20{z-index:20}.z-30{z-index:30}.z-40{z-index:40}.z-50{z-index:50}.z-\[1\]{z-index:1}.z-\[60\]{z-index:60}.z-\[100\]{z-index:100}.z-\[110\]{z-index:110}.col-span-2{grid-column:span 2/span 2}.col-span-full{grid-column:1/-1}.container{width:100%}@media (min-width:40rem){.container{max-width:40rem}}@media (min-width:48rem){.container{max-width:48rem}}@media (min-width:64rem){.container{max-width:64rem}}@media (min-width:80rem){.container{max-width:80rem}}@media (min-width:96rem){.container{max-width:96rem}}.mx-1{margin-inline:var(--spacing)}.mx-auto{margin-inline:auto}.my-2{margin-block:calc(var(--spacing) * 2)}.\!mt-0{margin-top:0!important}.\!mt-2{margin-top:calc(var(--spacing) * 2)!important}.mt-0\.5{margin-top:calc(var(--spacing) * .5)}.mt-1{margin-top:var(--spacing)}.mt-1\.5{margin-top:calc(var(--spacing) * 1.5)}.mt-2{margin-top:calc(var(--spacing) * 2)}.mt-3{margin-top:calc(var(--spacing) * 3)}.mt-4{margin-top:calc(var(--spacing) * 4)}.mt-5{margin-top:calc(var(--spacing) * 5)}.mt-6{margin-top:calc(var(--spacing) * 6)}.mt-8{margin-top:calc(var(--spacing) * 8)}.mr-1{margin-right:var(--spacing)}.\!mb-0{margin-bottom:0!important}.mb-1{margin-bottom:var(--spacing)}.mb-1\.5{margin-bottom:calc(var(--spacing) * 1.5)}.mb-2{margin-bottom:calc(var(--spacing) * 2)}.mb-3{margin-bottom:calc(var(--spacing) * 3)}.mb-4{margin-bottom:calc(var(--spacing) * 4)}.mb-5{margin-bottom:calc(var(--spacing) * 5)}.-ml-2{margin-left:calc(var(--spacing) * -2)}.ml-0\.5{margin-left:calc(var(--spacing) * .5)}.ml-1{margin-left:var(--spacing)}.ml-2{margin-left:calc(var(--spacing) * 2)}.ml-4{margin-left:calc(var(--spacing) * 4)}.ml-\[140px\]{margin-left:140px}.ml-auto{margin-left:auto}.line-clamp-2{-webkit-line-clamp:2;-webkit-box-orient:vertical;display:-webkit-box;overflow:hidden}.block{display:block}.contents{display:contents}.flex{display:flex}.grid{display:grid}.hidden{display:none}.inline{display:inline}.inline-flex{display:inline-flex}.table{display:table}.table-cell{display:table-cell}.table-row{display:table-row}.aspect-\[2\/3\]{aspect-ratio:2/3}.h-1\.5{height:calc(var(--spacing) * 1.5)}.h-2{height:calc(var(--spacing) * 2)}.h-2\.5{height:calc(var(--spacing) * 2.5)}.h-3{height:calc(var(--spacing) * 3)}.h-3\.5{height:calc(var(--spacing) * 3.5)}.h-4{height:calc(var(--spacing) * 4)}.h-4\.5{height:calc(var(--spacing) * 4.5)}.h-5{height:calc(var(--spacing) * 5)}.h-6{height:calc(var(--spacing) * 6)}.h-7{height:calc(var(--spacing) * 7)}.h-8{height:calc(var(--spacing) * 8)}.h-9{height:calc(var(--spacing) * 9)}.h-10{height:calc(var(--spacing) * 10)}.h-11{height:calc(var(--spacing) * 11)}.h-12{height:calc(var(--spacing) * 12)}.h-14{height:calc(var(--spacing) * 14)}.h-16{height:calc(var(--spacing) * 16)}.h-32{height:calc(var(--spacing) * 32)}.h-96{height:calc(var(--spacing) * 96)}.h-\[18px\]{height:18px}.h-\[100dvh\]{height:100dvh}.h-full{height:100%}.max-h-64{max-height:calc(var(--spacing) * 64)}.max-h-72{max-height:calc(var(--spacing) * 72)}.max-h-80{max-height:calc(var(--spacing) * 80)}.max-h-\[28rem\]{max-height:28rem}.max-h-\[62vh\]{max-height:62vh}.max-h-\[70vh\]{max-height:70vh}.max-h-\[85vh\]{max-height:85vh}.max-h-\[90vh\]{max-height:90vh}.max-h-\[95vh\]{max-height:95vh}.max-h-\[calc\(100vh-5rem\)\]{max-height:calc(100vh - 5rem)}.\!min-h-8{min-height:calc(var(--spacing) * 8)!important}.\!min-h-10{min-height:calc(var(--spacing) * 10)!important}.min-h-0{min-height:0}.min-h-8{min-height:calc(var(--spacing) * 8)}.min-h-56{min-height:calc(var(--spacing) * 56)}.min-h-\[36px\]{min-height:36px}.min-h-\[200px\]{min-height:200px}.\!w-8{width:calc(var(--spacing) * 8)!important}.\!w-10{width:calc(var(--spacing) * 10)!important}.w-1\.5{width:calc(var(--spacing) * 1.5)}.w-1\/3{width:33.3333%}.w-2{width:calc(var(--spacing) * 2)}.w-2\.5{width:calc(var(--spacing) * 2.5)}.w-3{width:calc(var(--spacing) * 3)}.w-3\.5{width:calc(var(--spacing) * 3.5)}.w-4{width:calc(var(--spacing) * 4)}.w-4\.5{width:calc(var(--spacing) * 4.5)}.w-5{width:calc(var(--spacing) * 5)}.w-6{width:calc(var(--spacing) * 6)}.w-7{width:calc(var(--spacing) * 7)}.w-8{width:calc(var(--spacing) * 8)}.w-9{width:calc(var(--spacing) * 9)}.w-10{width:calc(var(--spacing) * 10)}.w-11{width:calc(var(--spacing) * 11)}.w-12{width:calc(var(--spacing) * 12)}.w-16{width:calc(var(--spacing) * 16)}.w-20{width:calc(var(--spacing) * 20)}.w-24{width:calc(var(--spacing) * 24)}.w-28{width:calc(var(--spacing) * 28)}.w-32{width:calc(var(--spacing) * 32)}.w-36{width:calc(var(--spacing) * 36)}.w-40{width:calc(var(--spacing) * 40)}.w-44{width:calc(var(--spacing) * 44)}.w-48{width:calc(var(--spacing) * 48)}.w-56{width:calc(var(--spacing) * 56)}.w-64{width:calc(var(--spacing) * 64)}.w-80{width:calc(var(--spacing) * 80)}.w-96{width:calc(var(--spacing) * 96)}.w-\[10\%\]{width:10%}.w-\[12\%\]{width:12%}.w-\[14\%\]{width:14%}.w-\[18px\]{width:18px}.w-\[22rem\]{width:22rem}.w-\[24\%\]{width:24%}.w-\[26\%\]{width:26%}.w-\[28px\]{width:28px}.w-\[28rem\]{width:28rem}.w-\[30rem\]{width:30rem}.w-\[32px\]{width:32px}.w-\[52px\]{width:52px}.w-\[60px\]{width:60px}.w-\[64px\]{width:64px}.w-\[68px\]{width:68px}.w-\[72px\]{width:72px}.w-\[76px\]{width:76px}.w-\[80px\]{width:80px}.w-\[82px\]{width:82px}.w-\[84px\]{width:84px}.w-\[88px\]{width:88px}.w-\[90px\]{width:90px}.w-\[92px\]{width:92px}.w-\[96px\]{width:96px}.w-\[98px\]{width:98px}.w-\[100px\]{width:100px}.w-\[104px\]{width:104px}.w-\[108px\]{width:108px}.w-\[110px\]{width:110px}.w-\[112px\]{width:112px}.w-\[116px\]{width:116px}.w-\[118px\]{width:118px}.w-\[120px\]{width:120px}.w-\[128px\]{width:128px}.w-\[136px\]{width:136px}.w-\[140px\]{width:140px}.w-\[148px\]{width:148px}.w-\[152px\]{width:152px}.w-\[156px\]{width:156px}.w-\[168px\]{width:168px}.w-\[180px\]{width:180px}.w-\[240px\]{width:240px}.w-auto{width:auto}.w-full{width:100%}.max-w-0{max-width:0}.max-w-2xl{max-width:var(--container-2xl)}.max-w-3xl{max-width:var(--container-3xl)}.max-w-4xl{max-width:var(--container-4xl)}.max-w-5xl{max-width:var(--container-5xl)}.max-w-\[16rem\]{max-width:16rem}.max-w-\[140px\]{max-width:140px}.max-w-\[150px\]{max-width:150px}.max-w-\[160px\]{max-width:160px}.max-w-\[220px\]{max-width:220px}.max-w-\[240px\]{max-width:240px}.max-w-\[260px\]{max-width:260px}.max-w-\[280px\]{max-width:280px}.max-w-\[300px\]{max-width:300px}.max-w-\[320px\]{max-width:320px}.max-w-\[calc\(100vw-2rem\)\]{max-width:calc(100vw - 2rem)}.max-w-full{max-width:100%}.max-w-lg{max-width:var(--container-lg)}.max-w-md{max-width:var(--container-md)}.max-w-none{max-width:none}.max-w-sm{max-width:var(--container-sm)}.max-w-xl{max-width:var(--container-xl)}.max-w-xs{max-width:var(--container-xs)}.min-w-0{min-width:0}.min-w-5{min-width:calc(var(--spacing) * 5)}.min-w-\[11\.5rem\]{min-width:11.5rem}.min-w-\[20px\]{min-width:20px}.min-w-\[60px\]{min-width:60px}.min-w-\[440px\]{min-width:440px}.min-w-\[620px\]{min-width:620px}.min-w-\[640px\]{min-width:640px}.min-w-\[720px\]{min-width:720px}.min-w-\[760px\]{min-width:760px}.min-w-\[860px\]{min-width:860px}.min-w-\[880px\]{min-width:880px}.min-w-\[980px\]{min-width:980px}.min-w-\[1040px\]{min-width:1040px}.flex-1{flex:1}.flex-shrink{flex-shrink:1}.shrink-0{flex-shrink:0}.table-fixed{table-layout:fixed}.-translate-x-full{--tw-translate-x:-100%;translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-x-0{--tw-translate-x:0px;translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-x-full{--tw-translate-x:100%;translate:var(--tw-translate-x) var(--tw-translate-y)}.-translate-y-1{--tw-translate-y:calc(var(--spacing) * -1);translate:var(--tw-translate-x) var(--tw-translate-y)}.-translate-y-1\/2{--tw-translate-y:calc(calc(1 / 2 * 100%) * -1);translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-y-0{--tw-translate-y:0px;translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-y-1{--tw-translate-y:var(--spacing);translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-y-2{--tw-translate-y:calc(var(--spacing) * 2);translate:var(--tw-translate-x) var(--tw-translate-y)}.scale-95{--tw-scale-x:95%;--tw-scale-y:95%;--tw-scale-z:95%;scale:var(--tw-scale-x) var(--tw-scale-y)}.scale-100{--tw-scale-x:100%;--tw-scale-y:100%;--tw-scale-z:100%;scale:var(--tw-scale-x) var(--tw-scale-y)}.rotate-90{rotate:90deg}.rotate-180{rotate:180deg}.animate-pulse{animation:var(--animate-pulse)}.animate-spin{animation:var(--animate-spin)}.cursor-default{cursor:default}.cursor-help{cursor:help}.cursor-not-allowed{cursor:not-allowed}.cursor-pointer{cursor:pointer}.cursor-wait{cursor:wait}.resize{resize:both}.list-disc{list-style-type:disc}.grid-cols-1{grid-template-columns:repeat(1,minmax(0,1fr))}.grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.grid-cols-\[1fr_auto\]{grid-template-columns:1fr auto}.flex-col{flex-direction:column}.flex-col-reverse{flex-direction:column-reverse}.flex-nowrap{flex-wrap:nowrap}.flex-wrap{flex-wrap:wrap}.items-baseline{align-items:baseline}.items-center{align-items:center}.items-end{align-items:flex-end}.items-start{align-items:flex-start}.justify-between{justify-content:space-between}.justify-center{justify-content:center}.justify-end{justify-content:flex-end}.justify-start{justify-content:flex-start}.gap-0{gap:0}.gap-0\.5{gap:calc(var(--spacing) * .5)}.gap-1{gap:var(--spacing)}.gap-1\.5{gap:calc(var(--spacing) * 1.5)}.gap-2{gap:calc(var(--spacing) * 2)}.gap-2\.5{gap:calc(var(--spacing) * 2.5)}.gap-3{gap:calc(var(--spacing) * 3)}.gap-4{gap:calc(var(--spacing) * 4)}.gap-5{gap:calc(var(--spacing) * 5)}.gap-6{gap:calc(var(--spacing) * 6)}.gap-8{gap:calc(var(--spacing) * 8)}:where(.space-y-0>:not(:last-child)){--tw-space-y-reverse:0;margin-block:0}:where(.space-y-0\.5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * .5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * .5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-1>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-1\.5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 1.5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 1.5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-2>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-3>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-4>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-6>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.gap-x-3{column-gap:calc(var(--spacing) * 3)}.gap-x-4{column-gap:calc(var(--spacing) * 4)}.gap-x-5{column-gap:calc(var(--spacing) * 5)}.gap-x-6{column-gap:calc(var(--spacing) * 6)}.gap-y-1{row-gap:var(--spacing)}.gap-y-1\.5{row-gap:calc(var(--spacing) * 1.5)}.gap-y-2{row-gap:calc(var(--spacing) * 2)}:where(.divide-y>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)))}:where(.divide-pb-border>:not(:last-child)){border-color:var(--pb-border)}:where(.divide-pb-border-subtle>:not(:last-child)){border-color:var(--pb-border-subtle)}:where(.divide-pb-border\/40>:not(:last-child)){border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){:where(.divide-pb-border\/40>:not(:last-child)){border-color:color-mix(in oklab, var(--pb-border) 40%, transparent)}}.self-start{align-self:flex-start}.truncate{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.overflow-auto{overflow:auto}.overflow-hidden{overflow:hidden}.overflow-visible{overflow:visible}.overflow-x-auto{overflow-x:auto}.overflow-x-hidden{overflow-x:hidden}.overflow-y-auto{overflow-y:auto}.overscroll-contain{overscroll-behavior:contain}.\!rounded-none{border-radius:0!important}.rounded{border-radius:.25rem}.rounded-2xl{border-radius:var(--radius-2xl)}.rounded-3xl{border-radius:var(--radius-3xl)}.rounded-\[var\(--radius-card\)\]{border-radius:var(--radius-card)}.rounded-full{border-radius:3.40282e38px}.rounded-lg{border-radius:var(--radius-lg)}.rounded-md{border-radius:var(--radius-md)}.rounded-sm{border-radius:var(--radius-sm)}.rounded-xl{border-radius:var(--radius-xl)}.rounded-t-xl{border-top-left-radius:var(--radius-xl);border-top-right-radius:var(--radius-xl)}.rounded-b-xl{border-bottom-right-radius:var(--radius-xl);border-bottom-left-radius:var(--radius-xl)}.\!border-0{border-style:var(--tw-border-style)!important;border-width:0!important}.border{border-style:var(--tw-border-style);border-width:1px}.border-0{border-style:var(--tw-border-style);border-width:0}.border-\[3px\]{border-style:var(--tw-border-style);border-width:3px}.border-t{border-top-style:var(--tw-border-style);border-top-width:1px}.border-b{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.border-l{border-left-style:var(--tw-border-style);border-left-width:1px}.border-l-2{border-left-style:var(--tw-border-style);border-left-width:2px}.border-dashed{--tw-border-style:dashed;border-style:dashed}.\!border-pb-warning\/35{border-color:var(--pb-warning)!important}@supports (color:color-mix(in lab, red, red)){.\!border-pb-warning\/35{border-color:color-mix(in oklab, var(--pb-warning) 35%, transparent)!important}}.border-current\/15{border-color:currentColor}@supports (color:color-mix(in lab, red, red)){.border-current\/15{border-color:color-mix(in oklab, currentcolor 15%, transparent)}}.border-pb-border{border-color:var(--pb-border)}.border-pb-border-hover,.border-pb-border-hover\/50{border-color:var(--pb-border-hover)}@supports (color:color-mix(in lab, red, red)){.border-pb-border-hover\/50{border-color:color-mix(in oklab, var(--pb-border-hover) 50%, transparent)}}.border-pb-border-subtle{border-color:var(--pb-border-subtle)}.border-pb-border\/50{border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.border-pb-border\/50{border-color:color-mix(in oklab, var(--pb-border) 50%, transparent)}}.border-pb-error,.border-pb-error\/30{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.border-pb-error\/30{border-color:color-mix(in oklab, var(--pb-error) 30%, transparent)}}.border-pb-error\/35{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.border-pb-error\/35{border-color:color-mix(in oklab, var(--pb-error) 35%, transparent)}}.border-pb-error\/40{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.border-pb-error\/40{border-color:color-mix(in oklab, var(--pb-error) 40%, transparent)}}.border-pb-info\/30{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.border-pb-info\/30{border-color:color-mix(in oklab, var(--pb-info) 30%, transparent)}}.border-pb-info\/40{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.border-pb-info\/40{border-color:color-mix(in oklab, var(--pb-info) 40%, transparent)}}.border-pb-interactive,.border-pb-interactive\/25{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.border-pb-interactive\/25{border-color:color-mix(in oklab, var(--pb-interactive) 25%, transparent)}}.border-pb-interactive\/30{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.border-pb-interactive\/30{border-color:color-mix(in oklab, var(--pb-interactive) 30%, transparent)}}.border-pb-success\/30{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.border-pb-success\/30{border-color:color-mix(in oklab, var(--pb-success) 30%, transparent)}}.border-pb-success\/35{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.border-pb-success\/35{border-color:color-mix(in oklab, var(--pb-success) 35%, transparent)}}.border-pb-success\/40{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.border-pb-success\/40{border-color:color-mix(in oklab, var(--pb-success) 40%, transparent)}}.border-pb-surface{border-color:var(--pb-bg-surface)}.border-pb-warning\/20{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/20{border-color:color-mix(in oklab, var(--pb-warning) 20%, transparent)}}.border-pb-warning\/25{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/25{border-color:color-mix(in oklab, var(--pb-warning) 25%, transparent)}}.border-pb-warning\/30{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/30{border-color:color-mix(in oklab, var(--pb-warning) 30%, transparent)}}.border-pb-warning\/40{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/40{border-color:color-mix(in oklab, var(--pb-warning) 40%, transparent)}}.border-transparent{border-color:#0000}.border-t-pb-interactive{border-top-color:var(--pb-interactive)}.\!bg-pb-warning-dim\/45{background-color:var(--pb-warning-dim)!important}@supports (color:color-mix(in lab, red, red)){.\!bg-pb-warning-dim\/45{background-color:color-mix(in oklab, var(--pb-warning-dim) 45%, transparent)!important}}.\!bg-transparent{background-color:#0000!important}.bg-black\/80{background-color:#000c}@supports (color:color-mix(in lab, red, red)){.bg-black\/80{background-color:color-mix(in oklab, var(--color-black) 80%, transparent)}}.bg-pb-base{background-color:var(--pb-bg-base)}.bg-pb-card{background-color:var(--pb-bg-card)}.bg-pb-card-hover,.bg-pb-card-hover\/20{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/20{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 20%, transparent)}}.bg-pb-card-hover\/35{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/35{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 35%, transparent)}}.bg-pb-card-hover\/40{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/40{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 40%, transparent)}}.bg-pb-card-hover\/45{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/45{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 45%, transparent)}}.bg-pb-card-hover\/50{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/50{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 50%, transparent)}}.bg-pb-card-hover\/60{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/60{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 60%, transparent)}}.bg-pb-card\/40{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/40{background-color:color-mix(in oklab, var(--pb-bg-card) 40%, transparent)}}.bg-pb-card\/50{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/50{background-color:color-mix(in oklab, var(--pb-bg-card) 50%, transparent)}}.bg-pb-card\/60{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/60{background-color:color-mix(in oklab, var(--pb-bg-card) 60%, transparent)}}.bg-pb-card\/70{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/70{background-color:color-mix(in oklab, var(--pb-bg-card) 70%, transparent)}}.bg-pb-card\/80{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/80{background-color:color-mix(in oklab, var(--pb-bg-card) 80%, transparent)}}.bg-pb-card\/85{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/85{background-color:color-mix(in oklab, var(--pb-bg-card) 85%, transparent)}}.bg-pb-card\/95{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/95{background-color:color-mix(in oklab, var(--pb-bg-card) 95%, transparent)}}.bg-pb-error{background-color:var(--pb-error)}.bg-pb-error-dim,.bg-pb-error-dim\/70{background-color:var(--pb-error-dim)}@supports (color:color-mix(in lab, red, red)){.bg-pb-error-dim\/70{background-color:color-mix(in oklab, var(--pb-error-dim) 70%, transparent)}}.bg-pb-error\/15{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.bg-pb-error\/15{background-color:color-mix(in oklab, var(--pb-error) 15%, transparent)}}.bg-pb-info{background-color:var(--pb-info)}.bg-pb-info-dim{background-color:var(--pb-info-dim)}.bg-pb-info\/10{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.bg-pb-info\/10{background-color:color-mix(in oklab, var(--pb-info) 10%, transparent)}}.bg-pb-input{background-color:var(--pb-bg-input)}.bg-pb-interactive{background-color:var(--pb-interactive)}.bg-pb-interactive-dim{background-color:var(--pb-interactive-dim)}.bg-pb-interactive\/15{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.bg-pb-interactive\/15{background-color:color-mix(in oklab, var(--pb-interactive) 15%, transparent)}}.bg-pb-overlay,.bg-pb-overlay\/45{background-color:var(--pb-bg-overlay)}@supports (color:color-mix(in lab, red, red)){.bg-pb-overlay\/45{background-color:color-mix(in oklab, var(--pb-bg-overlay) 45%, transparent)}}.bg-pb-purple-dim{background-color:var(--pb-purple-dim)}.bg-pb-success{background-color:var(--pb-success)}.bg-pb-success-dim,.bg-pb-success-dim\/70{background-color:var(--pb-success-dim)}@supports (color:color-mix(in lab, red, red)){.bg-pb-success-dim\/70{background-color:color-mix(in oklab, var(--pb-success-dim) 70%, transparent)}}.bg-pb-success\/15{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.bg-pb-success\/15{background-color:color-mix(in oklab, var(--pb-success) 15%, transparent)}}.bg-pb-surface,.bg-pb-surface\/40{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/40{background-color:color-mix(in oklab, var(--pb-bg-surface) 40%, transparent)}}.bg-pb-surface\/50{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/50{background-color:color-mix(in oklab, var(--pb-bg-surface) 50%, transparent)}}.bg-pb-surface\/60{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/60{background-color:color-mix(in oklab, var(--pb-bg-surface) 60%, transparent)}}.bg-pb-surface\/80{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/80{background-color:color-mix(in oklab, var(--pb-bg-surface) 80%, transparent)}}.bg-pb-text-dim{background-color:var(--pb-text-dim)}.bg-pb-warning{background-color:var(--pb-warning)}.bg-pb-warning-dim{background-color:var(--pb-warning-dim)}.bg-pb-warning\/5{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.bg-pb-warning\/5{background-color:color-mix(in oklab, var(--pb-warning) 5%, transparent)}}.bg-slate-950\/70{background-color:#020618b3}@supports (color:color-mix(in lab, red, red)){.bg-slate-950\/70{background-color:color-mix(in oklab, var(--color-slate-950) 70%, transparent)}}.bg-transparent{background-color:#0000}.object-contain{object-fit:contain}.object-cover{object-fit:cover}.object-top{object-position:top}.\!p-0{padding:0!important}.p-0{padding:0}.p-0\.5{padding:calc(var(--spacing) * .5)}.p-1{padding:var(--spacing)}.p-2{padding:calc(var(--spacing) * 2)}.p-2\.5{padding:calc(var(--spacing) * 2.5)}.p-3{padding:calc(var(--spacing) * 3)}.p-4{padding:calc(var(--spacing) * 4)}.p-5{padding:calc(var(--spacing) * 5)}.p-6{padding:calc(var(--spacing) * 6)}.p-8{padding:calc(var(--spacing) * 8)}.\!px-0{padding-inline:0!important}.\!px-2\.5{padding-inline:calc(var(--spacing) * 2.5)!important}.\!px-3{padding-inline:calc(var(--spacing) * 3)!important}.\!px-4{padding-inline:calc(var(--spacing) * 4)!important}.px-0{padding-inline:0}.px-1{padding-inline:var(--spacing)}.px-1\.5{padding-inline:calc(var(--spacing) * 1.5)}.px-2{padding-inline:calc(var(--spacing) * 2)}.px-2\.5{padding-inline:calc(var(--spacing) * 2.5)}.px-3{padding-inline:calc(var(--spacing) * 3)}.px-3\.5{padding-inline:calc(var(--spacing) * 3.5)}.px-4{padding-inline:calc(var(--spacing) * 4)}.px-5{padding-inline:calc(var(--spacing) * 5)}.px-6{padding-inline:calc(var(--spacing) * 6)}.\!py-0{padding-block:0!important}.\!py-1\.5{padding-block:calc(var(--spacing) * 1.5)!important}.\!py-2{padding-block:calc(var(--spacing) * 2)!important}.\!py-2\.5{padding-block:calc(var(--spacing) * 2.5)!important}.py-0\.5{padding-block:calc(var(--spacing) * .5)}.py-1{padding-block:var(--spacing)}.py-1\.5{padding-block:calc(var(--spacing) * 1.5)}.py-2{padding-block:calc(var(--spacing) * 2)}.py-2\.5{padding-block:calc(var(--spacing) * 2.5)}.py-3{padding-block:calc(var(--spacing) * 3)}.py-4{padding-block:calc(var(--spacing) * 4)}.py-5{padding-block:calc(var(--spacing) * 5)}.py-6{padding-block:calc(var(--spacing) * 6)}.py-8{padding-block:calc(var(--spacing) * 8)}.py-10{padding-block:calc(var(--spacing) * 10)}.py-12{padding-block:calc(var(--spacing) * 12)}.py-\[3px\]{padding-block:3px}.pt-0\.5{padding-top:calc(var(--spacing) * .5)}.pt-1{padding-top:var(--spacing)}.pt-1\.5{padding-top:calc(var(--spacing) * 1.5)}.pt-2{padding-top:calc(var(--spacing) * 2)}.pt-3{padding-top:calc(var(--spacing) * 3)}.pt-4{padding-top:calc(var(--spacing) * 4)}.pt-5{padding-top:calc(var(--spacing) * 5)}.pt-6{padding-top:calc(var(--spacing) * 6)}.pt-12{padding-top:calc(var(--spacing) * 12)}.pr-1{padding-right:var(--spacing)}.pr-3{padding-right:calc(var(--spacing) * 3)}.\!pb-0{padding-bottom:0!important}.pb-2{padding-bottom:calc(var(--spacing) * 2)}.pb-4{padding-bottom:calc(var(--spacing) * 4)}.pl-5{padding-left:calc(var(--spacing) * 5)}.pl-6{padding-left:calc(var(--spacing) * 6)}.\!text-left{text-align:left!important}.text-center{text-align:center}.text-left{text-align:left}.text-right{text-align:right}.align-top{vertical-align:top}.font-display{font-family:Bricolage Grotesque,sans-serif}.font-mono{font-family:JetBrains Mono,Fira Code,monospace}.font-sans{font-family:DM Sans,sans-serif}.text-2xl{font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height))}.text-base{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height))}.text-lg{font-size:var(--text-lg);line-height:var(--tw-leading,var(--text-lg--line-height))}.text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.text-xl{font-size:var(--text-xl);line-height:var(--tw-leading,var(--text-xl--line-height))}.text-xs{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height))}.text-\[0\.72rem\]{font-size:.72rem}.text-\[0\.78rem\]{font-size:.78rem}.text-\[0\.82rem\]{font-size:.82rem}.text-\[0\.88rem\]{font-size:.88rem}.text-\[9px\]{font-size:9px}.text-\[10px\]{font-size:10px}.text-\[11px\]{font-size:11px}.text-\[12px\]{font-size:12px}.text-\[13px\]{font-size:13px}.leading-4{--tw-leading:calc(var(--spacing) * 4);line-height:calc(var(--spacing) * 4)}.leading-5{--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5)}.leading-6{--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6)}.leading-7{--tw-leading:calc(var(--spacing) * 7);line-height:calc(var(--spacing) * 7)}.leading-\[1\.5\]{--tw-leading:1.5;line-height:1.5}.leading-\[1\.6\]{--tw-leading:1.6;line-height:1.6}.leading-none{--tw-leading:1;line-height:1}.leading-relaxed{--tw-leading:var(--leading-relaxed);line-height:var(--leading-relaxed)}.leading-tight{--tw-leading:var(--leading-tight);line-height:var(--leading-tight)}.font-black{--tw-font-weight:var(--font-weight-black);font-weight:var(--font-weight-black)}.font-bold{--tw-font-weight:var(--font-weight-bold);font-weight:var(--font-weight-bold)}.font-extrabold{--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold)}.font-medium{--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium)}.font-normal{--tw-font-weight:var(--font-weight-normal);font-weight:var(--font-weight-normal)}.font-semibold{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.tracking-\[0\.2em\]{--tw-tracking:.2em;letter-spacing:.2em}.tracking-\[0\.12em\]{--tw-tracking:.12em;letter-spacing:.12em}.tracking-\[0\.14em\]{--tw-tracking:.14em;letter-spacing:.14em}.tracking-\[0\.16em\]{--tw-tracking:.16em;letter-spacing:.16em}.tracking-\[0\.18em\]{--tw-tracking:.18em;letter-spacing:.18em}.tracking-\[0\.22em\]{--tw-tracking:.22em;letter-spacing:.22em}.tracking-tight{--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight)}.tracking-wide{--tw-tracking:var(--tracking-wide);letter-spacing:var(--tracking-wide)}.tracking-wider{--tw-tracking:var(--tracking-wider);letter-spacing:var(--tracking-wider)}.break-words{overflow-wrap:break-word}.break-all{word-break:break-all}.whitespace-normal{white-space:normal}.whitespace-nowrap{white-space:nowrap}.whitespace-pre-wrap{white-space:pre-wrap}.\!text-pb-text{color:var(--pb-text-primary)!important}.\!text-pb-warning{color:var(--pb-warning)!important}.text-pb-border{color:var(--pb-border)}.text-pb-brand{color:var(--pb-brand)}.text-pb-error{color:var(--pb-error)}.text-pb-info{color:var(--pb-info)}.text-pb-interactive{color:var(--pb-interactive)}.text-pb-purple{color:var(--pb-purple)}.text-pb-success{color:var(--pb-success)}.text-pb-text{color:var(--pb-text-primary)}.text-pb-text-dim{color:var(--pb-text-dim)}.text-pb-text-inverse{color:var(--pb-text-inverse)}.text-pb-text-sec{color:var(--pb-text-secondary)}.text-pb-warning,.text-pb-warning\/70{color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.text-pb-warning\/70{color:color-mix(in oklab, var(--pb-warning) 70%, transparent)}}.text-white{color:var(--color-white)}.capitalize{text-transform:capitalize}.lowercase{text-transform:lowercase}.uppercase{text-transform:uppercase}.italic{font-style:italic}.tabular-nums{--tw-numeric-spacing:tabular-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.no-underline{text-decoration-line:none}.underline{text-decoration-line:underline}.underline-offset-4{text-underline-offset:4px}.accent-pb-interactive{accent-color:var(--pb-interactive)}.opacity-0{opacity:0}.opacity-25{opacity:.25}.opacity-30{opacity:.3}.opacity-40{opacity:.4}.opacity-50{opacity:.5}.opacity-60{opacity:.6}.opacity-70{opacity:.7}.opacity-75{opacity:.75}.opacity-90{opacity:.9}.opacity-100{opacity:1}.shadow-2xl{--tw-shadow:0 25px 50px -12px var(--tw-shadow-color,#00000040);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-lg{--tw-shadow:0 10px 15px -3px var(--tw-shadow-color,#0000001a), 0 4px 6px -4px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-none{--tw-shadow:0 0 #0000;box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-sm{--tw-shadow:0 1px 3px 0 var(--tw-shadow-color,#0000001a), 0 1px 2px -1px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-xl{--tw-shadow:0 20px 25px -5px var(--tw-shadow-color,#0000001a), 0 8px 10px -6px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-xs{--tw-shadow:0 1px 2px 0 var(--tw-shadow-color,#0000000d);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.outline{outline-style:var(--tw-outline-style);outline-width:1px}.blur{--tw-blur:blur(8px);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}.filter{filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}.backdrop-blur-\[1px\]{--tw-backdrop-blur:blur(1px);-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.backdrop-blur-xs{--tw-backdrop-blur:blur(var(--blur-xs));-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.backdrop-filter{-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.transition{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to,opacity,box-shadow,transform,translate,scale,rotate,filter,-webkit-backdrop-filter,backdrop-filter,display,content-visibility,overlay,pointer-events;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-all{transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-colors{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-opacity{transition-property:opacity;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-transform{transition-property:transform,translate,scale,rotate;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-none{transition-property:none}.delay-75{transition-delay:75ms}.duration-75{--tw-duration:75ms;transition-duration:75ms}.duration-100{--tw-duration:.1s;transition-duration:.1s}.duration-150{--tw-duration:.15s;transition-duration:.15s}.duration-200{--tw-duration:.2s;transition-duration:.2s}.duration-300{--tw-duration:.3s;transition-duration:.3s}.duration-500{--tw-duration:.5s;transition-duration:.5s}.ease-in{--tw-ease:var(--ease-in);transition-timing-function:var(--ease-in)}.ease-linear{--tw-ease:linear;transition-timing-function:linear}.ease-out{--tw-ease:var(--ease-out);transition-timing-function:var(--ease-out)}.outline-none{--tw-outline-style:none;outline-style:none}.select-none{-webkit-user-select:none;user-select:none}@media (hover:hover){.group-hover\:border-pb-text-sec:is(:where(.group):hover *){border-color:var(--pb-text-secondary)}.group-hover\:text-pb-interactive:is(:where(.group):hover *){color:var(--pb-interactive)}.group-hover\:text-pb-text:is(:where(.group):hover *){color:var(--pb-text-primary)}.group-hover\:text-pb-text-sec:is(:where(.group):hover *){color:var(--pb-text-secondary)}.group-hover\:opacity-40:is(:where(.group):hover *){opacity:.4}}.placeholder\:text-pb-text-dim::placeholder{color:var(--pb-text-dim)}.placeholder\:text-pb-text-sec::placeholder{color:var(--pb-text-secondary)}.last\:mb-0:last-child{margin-bottom:0}.last\:border-b-0:last-child{border-bottom-style:var(--tw-border-style);border-bottom-width:0}@media (hover:hover){.hover\:\!border-pb-warning\/45:hover{border-color:var(--pb-warning)!important}@supports (color:color-mix(in lab, red, red)){.hover\:\!border-pb-warning\/45:hover{border-color:color-mix(in oklab, var(--pb-warning) 45%, transparent)!important}}.hover\:border-pb-border-hover:hover{border-color:var(--pb-border-hover)}.hover\:border-pb-border-strong:hover{border-color:var(--pb-border-strong)}.hover\:border-pb-error\/40:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-error\/40:hover{border-color:color-mix(in oklab, var(--pb-error) 40%, transparent)}}.hover\:border-pb-info\/40:hover{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-info\/40:hover{border-color:color-mix(in oklab, var(--pb-info) 40%, transparent)}}.hover\:border-pb-purple\/40:hover{border-color:var(--pb-purple)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-purple\/40:hover{border-color:color-mix(in oklab, var(--pb-purple) 40%, transparent)}}.hover\:border-pb-success\/40:hover{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-success\/40:hover{border-color:color-mix(in oklab, var(--pb-success) 40%, transparent)}}.hover\:border-pb-warning\/40:hover{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-warning\/40:hover{border-color:color-mix(in oklab, var(--pb-warning) 40%, transparent)}}.hover\:\!bg-pb-warning-dim:hover{background-color:var(--pb-warning-dim)!important}.hover\:\!bg-transparent:hover{background-color:#0000!important}.hover\:bg-pb-card:hover{background-color:var(--pb-bg-card)}.hover\:bg-pb-card-hover:hover,.hover\:bg-pb-card-hover\/20:hover{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-card-hover\/20:hover{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 20%, transparent)}}.hover\:bg-pb-card-hover\/45:hover{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-card-hover\/45:hover{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 45%, transparent)}}.hover\:bg-pb-error-dim:hover{background-color:var(--pb-error-dim)}.hover\:bg-pb-error\/25:hover{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-error\/25:hover{background-color:color-mix(in oklab, var(--pb-error) 25%, transparent)}}.hover\:bg-pb-info-dim:hover{background-color:var(--pb-info-dim)}.hover\:bg-pb-interactive-hover:hover{background-color:var(--pb-interactive-hover)}.hover\:bg-pb-interactive\/10:hover{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-interactive\/10:hover{background-color:color-mix(in oklab, var(--pb-interactive) 10%, transparent)}}.hover\:bg-pb-interactive\/25:hover{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-interactive\/25:hover{background-color:color-mix(in oklab, var(--pb-interactive) 25%, transparent)}}.hover\:bg-pb-purple-dim:hover{background-color:var(--pb-purple-dim)}.hover\:bg-pb-success-dim:hover{background-color:var(--pb-success-dim)}.hover\:bg-pb-success\/25:hover{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-success\/25:hover{background-color:color-mix(in oklab, var(--pb-success) 25%, transparent)}}.hover\:bg-pb-warning-dim:hover{background-color:var(--pb-warning-dim)}.hover\:\!text-pb-text:hover{color:var(--pb-text-primary)!important}.hover\:\!text-pb-warning:hover{color:var(--pb-warning)!important}.hover\:text-pb-error:hover{color:var(--pb-error)}.hover\:text-pb-interactive:hover{color:var(--pb-interactive)}.hover\:text-pb-interactive-hover:hover{color:var(--pb-interactive-hover)}.hover\:text-pb-text:hover{color:var(--pb-text-primary)}.hover\:text-pb-text-sec:hover{color:var(--pb-text-secondary)}.hover\:underline:hover{text-decoration-line:underline}.hover\:opacity-95:hover{opacity:.95}.hover\:opacity-100:hover{opacity:1}.hover\:brightness-90:hover{--tw-brightness:brightness(90%);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}}.focus\:border-transparent:focus{border-color:#0000}.focus\:ring-1:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(1px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.focus\:ring-2:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.focus\:ring-pb-border-hover:focus{--tw-ring-color:var(--pb-border-hover)}.focus\:ring-pb-error:focus{--tw-ring-color:var(--pb-error)}.focus\:ring-pb-interactive:focus{--tw-ring-color:var(--pb-interactive)}.focus\:ring-pb-warning:focus{--tw-ring-color:var(--pb-warning)}.focus\:ring-offset-0:focus{--tw-ring-offset-width:0px;--tw-ring-offset-shadow:var(--tw-ring-inset,) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color)}.focus\:ring-offset-2:focus{--tw-ring-offset-width:2px;--tw-ring-offset-shadow:var(--tw-ring-inset,) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color)}.focus\:ring-offset-pb-card:focus{--tw-ring-offset-color:var(--pb-bg-card)}.focus\:outline-hidden:focus{--tw-outline-style:none;outline-style:none}@media (forced-colors:active){.focus\:outline-hidden:focus{outline-offset:2px;outline:2px solid #0000}}.focus-visible\:ring-2:focus-visible{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.focus-visible\:ring-offset-2:focus-visible{--tw-ring-offset-width:2px;--tw-ring-offset-shadow:var(--tw-ring-inset,) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color)}.focus-visible\:outline-none:focus-visible{--tw-outline-style:none;outline-style:none}.disabled\:cursor-default:disabled{cursor:default}.disabled\:cursor-not-allowed:disabled{cursor:not-allowed}.disabled\:cursor-wait:disabled{cursor:wait}.disabled\:opacity-20:disabled{opacity:.2}.disabled\:opacity-30:disabled{opacity:.3}.disabled\:opacity-40:disabled{opacity:.4}.disabled\:opacity-50:disabled{opacity:.5}.disabled\:opacity-100:disabled{opacity:1}@media (min-width:40rem){.sm\:col-span-2{grid-column:span 2/span 2}.sm\:flex{display:flex}.sm\:hidden{display:none}.sm\:inline{display:inline}.sm\:table-cell{display:table-cell}.sm\:max-h-\[90vh\]{max-height:90vh}.sm\:w-14{width:calc(var(--spacing) * 14)}.sm\:w-44{width:calc(var(--spacing) * 44)}.sm\:w-48{width:calc(var(--spacing) * 48)}.sm\:w-56{width:calc(var(--spacing) * 56)}.sm\:w-auto{width:auto}.sm\:max-w-xs{max-width:var(--container-xs)}.sm\:min-w-\[760px\]{min-width:760px}.sm\:shrink-0{flex-shrink:0}.sm\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.sm\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.sm\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.sm\:grid-cols-\[8rem_minmax\(0\,1fr\)\]{grid-template-columns:8rem minmax(0,1fr)}.sm\:grid-cols-\[9rem_minmax\(0\,1fr\)\]{grid-template-columns:9rem minmax(0,1fr)}.sm\:grid-cols-\[minmax\(0\,1fr\)_minmax\(0\,1\.4fr\)_auto\]{grid-template-columns:minmax(0,1fr) minmax(0,1.4fr) auto}.sm\:grid-cols-\[minmax\(0\,3fr\)_minmax\(0\,2fr\)\]{grid-template-columns:minmax(0,3fr) minmax(0,2fr)}.sm\:flex-row{flex-direction:row}.sm\:items-center{align-items:center}.sm\:items-end{align-items:flex-end}.sm\:items-start{align-items:flex-start}.sm\:justify-between{justify-content:space-between}.sm\:justify-end{justify-content:flex-end}.sm\:self-auto{align-self:auto}.sm\:rounded-xl{border-radius:var(--radius-xl)}.sm\:p-4{padding:calc(var(--spacing) * 4)}.sm\:p-6{padding:calc(var(--spacing) * 6)}.sm\:p-7{padding:calc(var(--spacing) * 7)}.sm\:px-5{padding-inline:calc(var(--spacing) * 5)}.sm\:px-6{padding-inline:calc(var(--spacing) * 6)}.sm\:pt-5{padding-top:calc(var(--spacing) * 5)}.sm\:pt-6{padding-top:calc(var(--spacing) * 6)}.sm\:pt-7{padding-top:calc(var(--spacing) * 7)}.sm\:text-2xl{font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height))}}@media (min-width:48rem){.md\:flex{display:flex}.md\:table-cell{display:table-cell}.md\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.md\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.md\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.md\:grid-cols-\[minmax\(0\,1fr\)_auto_minmax\(0\,1fr\)\]{grid-template-columns:minmax(0,1fr) auto minmax(0,1fr)}.md\:flex-row{flex-direction:row}.md\:items-center{align-items:center}.md\:items-end{align-items:flex-end}.md\:items-start{align-items:flex-start}.md\:justify-between{justify-content:space-between}.md\:justify-center{justify-content:center}}@media (min-width:64rem){.lg\:flex{display:flex}.lg\:hidden{display:none}.lg\:table-cell{display:table-cell}.lg\:w-52{width:calc(var(--spacing) * 52)}.lg\:w-auto{width:auto}.lg\:max-w-\[11rem\]{max-width:11rem}.lg\:max-w-md{max-width:var(--container-md)}.lg\:min-w-\[920px\]{min-width:920px}.lg\:min-w-\[980px\]{min-width:980px}.lg\:min-w-\[1060px\]{min-width:1060px}.lg\:translate-x-0{--tw-translate-x:0px;translate:var(--tw-translate-x) var(--tw-translate-y)}.lg\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.lg\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.lg\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.lg\:grid-cols-\[minmax\(0\,1\.1fr\)_minmax\(280px\,0\.9fr\)\]{grid-template-columns:minmax(0,1.1fr) minmax(280px,.9fr)}.lg\:grid-cols-\[minmax\(0\,1\.15fr\)_minmax\(280px\,0\.85fr\)\]{grid-template-columns:minmax(0,1.15fr) minmax(280px,.85fr)}.lg\:grid-cols-\[minmax\(0\,1\.25fr\)_minmax\(280px\,0\.75fr\)\]{grid-template-columns:minmax(0,1.25fr) minmax(280px,.75fr)}.lg\:grid-cols-\[minmax\(0\,280px\)_minmax\(0\,1fr\)\]{grid-template-columns:minmax(0,280px) minmax(0,1fr)}.lg\:grid-cols-\[repeat\(5\,minmax\(0\,1fr\)\)\]{grid-template-columns:repeat(5,minmax(0,1fr))}.lg\:flex-col{flex-direction:column}.lg\:flex-row{flex-direction:row}.lg\:items-center{align-items:center}.lg\:items-end{align-items:flex-end}.lg\:items-start{align-items:flex-start}.lg\:justify-between{justify-content:space-between}.lg\:text-right{text-align:right}}@media (min-width:80rem){.xl\:col-span-3{grid-column:span 3/span 3}.xl\:w-32{width:calc(var(--spacing) * 32)}.xl\:w-48{width:calc(var(--spacing) * 48)}.xl\:max-w-sm{max-width:var(--container-sm)}.xl\:min-w-\[30rem\]{min-width:30rem}.xl\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.xl\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.xl\:grid-cols-5{grid-template-columns:repeat(5,minmax(0,1fr))}.xl\:grid-cols-7{grid-template-columns:repeat(7,minmax(0,1fr))}.xl\:grid-cols-\[minmax\(0\,1\.4fr\)_minmax\(320px\,0\.8fr\)\]{grid-template-columns:minmax(0,1.4fr) minmax(320px,.8fr)}.xl\:grid-cols-\[minmax\(0\,1\.45fr\)_minmax\(17rem\,1fr\)\]{grid-template-columns:minmax(0,1.45fr) minmax(17rem,1fr)}.xl\:grid-cols-\[minmax\(0\,1\.55fr\)_minmax\(320px\,0\.85fr\)\]{grid-template-columns:minmax(0,1.55fr) minmax(320px,.85fr)}.xl\:flex-row{flex-direction:row}.xl\:items-end{align-items:flex-end}.xl\:items-start{align-items:flex-start}.xl\:justify-between{justify-content:space-between}.xl\:justify-end{justify-content:flex-end}}@media (min-width:96rem){.\32 xl\:grid-cols-\[minmax\(0\,1\.45fr\)_minmax\(320px\,0\.75fr\)\]{grid-template-columns:minmax(0,1.45fr) minmax(320px,.75fr)}}}@font-face{font-family:Bricolage Grotesque;font-style:normal;font-weight:800;font-display:swap;src:url(/static/fonts/bricolage-grotesque-800.woff2)format("woff2")}@font-face{font-family:DM Sans;font-style:normal;font-weight:300 700;font-display:swap;src:url(/static/fonts/dm-sans-variable.woff2)format("woff2")}@font-face{font-family:Syne;font-style:normal;font-weight:400;font-display:swap;src:url(/static/fonts/syne-400.ttf)format("truetype")}@font-face{font-family:Syne;font-style:normal;font-weight:500;font-display:swap;src:url(/static/fonts/syne-500.ttf)format("truetype")}@font-face{font-family:Syne;font-style:normal;font-weight:600;font-display:swap;src:url(/static/fonts/syne-600.ttf)format("truetype")}@font-face{font-family:Syne;font-style:normal;font-weight:700;font-display:swap;src:url(/static/fonts/syne-700.ttf)format("truetype")}@font-face{font-family:Syne;font-style:normal;font-weight:800;font-display:swap;src:url(/static/fonts/syne-800.ttf)format("truetype")}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:400;font-display:swap;src:url(/static/fonts/jetbrains-mono-400.ttf)format("truetype")}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:500;font-display:swap;src:url(/static/fonts/jetbrains-mono-500.ttf)format("truetype")}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:600;font-display:swap;src:url(/static/fonts/jetbrains-mono-600.ttf)format("truetype")}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:700;font-display:swap;src:url(/static/fonts/jetbrains-mono-700.ttf)format("truetype")}.no-transitions,.no-transitions *,.no-transitions :before,.no-transitions :after,.boot-no-transitions,.boot-no-transitions *,.boot-no-transitions :before,.boot-no-transitions :after{transition-duration:0s!important}.safe-area-pb{padding-bottom:env(safe-area-inset-bottom,0px)}.skip-link{z-index:100;background:var(--pb-brand);color:var(--pb-text-inverse);border-radius:0 0 .5rem .5rem;padding:.5rem 1rem;font-size:.875rem;font-weight:600;text-decoration:none;transition:top .15s ease-out;position:absolute;top:-100%;left:1rem}.skip-link:focus{outline:2px solid var(--pb-interactive);outline-offset:2px;top:0}input[type=range].range-pb{appearance:none;background:var(--pb-border-hover);cursor:pointer;border-radius:9999px;outline:none;height:6px}input[type=range].range-pb::-webkit-slider-thumb{appearance:none;background:var(--pb-interactive);border:2px solid var(--pb-bg-card);width:18px;height:18px;box-shadow:var(--pb-shadow-1);cursor:pointer;border-radius:50%}input[type=range].range-pb::-moz-range-thumb{background:var(--pb-interactive);border:2px solid var(--pb-bg-card);width:18px;height:18px;box-shadow:var(--pb-shadow-1);cursor:pointer;border-radius:50%}.scrollbar-hidden{scrollbar-width:none;-ms-overflow-style:none}.scrollbar-hidden::-webkit-scrollbar{display:none}::-webkit-scrollbar{width:8px;height:8px}::-webkit-scrollbar-track{background:var(--pb-bg-surface)}::-webkit-scrollbar-thumb{background:var(--pb-border-strong);border-radius:4px}::-webkit-scrollbar-thumb:hover{background:var(--pb-text-dim)}*{scrollbar-width:thin;scrollbar-color:var(--pb-border-strong) var(--pb-bg-surface)}.htmx-indicator{opacity:0;transition:opacity .2s ease-in}.htmx-request .htmx-indicator,.htmx-request.htmx-indicator{opacity:1}#content[data-page-swap-phase=leaving],#content[data-page-swap-phase=entering]{pointer-events:none}#content[data-detail-history-hidden=true]{opacity:0;pointer-events:none}@media (prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,:before,:after{scroll-behavior:auto!important;transition-duration:.01ms!important;animation-duration:.01ms!important;animation-iteration-count:1!important}#content{transition:none}}@keyframes toast-enter{0%{opacity:0;transform:translate(100%)}to{opacity:1;transform:translate(0)}}@keyframes toast-exit{0%{opacity:1;transform:translate(0)}to{opacity:0;transform:translate(100%)}}.toast-enter{animation:.3s ease-out forwards toast-enter}.toast-exit{animation:.3s ease-in forwards toast-exit}.sidebar-transition{transition:width .2s ease-in-out,transform .2s ease-in-out}.app-sidebar-backdrop{background:var(--pb-bg-overlay)}@supports (color:color-mix(in lab, red, red)){.app-sidebar-backdrop{background:color-mix(in srgb, var(--pb-bg-overlay) 96%, transparent)}}.app-sidebar-shell{border-right:1px solid var(--pb-border);background:var(--pb-bg-surface);box-shadow:none;overflow:visible}.app-sidebar-brand{background:0 0;padding:1rem 1rem .75rem}.app-sidebar-brand-collapsed{background:0 0;justify-content:center;padding:1rem 0 .75rem;display:flex}.app-sidebar-brand-copy-collapsed{display:none!important}.app-sidebar-brand-card{align-items:center;gap:.75rem;padding:0;transition:color .18s,transform .18s;display:flex}.app-sidebar-brand-card-collapsed{justify-content:center;width:auto;min-width:0;min-height:0;padding:0}.app-sidebar-brand:hover .app-sidebar-brand-card{transform:translateY(-1px)}.app-sidebar-brand-collapsed:hover .app-sidebar-brand-card{transform:none}.app-sidebar-brand-mark{flex-shrink:0;justify-content:center;align-items:center;width:2.25rem;height:2.25rem;display:flex}.app-sidebar-brand-copy{white-space:nowrap;flex-direction:column;min-width:0;display:flex;overflow:hidden}.app-sidebar-brand-title{letter-spacing:.04em;text-transform:uppercase;color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:1.1rem;font-weight:800;line-height:1}.app-sidebar-brand-version{color:var(--pb-text-tertiary);margin-top:2px;font-family:JetBrains Mono,monospace;font-size:.58rem}.app-sidebar-nav{padding:.25rem .625rem 1rem}.app-sidebar-nav-collapsed{padding:.25rem 0 1rem;overflow:visible!important}.app-sidebar-link{color:var(--pb-text-secondary);border:0;border-radius:.5rem;align-items:center;gap:.625rem;margin:1px 0;padding:.4375rem .625rem;font-size:.82rem;font-weight:500;transition:background-color .18s,color .18s,transform .18s;display:flex;position:relative}.app-sidebar-link-collapsed{justify-content:center;gap:0;width:2.75rem;margin-inline:auto;padding:.5rem}.app-sidebar-link-copy-collapsed{display:none!important}.app-sidebar-link:hover{background:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.app-sidebar-link:hover{background:color-mix(in srgb, var(--pb-text-secondary) 4%, transparent)}}.app-sidebar-link:hover{color:var(--pb-text-primary)}.app-sidebar-link-collapsed:hover{transform:none}.app-sidebar-link-active{background:var(--pb-selected);color:var(--pb-interactive);font-weight:600}.app-sidebar-link-active:before{content:"";background:var(--pb-interactive);border-radius:0 2px 2px 0;width:3px;position:absolute;top:6px;bottom:6px;left:0}.app-sidebar-link-icon{width:1.25rem;height:1.25rem;color:inherit;flex-shrink:0;justify-content:center;align-items:center;transition:color .18s;display:flex}.app-sidebar-link-copy{flex:1;justify-content:space-between;align-items:center;gap:.75rem;min-width:0;display:flex}.app-sidebar-link-title{text-overflow:ellipsis;white-space:nowrap;min-width:0;font-size:.82rem;font-weight:inherit;overflow:hidden}.app-sidebar-badge-slot{flex-shrink:0;justify-content:flex-end;align-items:center;min-width:1.125rem;display:flex}.app-sidebar-badge-slot .count-badge{border-radius:6px;min-width:18px;min-height:auto;padding:1px 6px;font-family:JetBrains Mono,monospace;font-size:.6rem;font-weight:700;line-height:1.1}.app-sidebar-link-collapsed .app-sidebar-badge-slot{min-width:0;position:absolute;top:2px;right:2px}.app-sidebar-link-collapsed .app-sidebar-badge-slot .count-badge{border-radius:4px;min-width:14px;padding:0 4px;font-size:.52rem}.app-sidebar-section{margin:1rem 0 0}.app-sidebar-section-collapsed{width:3rem;margin:.75rem auto .375rem}.app-sidebar-section-label{letter-spacing:.14em;text-transform:uppercase;color:var(--pb-text-tertiary);align-items:center;gap:.375rem;margin-bottom:.25rem;padding:0 .5rem;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700;display:flex}.app-sidebar-section-label-collapsed{display:none!important}.app-sidebar-section-label:after{content:"";background:var(--pb-border-subtle);flex:auto;height:1px}.app-sidebar-section-rule{border-color:var(--pb-border-subtle);width:1.5rem;margin:.5rem auto 0}.app-sidebar-toggle{z-index:70;border:1px solid var(--pb-border-hover);background:var(--pb-bg-card);width:1.75rem;height:1.75rem;color:var(--pb-text-secondary);pointer-events:auto;box-shadow:var(--pb-shadow-1);border-radius:999px;justify-content:center;align-items:center;transition:border-color .18s,background-color .18s,color .18s,box-shadow .18s;position:absolute;top:72px;right:-.75rem}.app-sidebar-toggle[data-tip]{position:absolute}.app-sidebar-toggle:hover{border-color:var(--pb-interactive);background:var(--pb-bg-card);color:var(--pb-interactive);box-shadow:var(--pb-shadow-2)}[data-tip]{--pb-tooltip-max-width:min(360px, calc(100vw - 32px), calc(100dvw - 32px));position:relative}.tooltips-overlay-enabled [data-tip]:hover:after,.tooltips-overlay-enabled [data-tip]:focus-visible:after{content:none;display:none}[data-tip]:hover:after,[data-tip]:focus-visible:after{content:attr(data-tip);width:max-content;max-width:var(--pb-tooltip-max-width);background:var(--pb-bg-card);border:1px solid var(--pb-border-hover);color:var(--pb-text-primary);white-space:normal;overflow-wrap:break-word;box-shadow:var(--pb-shadow-overlay);z-index:50;pointer-events:none;border-radius:8px;padding:6px 10px;font-size:11px;line-height:1.4;animation:.15s ease-out tip-in;position:absolute;bottom:calc(100% + 6px);left:0}.app-tooltip-host{z-index:140;pointer-events:none;position:fixed;inset:0}.app-tooltip-overlay{width:max-content;max-width:var(--pb-tooltip-max-width,min(360px, calc(100vw - 32px), calc(100dvw - 32px)));background:var(--pb-bg-card);border:1px solid var(--pb-border-hover);color:var(--pb-text-primary);white-space:normal;overflow-wrap:break-word;box-shadow:var(--pb-shadow-overlay);opacity:0;visibility:hidden;will-change:left, top, opacity;border-radius:8px;padding:6px 10px;font-size:11px;line-height:1.4;transition:opacity .12s ease-out,visibility .12s ease-out;position:absolute;top:0;left:0;transform:translate(0)}.app-tooltip-host[data-visible=true] .app-tooltip-overlay{opacity:1;visibility:visible}[data-search-field-clear][data-tip]{position:absolute}[data-tip-pos=left]:hover:after,[data-tip-pos=left]:focus-visible:after{animation:.15s ease-out tip-in-left;inset:50% calc(100% + 6px) auto auto;transform:translateY(-50%)}[data-tip-pos=right]:hover:after,[data-tip-pos=right]:focus-visible:after{animation:.15s ease-out tip-in-right;inset:50% auto auto calc(100% + 6px);transform:translateY(-50%)}[data-tip-pos=bottom]:hover:after,[data-tip-pos=bottom]:focus-visible:after{animation:.15s ease-out tip-in-bottom;top:calc(100% + 6px);bottom:auto;left:0}[data-tip-size=wide]:hover:after,[data-tip-size=wide]:focus-visible:after{max-width:var(--pb-tooltip-max-width,min(480px, calc(100vw - 32px), calc(100dvw - 32px)))}[data-tip-size=narrow]:hover:after,[data-tip-size=narrow]:focus-visible:after{max-width:var(--pb-tooltip-max-width,min(220px, calc(100vw - 32px), calc(100dvw - 32px)))}.app-header-icon-tip:hover:after,.app-header-icon-tip:focus-visible:after{text-align:center;min-width:7.25rem}.tooltip-wrap{position:relative}.tooltip-panel{z-index:50;visibility:hidden;opacity:0;pointer-events:none;background:var(--pb-bg-card);border:1px solid var(--pb-border-hover);width:max-content;max-width:360px;color:var(--pb-text-primary);white-space:normal;overflow-wrap:break-word;box-shadow:var(--pb-shadow-overlay);border-radius:.5rem;padding:.5rem .75rem;font-size:.75rem;line-height:1.4;transition:opacity .15s ease-out,transform .15s ease-out,visibility .15s ease-out;position:absolute}.tooltip-wrap:hover>.tooltip-panel,.tooltip-wrap:focus-within>.tooltip-panel{visibility:visible;opacity:1}.tooltip-panel-top{bottom:calc(100% + .5rem);left:0;transform:translateY(4px)}.tooltip-wrap:hover>.tooltip-panel-top,.tooltip-wrap:focus-within>.tooltip-panel-top{transform:translateY(0)}.tooltip-panel-left{top:50%;right:calc(100% + .5rem);transform:translate(4px,-50%)}.tooltip-wrap:hover>.tooltip-panel-left,.tooltip-wrap:focus-within>.tooltip-panel-left{transform:translateY(-50%)}.tooltip-panel-right{top:50%;left:calc(100% + .5rem);transform:translate(-4px,-50%)}.tooltip-wrap:hover>.tooltip-panel-right,.tooltip-wrap:focus-within>.tooltip-panel-right{transform:translateY(-50%)}.tooltip-panel-bottom{top:calc(100% + .5rem);left:0;transform:translateY(-4px)}.tooltip-wrap:hover>.tooltip-panel-bottom,.tooltip-wrap:focus-within>.tooltip-panel-bottom{transform:translateY(0)}.tooltip-panel-nowrap{white-space:nowrap}.tooltip-panel-wide{max-width:500px}@keyframes tip-in{0%{opacity:0;transform:translateY(4px)}to{opacity:1;transform:translateY(0)}}@keyframes tip-in-left{0%{opacity:0;transform:translate(4px,-50%)}to{opacity:1;transform:translateY(-50%)}}@keyframes tip-in-right{0%{opacity:0;transform:translate(-4px,-50%)}to{opacity:1;transform:translateY(-50%)}}@keyframes tip-in-bottom{0%{opacity:0;transform:translateY(-4px)}to{opacity:1;transform:translateY(0)}}.log-terminal{contain:inline-size;background:var(--pb-bg-surface);width:100%;min-width:0;max-width:100%;overflow-x:hidden}@supports (color:color-mix(in lab, red, red)){.log-terminal{background:color-mix(in srgb, var(--pb-bg-surface) 82%, var(--pb-bg-base))}}.log-terminal{background-image:repeating-linear-gradient(0deg, transparent, transparent 2px, var(--pb-border-subtle) 2px, var(--pb-border-subtle) 4px)}@supports (color:color-mix(in lab, red, red)){.log-terminal{background-image:repeating-linear-gradient(0deg, transparent, transparent 2px, color-mix(in srgb, var(--pb-border-subtle) 32%, transparent) 2px, color-mix(in srgb, var(--pb-border-subtle) 32%, transparent) 4px)}}.log-line{box-sizing:border-box;border-left:3px solid #0000;width:100%;min-width:0;max-width:100%;overflow:hidden}.log-line:hover{background:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.log-line:hover{background:color-mix(in srgb, var(--pb-bg-card-hover) 36%, transparent)}}.log-line[data-level=error],.log-line[data-level=critical]{border-left-color:var(--pb-error);background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.log-line[data-level=error],.log-line[data-level=critical]{background:color-mix(in srgb, var(--pb-error) 12%, transparent)}}.log-line[data-level=warning]{border-left-color:var(--pb-warning)}.log-line[data-level=info]{border-left-color:var(--pb-info)}.log-line[data-level=debug]{border-left-color:var(--pb-text-dim)}.log-line.expanded{background:var(--pb-bg-card-hover)!important}@supports (color:color-mix(in lab, red, red)){.log-line.expanded{background:color-mix(in srgb, var(--pb-bg-card-hover) 54%, transparent)!important}}.log-detail{box-sizing:border-box;max-width:calc(100% - 140px);overflow:hidden}.log-path{overflow-wrap:anywhere}.log-badge{letter-spacing:.08em;text-transform:uppercase;text-align:center;border-radius:3px;min-width:38px;padding:1px 5px;font-size:9px;font-weight:700;display:inline-block}.badge-debug{color:var(--pb-text-dim);background:var(--pb-text-dim)}@supports (color:color-mix(in lab, red, red)){.badge-debug{background:color-mix(in srgb, var(--pb-text-dim) 16%, transparent)}}.badge-info{color:var(--pb-info);background:var(--pb-info-dim)}.badge-warning{color:var(--pb-warning);background:var(--pb-warning-dim)}.badge-error{color:var(--pb-error);background:var(--pb-error-dim)}.badge-critical{color:var(--pb-text-inverse);background:var(--pb-error)}.live-dot{background:var(--pb-success);border-radius:50%;width:6px;height:6px;animation:2s ease-in-out infinite live-pulse}@keyframes live-pulse{0%,to{opacity:1;box-shadow:0 0 0 0 color-mix(in srgb, var(--pb-success) 42%, transparent)}50%{opacity:.6;box-shadow:0 0 6px 2px color-mix(in srgb, var(--pb-success) 24%, transparent)}}.json-key{color:var(--pb-info)}.json-str{color:var(--pb-success)}.json-num{color:var(--pb-warning)}.json-bool{color:var(--pb-purple)}.json-null{color:var(--pb-text-dim)}.downloads-page-shell{width:100%}.downloads-view{width:100%;max-width:none;padding:0 0 var(--pb-page-footer-clearance);margin:0}.downloads-header{display:block}.downloads-header-main{flex-wrap:wrap;justify-content:space-between;align-items:flex-start;gap:1.25rem;width:100%;display:flex}.downloads-header-summary{flex-wrap:wrap;flex:auto;align-items:flex-start;gap:1.5rem;min-width:0;display:flex}.downloads-header-copy{min-width:0}.downloads-header-actions{flex:none;justify-content:flex-end;align-items:flex-start;margin-left:auto;display:flex}.downloads-title{letter-spacing:.04em;text-transform:uppercase;color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:1.6rem;font-weight:800;line-height:1}.downloads-title span{color:var(--pb-brand)}.downloads-subtitle{color:var(--pb-text-tertiary);letter-spacing:.01em;margin-top:.25rem;font-size:.78rem}.downloads-gauges{align-items:flex-end;gap:1rem;display:flex}.downloads-gauge{text-align:center}.downloads-gauge-ring{width:56px;height:56px;margin:0 auto;position:relative}.downloads-gauge-ring svg{transform:rotate(-90deg)}.downloads-gauge-bg{fill:none;stroke:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.downloads-gauge-bg{stroke:color-mix(in srgb, var(--pb-text-secondary) 10%, transparent)}}.downloads-gauge-bg{stroke-width:4.5px}.downloads-gauge-fill{fill:none;stroke-width:4.5px;stroke-linecap:round;transition:stroke-dashoffset .6s}.downloads-gauge-fill-success{stroke:var(--pb-status-success)}.downloads-gauge-fill-info{stroke:var(--pb-status-info)}.downloads-gauge-fill-warning{stroke:var(--pb-status-warning)}.downloads-gauge-fill-error{stroke:var(--pb-status-danger)}.downloads-gauge-fill-muted{stroke:var(--pb-text-dim)}.downloads-gauge-value{color:var(--pb-text-primary);justify-content:center;align-items:center;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:700;display:flex;position:absolute;inset:0}.downloads-gauge-value-success{color:var(--pb-status-success)}.downloads-gauge-value-info{color:var(--pb-status-info)}.downloads-gauge-value-warning{color:var(--pb-status-warning)}.downloads-gauge-value-error{color:var(--pb-status-danger)}.downloads-gauge-value-muted{color:var(--pb-text-dim)}.downloads-gauge-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);margin-top:.25rem;font-size:.6rem;font-weight:600}.downloads-tab-rail{border:1px solid var(--pb-border);background:0 0;border-radius:10px;align-self:flex-start;display:inline-flex;overflow:hidden}.downloads-tab-btn{letter-spacing:.06em;text-transform:uppercase;color:var(--pb-text-dim);background:0 0;border:0;padding:.5rem 1.25rem;font-family:Syne,sans-serif;font-size:.78rem;font-weight:700;transition:background-color .14s,color .14s}.downloads-tab-btn+.downloads-tab-btn{border-left:1px solid var(--pb-border)}.downloads-tab-btn:hover{color:var(--pb-text);background:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.downloads-tab-btn:hover{background:color-mix(in srgb, var(--pb-text-secondary) 4%, transparent)}}.downloads-tab-btn.is-active{background:var(--pb-interactive);color:var(--pb-text-inverse)}.downloads-panel-stack{gap:1.25rem;margin-top:.875rem;display:grid}.import-history-page-shell{margin-top:0}.downloads-section{gap:.625rem;display:grid}.downloads-section-label{border-bottom:1px solid var(--pb-border-subtle);letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);padding-bottom:.375rem;font-family:Syne,sans-serif;font-size:.62rem;font-weight:700}.downloads-table-wrap{border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);border-radius:14px;position:relative;overflow:visible;box-shadow:0 1px 3px #1e1a170f,0 1px 2px #1e1a170a}.downloads-table-wrap.is-clipped{overflow:hidden}.downloads-table{border-collapse:collapse;width:100%}.downloads-table th{border-bottom:2px solid var(--pb-border);background:var(--pb-bg-surface);text-align:left;letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);padding:.5625rem .875rem;font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.downloads-table td{border-bottom:1px solid var(--pb-border-subtle);color:var(--pb-text);vertical-align:middle;padding:.6875rem .875rem;font-size:.82rem;transition:background-color .1s}.downloads-table tbody:last-child tr:last-child td{border-bottom:0}.downloads-table tr:not(.table-detail-row):hover td{background:var(--pb-surface-selected)}.downloads-table th.is-center,.downloads-table td.is-center{text-align:center}.downloads-table th.is-right,.downloads-table td.is-right{text-align:right}.issue-search-results-table-wrap{max-height:62vh;overflow-y:auto}.issue-search-results-table thead th{z-index:1;position:sticky;top:0}.issue-search-sort-indicator{color:var(--pb-interactive)}.issue-search-release-link{color:var(--pb-text);transition:color .14s}.issue-search-release-link:hover{color:var(--pb-interactive)}.issue-search-release-title{text-overflow:ellipsis;white-space:nowrap;max-width:320px;color:inherit;font-weight:600;display:block;overflow:hidden}.issue-search-release-title-muted{color:var(--pb-text-dim)}.issue-search-release-match{color:var(--pb-text-dim);margin-top:.125rem;font-size:.64rem}.issue-search-action-row{justify-content:flex-end;align-items:center;gap:.25rem;display:inline-flex}.issue-search-rejected-row td{opacity:.45}.issue-search-rejected-label{color:var(--pb-text-dim);font-family:JetBrains Mono,monospace;font-size:.64rem}.issue-search-empty-state{text-align:center;padding:2.5rem 1.25rem}.issue-search-empty-icon{width:2.25rem;height:2.25rem;color:var(--pb-text-dim);margin-inline:auto}.issue-search-empty-title{color:var(--pb-text);margin-top:.625rem;font-size:.85rem;font-weight:600}.issue-search-empty-copy{color:var(--pb-text-sec);margin-top:.25rem;font-size:.78rem}.downloads-release-name{max-width:260px;color:var(--pb-text);font-weight:600;display:block}.downloads-release-meta{color:var(--pb-text-dim);margin-top:.2rem;font-size:.72rem}.downloads-issue-link{color:var(--pb-interactive);transition:color .14s}.downloads-issue-link:hover{color:var(--pb-interactive-hover)}.table-mono-dim,td.table-mono-dim,.downloads-mono-dim-cell,td.downloads-mono-dim-cell{font-family:JetBrains Mono,monospace;font-size:.72rem}.table-mono-dim{color:var(--pb-text-dim)}.downloads-table td.downloads-mono-cell,.downloads-table td.downloads-mono-dim-cell,.downloads-table .downloads-mono-cell,.downloads-table .downloads-mono-dim-cell{font-variant-numeric:tabular-nums;font-family:JetBrains Mono,monospace}.downloads-table td.downloads-mono-cell,.downloads-table .downloads-mono-cell{color:var(--pb-text-secondary);font-size:.75rem;font-weight:400;line-height:1.2}.downloads-table td.downloads-mono-dim-cell,.downloads-table .downloads-mono-dim-cell{color:var(--pb-text-dim);font-size:.72rem}.downloads-muted-text{color:var(--pb-text-dim)}.downloads-progress-cell{align-items:center;gap:.375rem;display:inline-flex}.downloads-progress-track{border:1px solid var(--pb-border-subtle);background:var(--pb-text-secondary);border-radius:2px;width:100px;height:6px;display:inline-block;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.downloads-progress-track{background:color-mix(in srgb, var(--pb-text-secondary) 6%, transparent)}}.downloads-progress-track{vertical-align:middle}.downloads-progress-fill{height:100%;display:block;position:relative}.downloads-progress-fill.is-blue{background:linear-gradient(90deg, var(--pb-info), var(--pb-info))}@supports (color:color-mix(in lab, red, red)){.downloads-progress-fill.is-blue{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-info) 45%, transparent), var(--pb-info))}}.downloads-progress-fill.is-amber{background:linear-gradient(90deg, var(--pb-warning), var(--pb-warning))}@supports (color:color-mix(in lab, red, red)){.downloads-progress-fill.is-amber{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-warning) 45%, transparent), var(--pb-warning))}}.downloads-progress-fill.is-green{background:linear-gradient(90deg, var(--pb-success), var(--pb-success))}@supports (color:color-mix(in lab, red, red)){.downloads-progress-fill.is-green{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-success) 45%, transparent), var(--pb-success))}}.downloads-progress-fill:after{content:"";background:repeating-linear-gradient(90deg,#0000 0 2px,#ffffff1f 2px 4px);position:absolute;inset:0}@keyframes downloads-progress-indeterminate{0%{transform:translate(-130%)}to{transform:translate(280%)}}.downloads-progress-fill.is-indeterminate{will-change:transform;animation:1.25s ease-in-out infinite downloads-progress-indeterminate}@media (prefers-reduced-motion:reduce){.downloads-progress-fill.is-indeterminate{opacity:.7;animation:none;transform:translate(60%)}}.downloads-progress-pct{color:var(--pb-info);font-family:JetBrains Mono,monospace;font-size:.68rem;font-weight:600}.downloads-progress-pct-warning{color:var(--pb-warning)}.downloads-progress-pct-success{color:var(--pb-success)}.downloads-progress-pct-muted{color:var(--pb-text-dim)}.downloads-led{border-radius:999px;width:8px;height:8px;display:inline-block}.downloads-led-green{background:var(--pb-success);box-shadow:0 0 5px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.downloads-led-green{box-shadow:0 0 5px color-mix(in srgb, var(--pb-success) 35%, transparent)}}.downloads-led-blue{background:var(--pb-info);box-shadow:0 0 5px var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.downloads-led-blue{box-shadow:0 0 5px color-mix(in srgb, var(--pb-info) 35%, transparent)}}.downloads-led-amber{background:var(--pb-warning);box-shadow:0 0 5px var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.downloads-led-amber{box-shadow:0 0 5px color-mix(in srgb, var(--pb-warning) 35%, transparent)}}.downloads-led-off{background:var(--pb-text-dim);opacity:.35}.downloads-action-group{align-items:center;gap:.25rem;display:inline-flex}@media (hover:hover) and (pointer:fine){.downloads-action-group.is-hover-reveal{opacity:0;pointer-events:none;transition:opacity .14s,transform .14s;transform:translateY(2px)}.downloads-table tbody tr:hover .downloads-action-group.is-hover-reveal,.downloads-table tbody tr:focus-within .downloads-action-group.is-hover-reveal{opacity:1;pointer-events:auto;transform:translateY(0)}}.downloads-action-btn{border:1px solid var(--pb-border-subtle);width:28px;height:28px;color:var(--pb-text-dim);background:0 0;border-radius:6px;justify-content:center;align-items:center;transition:border-color .14s,color .14s,background-color .14s;display:inline-flex}.downloads-action-btn svg{width:12px;height:12px}.import-history-action-group{gap:.375rem}.import-history-action-btn{border-radius:8px;flex-shrink:0;width:30px;height:30px}.downloads-action-btn .import-history-action-icon{shape-rendering:geometricprecision;flex-shrink:0;width:12px;height:12px;overflow:visible}.downloads-action-btn .import-history-action-icon--detail{width:14px;height:14px}.downloads-action-btn:hover{border-color:var(--pb-interactive);background:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.downloads-action-btn:hover{background:color-mix(in srgb, var(--pb-info) 10%, transparent)}}.downloads-action-btn:hover{color:var(--pb-interactive)}.downloads-action-btn.is-danger:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.downloads-action-btn.is-danger:hover{border-color:color-mix(in srgb, var(--pb-error) 40%, transparent)}}.downloads-action-btn.is-danger:hover{background:var(--pb-error-dim);color:var(--pb-error)}.downloads-action-btn.is-warn:hover{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.downloads-action-btn.is-warn:hover{border-color:color-mix(in srgb, var(--pb-warning) 40%, transparent)}}.downloads-action-btn.is-warn:hover{background:var(--pb-warning-dim);color:var(--pb-warning)}.downloads-empty-state{text-align:center;padding:2.5rem 1.5rem}.downloads-empty-state.is-compact{padding:1.5rem 1.25rem}.downloads-empty-state.is-history{padding:2rem 1.5rem}.downloads-empty-icon{width:40px;height:40px;color:var(--pb-text-dim);margin:0 auto}.downloads-empty-title{letter-spacing:.04em;text-transform:uppercase;color:var(--pb-text);margin-top:.75rem;font-family:Syne,sans-serif;font-size:.88rem;font-weight:700}.downloads-empty-copy{color:var(--pb-text-dim);margin-top:.4rem;font-size:.78rem}.intervention-bulk-bar{border-top:1px solid var(--pb-border-subtle);border-bottom:1px solid var(--pb-border-subtle);background:var(--pb-info-bg);flex-wrap:wrap;justify-content:space-between;align-items:center;gap:.75rem;padding:.625rem .875rem;display:flex}@supports (color:color-mix(in lab, red, red)){.intervention-bulk-bar{background:color-mix(in srgb, var(--pb-info-bg) 40%, var(--pb-bg-surface))}}.intervention-bulk-left{flex-wrap:wrap;align-items:center;gap:.625rem;display:inline-flex}.intervention-bulk-count{color:var(--pb-text-dim);font-family:JetBrains Mono,monospace;font-size:.72rem}.intervention-bulk-actions{flex-wrap:wrap;align-items:center;gap:.5rem;display:inline-flex}.downloads-footer-strip{border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);color:var(--pb-text-dim);border-radius:10px;flex-wrap:wrap;align-items:center;gap:.375rem 1.5rem;margin-top:1.25rem;padding:.625rem 1rem;font-family:JetBrains Mono,monospace;font-size:.7rem;display:flex}.downloads-footer-strip strong{color:var(--pb-text);font-weight:600}.downloads-history-toolbar{z-index:2;border:0;border-bottom:1px solid var(--pb-border);border-top-left-radius:inherit;border-top-right-radius:inherit;background:var(--pb-bg-surface);box-shadow:none;border-bottom-right-radius:0;border-bottom-left-radius:0;position:relative}.downloads-toolbar{border-bottom:1px solid var(--pb-border);background:var(--pb-bg-surface);flex-wrap:wrap;justify-content:space-between;align-items:center;gap:.75rem;padding:.625rem .875rem;display:flex}.downloads-history-toolbar>.downloads-toolbar{border-top-left-radius:inherit;border-top-right-radius:inherit;border-bottom-right-radius:0;border-bottom-left-radius:0}.downloads-toolbar-left{flex-wrap:wrap;align-items:center;gap:.5rem;display:flex}.downloads-toolbar-search,.downloads-toolbar-select{border:1px solid var(--pb-border);background:var(--pb-bg-card);height:30px;color:var(--pb-text);border-radius:8px;outline:none;font-size:.78rem}.downloads-toolbar-search{width:200px;padding:0 .625rem}.downloads-toolbar-select{appearance:none;background-image:url("data:image/svg+xml,%3Csvg width='10' height='6' viewBox='0 0 10 6' fill='none' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M1 1L5 5L9 1' stroke='%2374675B' stroke-width='1.5' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E");background-position:right 8px center;background-repeat:no-repeat;padding:0 2rem 0 .625rem}[data-theme=dark] .downloads-toolbar-select{background-image:url("data:image/svg+xml,%3Csvg width='10' height='6' viewBox='0 0 10 6' fill='none' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M1 1L5 5L9 1' stroke='%239AAABA' stroke-width='1.5' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E")}.downloads-toolbar-search:focus,.downloads-toolbar-select:focus{border-color:var(--pb-interactive)}.downloads-clear-btn{background:var(--pb-error);color:var(--pb-text-inverse);white-space:nowrap;border:0;border-radius:8px;align-items:center;gap:.3125rem;padding:.375rem .75rem;font-size:.75rem;font-weight:600;transition:opacity .14s;display:inline-flex}.downloads-clear-btn svg{width:12px;height:12px}.downloads-clear-btn:hover{opacity:.92}.downloads-sort-btn{width:100%;color:inherit;font-family:inherit;font-size:inherit;font-weight:inherit;letter-spacing:inherit;line-height:inherit;text-align:inherit;text-transform:inherit;justify-content:flex-start;align-items:center;gap:.25rem;transition:color .14s;display:inline-flex}.import-review-cv-year-header{width:5.75rem;min-width:5.75rem}.import-review-cv-year-header .downloads-sort-btn{white-space:nowrap}.import-review-cv-year-header .downloads-sort-chevron{flex-shrink:0}.downloads-table th.is-center .downloads-sort-btn{text-align:center;justify-content:center}.downloads-table th.is-right .downloads-sort-btn{text-align:right;justify-content:flex-end}.downloads-sort-btn:hover,.downloads-sort-btn.is-active{color:var(--pb-text)}.downloads-sort-chevron{opacity:.35;width:12px;height:12px}.downloads-sort-btn.is-active .downloads-sort-chevron{opacity:1}.downloads-sort-chevron.is-desc{transform:rotate(180deg)}.downloads-error-row td{border-bottom:1px solid var(--pb-border-subtle);background:var(--pb-error-dim);padding:.5rem .875rem}@supports (color:color-mix(in lab, red, red)){.downloads-error-row td{background:color-mix(in srgb, var(--pb-error-dim) 50%, var(--pb-bg-card))}}.downloads-error-row td{color:var(--pb-error);font-size:.75rem}.downloads-error-content{align-items:flex-start;gap:.5rem;display:flex}.downloads-error-content svg{flex-shrink:0;width:14px;height:14px;margin-top:1px}.search-history-detail-row td{border-bottom:1px solid var(--pb-border-subtle);background:var(--pb-bg-surface);padding:.75rem .875rem}@supports (color:color-mix(in lab, red, red)){.search-history-detail-row td{background:color-mix(in srgb, var(--pb-bg-surface) 70%, var(--pb-bg-card))}}.search-history-detail-card{border:1px solid var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.search-history-detail-card{border:1px solid color-mix(in srgb, var(--pb-border) 78%, transparent)}}.search-history-detail-card{background:var(--pb-bg-card-hover);border-radius:.75rem}@supports (color:color-mix(in lab, red, red)){.search-history-detail-card{background:color-mix(in srgb, var(--pb-bg-card-hover) 55%, transparent)}}.search-history-detail-card{padding:.625rem .75rem}.search-history-detail-label{letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);font-size:10px;font-weight:600}.search-history-diagnostics-shell{border:1px solid var(--pb-warning);overflow:hidden}@supports (color:color-mix(in lab, red, red)){.search-history-diagnostics-shell{border:1px solid color-mix(in srgb, var(--pb-warning) 30%, transparent)}}.search-history-diagnostics-shell{background:var(--pb-warning-dim);border-radius:.75rem}@supports (color:color-mix(in lab, red, red)){.search-history-diagnostics-shell{background:color-mix(in srgb, var(--pb-warning-dim) 35%, transparent)}}.downloads-pagination{border-top:1px solid var(--pb-border-subtle);padding:.5rem .875rem}@media (max-width:1024px){.downloads-header{gap:1rem}.downloads-table-wrap{overflow-x:auto}.downloads-table-wrap.is-clipped{overflow:auto hidden}.downloads-table{min-width:760px}}@media (max-width:640px){.downloads-header-main{gap:.875rem}.downloads-gauges{gap:.625rem}.downloads-toolbar-search{width:100%}}.comic-reader{--pb-reader-canvas:#070b12;--pb-reader-surface:#0d141ef0;--pb-reader-surface-strong:#0d141efa;--pb-reader-text:#f7f1e8;--pb-reader-text-muted:#cbd5e1;--pb-reader-text-dim:#9aaaba;--pb-reader-interactive:#8fb9ee;--pb-reader-danger:#e38473;--pb-reader-warning:#f2c98d;background:var(--pb-reader-canvas);width:100vw;max-width:none;height:100dvh;max-height:none;color:var(--pb-reader-text);border:0;margin:0;padding:0;position:fixed;inset:0;overflow:hidden}.comic-reader::backdrop{background:var(--pb-reader-canvas)}.comic-reader__shell{background:var(--pb-reader-canvas);width:100%;min-width:0;height:100%;min-height:0;color:var(--pb-reader-text);grid-template-rows:auto minmax(0,1fr) auto;font-family:DM Sans,sans-serif;display:grid;overflow:hidden}.comic-reader__topbar,.comic-reader__controls{z-index:20;background:var(--pb-reader-surface);-webkit-backdrop-filter:blur(12px);backdrop-filter:blur(12px);border-color:#cbd5e129;transition:opacity .14s;position:relative}.comic-reader__topbar{min-height:58px;padding:calc(.45rem + env(safe-area-inset-top,0px)) max(.75rem, env(safe-area-inset-right,0px)) .45rem max(.75rem, env(safe-area-inset-left,0px));border-bottom-width:1px;grid-template-columns:auto minmax(0,1fr) auto auto;align-items:center;gap:.75rem;display:grid}.comic-reader__identity{min-width:0}.comic-reader__identity h2,.comic-reader__identity p{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.comic-reader__identity h2{color:var(--pb-reader-text);margin:0;font-family:Bricolage Grotesque,sans-serif;font-size:.95rem;font-weight:800;line-height:1.2}.comic-reader__identity p{color:var(--pb-reader-text-dim);margin:.1rem 0 0;font-size:.72rem;line-height:1.2}.comic-reader__top-actions,.comic-reader__issue-navigation,.comic-reader__navigation,.comic-reader__sizing,.comic-reader__state-actions,.comic-reader__help-heading{align-items:center;display:flex}.comic-reader__top-actions,.comic-reader__issue-navigation,.comic-reader__navigation,.comic-reader__sizing,.comic-reader__state-actions{gap:.4rem}.comic-reader__issue-navigation{min-width:0}.comic-reader__issue-navigation .comic-reader__button{gap:.35rem}.comic-reader__button,.comic-reader__zoom-label,.comic-reader__fit-select,.comic-reader__page-jump input{min-height:44px;color:var(--pb-reader-text);font:inherit;background:#f7f1e812;border:1px solid #cbd5e138;border-radius:.55rem}.comic-reader__button,.comic-reader__zoom-label{white-space:nowrap;cursor:pointer;justify-content:center;align-items:center;min-width:44px;padding:.55rem .7rem;font-size:.76rem;font-weight:700;line-height:1;display:inline-flex}.comic-reader__button:hover,.comic-reader__zoom-label:hover,.comic-reader__button[aria-pressed=true]{background:#8fb9ee29;border-color:#8fb9ee8c}.comic-reader__button:focus-visible,.comic-reader__zoom-label:focus-visible,.comic-reader__fit-select:focus-visible,.comic-reader__page-jump input:focus-visible,.comic-reader__viewport:focus-visible{outline:2px solid var(--pb-reader-interactive);outline-offset:-2px}.comic-reader__button:disabled{cursor:default;opacity:.38}.comic-reader__button svg{fill:none;stroke:currentColor;stroke-linecap:round;stroke-linejoin:round;stroke-width:1.8px;width:19px;height:19px}.comic-reader__button--prominent{background:#8fb9ee1f;border-color:#8fb9ee61}.comic-reader__viewport{z-index:1;overscroll-behavior:contain;background:var(--pb-reader-canvas);scrollbar-color:#9aaaba8c transparent;touch-action:pan-y pinch-zoom;min-width:0;min-height:0;position:relative;overflow:auto}.comic-reader__viewport.is-pannable{touch-action:auto}.comic-reader__page-stage{justify-content:center;align-items:center;width:100%;min-width:100%;height:100%;min-height:100%;padding:.75rem;display:flex}.comic-reader__page{z-index:1;object-fit:contain;-webkit-user-select:none;user-select:none;-webkit-user-drag:none;flex:none;max-width:none;max-height:none;display:block;position:relative;box-shadow:0 8px 30px #00000061}.comic-reader__page--page{width:auto;max-width:100%;height:auto;max-height:100%}.comic-reader__page--width{align-self:flex-start;width:100%;height:auto}.comic-reader__page--height{width:auto;height:100%}.comic-reader__page--actual{align-self:flex-start;width:auto;height:auto}.comic-reader__state,.comic-reader__page-busy{z-index:8;text-align:center;justify-content:center;align-items:center;display:flex;position:absolute;top:50%;left:50%;transform:translate(-50%,-50%)}.comic-reader__state{width:min(90vw,32rem);color:var(--pb-reader-text-muted);background:#0d141ef5;border:1px solid #cbd5e133;border-radius:.9rem;flex-direction:column;gap:.5rem;padding:1.5rem;box-shadow:0 20px 56px #00000075}.comic-reader__state strong{color:var(--pb-reader-text);font-size:1rem}.comic-reader__state--error{border-color:var(--pb-reader-danger)}@supports (color:color-mix(in lab, red, red)){.comic-reader__state--error{border-color:color-mix(in srgb, var(--pb-reader-danger) 45%, transparent)}}.comic-reader__state-actions{flex-wrap:wrap;justify-content:center;margin-top:.5rem}.comic-reader__completion{z-index:9;right:max(.75rem, env(safe-area-inset-right,0px));bottom:max(.75rem, env(safe-area-inset-bottom,0px));width:min(24rem,100% - 1.5rem);color:var(--pb-reader-text-muted);background:#0d141ef5;border:1px solid #8fb9ee6b;border-radius:.8rem;padding:.85rem;position:absolute;box-shadow:0 16px 42px #0000006b}.comic-reader__completion strong{color:var(--pb-reader-text);font-family:Bricolage Grotesque,sans-serif;font-size:1rem;display:block}.comic-reader__completion p{margin:.35rem 0 0;font-size:.78rem}.comic-reader__page-busy{color:var(--pb-reader-text-muted);background:#0d141ee6;border-radius:999px;padding:.45rem .7rem;font-size:.72rem;top:1rem;transform:translate(-50%)}.comic-reader__spinner{border:2px solid #8fb9ee40;border-top-color:var(--pb-reader-interactive);border-radius:999px;width:1.5rem;height:1.5rem;animation:.7s linear infinite comic-reader-spin}@keyframes comic-reader-spin{to{transform:rotate(360deg)}}.comic-reader__tap-zone{z-index:4;background:0 0;border:0;margin:0;padding:0;position:absolute;top:0;bottom:0}.comic-reader__tap-zone--left{width:30%;left:0}.comic-reader__tap-zone--center{width:40%;left:30%}.comic-reader__tap-zone--right{width:30%;right:0}.comic-reader__controls{min-width:0;padding:.45rem max(.75rem, env(safe-area-inset-right,0px)) calc(.45rem + env(safe-area-inset-bottom,0px)) max(.75rem, env(safe-area-inset-left,0px));border-top-width:1px;justify-content:space-between;align-items:center;gap:.75rem;display:flex}.comic-reader__navigation,.comic-reader__sizing{min-width:0;position:relative}.comic-reader__page-jump{color:var(--pb-reader-text-dim);white-space:nowrap;align-items:center;gap:.35rem;font-size:.75rem;display:inline-flex}.comic-reader__page-jump input{text-align:center;appearance:textfield;width:3.4rem;padding:.4rem}.comic-reader__page-jump input::-webkit-outer-spin-button{appearance:none;margin:0}.comic-reader__page-jump input::-webkit-inner-spin-button{appearance:none;margin:0}.comic-reader__fit-select{color-scheme:dark;max-width:8.5rem;padding:0 1.9rem 0 .65rem;display:none}.comic-reader__zoom-label{min-width:3.75rem;color:var(--pb-reader-text-muted);padding-inline:.45rem}.comic-reader__input-error{border:1px solid var(--pb-reader-danger);width:max-content;max-width:min(20rem,100vw - 1.5rem);padding:.35rem .5rem;position:absolute;bottom:calc(100% + .35rem);left:50%;transform:translate(-50%)}@supports (color:color-mix(in lab, red, red)){.comic-reader__input-error{border:1px solid color-mix(in srgb, var(--pb-reader-danger) 42%, transparent)}}.comic-reader__input-error{background:var(--pb-reader-surface-strong);color:var(--pb-reader-danger);border-radius:.45rem;font-size:.7rem}.comic-reader__help{z-index:30;background:var(--pb-reader-surface-strong);border:1px solid #8fb9ee66;border-radius:.9rem;width:min(92vw,34rem);max-height:min(78dvh,38rem);padding:1rem;position:absolute;top:50%;left:50%;overflow:auto;transform:translate(-50%,-50%);box-shadow:0 20px 56px #0009}.comic-reader__help-heading{justify-content:space-between;gap:1rem}.comic-reader__help h3{color:var(--pb-reader-text);margin:0;font-family:Bricolage Grotesque,sans-serif;font-size:1rem}.comic-reader__help dl{gap:.15rem;margin:.75rem 0 0;display:grid}.comic-reader__help dl>div{border-top:1px solid #cbd5e11f;grid-template-columns:minmax(7.5rem,.7fr) minmax(0,1.3fr);gap:.75rem;padding:.45rem 0;display:grid}.comic-reader__help dt{color:var(--pb-reader-interactive);font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:600}.comic-reader__help dd{color:var(--pb-reader-text-muted);margin:0;font-size:.75rem}.comic-reader__save-status{z-index:40;right:max(.75rem, env(safe-area-inset-right,0px));bottom:calc(4.1rem + env(safe-area-inset-bottom,0px));max-width:min(22rem,100vw - 1.5rem);color:var(--pb-reader-warning);background:#231b11f5;border:1px solid #d7a15b66;border-radius:.55rem;margin:0;padding:.45rem .65rem;font-size:.72rem;position:absolute}.comic-reader__shell.is-controls-hidden .comic-reader__topbar,.comic-reader__shell.is-controls-hidden .comic-reader__controls{visibility:hidden;opacity:0;pointer-events:none}@media (max-width:900px){.comic-reader__issue-navigation .comic-reader__button span{clip:rect(0, 0, 0, 0);white-space:nowrap;width:1px;height:1px;position:absolute;overflow:hidden}.comic-reader__fit-page,.comic-reader__fit-wide{display:none}.comic-reader__fit-select{display:block}}@media (max-width:600px){.comic-reader__topbar{grid-template-columns:auto minmax(0,1fr) auto;gap:.5rem;min-height:54px}.comic-reader__issue-navigation{grid-area:2/1/auto/-1;justify-content:center}.comic-reader__top-actions{grid-area:1/3}.comic-reader__desktop-action{display:none}.comic-reader__controls{flex-wrap:wrap;justify-content:center;gap:.35rem .6rem}.comic-reader__navigation,.comic-reader__sizing{justify-content:center}.comic-reader__page-stage{padding:.4rem}.comic-reader__help dl>div{grid-template-columns:1fr;gap:.15rem}}@media (max-height:480px){.comic-reader__topbar,.comic-reader__controls{padding-top:.25rem;padding-bottom:.25rem}.comic-reader__topbar{min-height:48px}}@media (prefers-reduced-motion:reduce){.comic-reader__topbar,.comic-reader__controls{transition:none;visibility:visible!important;opacity:1!important;pointer-events:auto!important}.comic-reader__spinner{animation-duration:1.4s}}.reading-workspace{flex-direction:column;gap:1rem;display:flex}.reading-workspace-header,.dashboard-reading-shelf-header{justify-content:space-between;align-items:flex-end;gap:1rem;display:flex}.reading-page-size{align-items:center;gap:.65rem;display:flex}.reading-tabs{border-bottom:1px solid var(--pb-border);gap:.4rem;display:flex;overflow-x:auto}.reading-tab{color:var(--pb-text-tertiary);letter-spacing:.08em;text-transform:uppercase;flex:none;padding:.65rem .85rem;font-family:Syne,sans-serif;font-size:.7rem;font-weight:700;position:relative}.reading-tab:after{content:"";background:0 0;border-radius:999px;height:2px;position:absolute;bottom:-1px;left:.65rem;right:.65rem}.reading-tab:hover,.reading-tab:focus-visible,.reading-tab.is-active{color:var(--pb-interactive)}.reading-tab.is-active:after{background:var(--pb-interactive)}.reading-card-grid,.dashboard-reading-card-grid{grid-template-columns:repeat(auto-fill,minmax(260px,1fr));gap:.85rem;display:grid}.dashboard-reading-card-grid{grid-template-columns:repeat(4,minmax(0,1fr))}.reading-card{border:1px solid var(--pb-border);background:linear-gradient(145deg, var(--pb-interactive), transparent 42%), var(--pb-bg-card);border-radius:12px;grid-template-columns:92px minmax(0,1fr);min-width:0;display:grid;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.reading-card{background:linear-gradient(145deg, color-mix(in srgb, var(--pb-interactive) 5%, transparent), transparent 42%), var(--pb-bg-card)}}.reading-card{box-shadow:var(--pb-shadow-1)}.reading-card:not(.reading-card-dashboard){min-height:15.25rem}.reading-card-cover-link,.reading-card-cover-frame{min-height:138px;display:block}.reading-card-cover-frame{border-right:1px solid var(--pb-border-subtle);background:var(--pb-bg-shell);height:100%;position:relative;overflow:hidden}.reading-card-cover{object-fit:cover;width:100%;height:100%}.reading-card-cover-placeholder{width:100%;height:100%;color:var(--pb-text-dim);justify-content:center;align-items:center;display:flex}.reading-card-cover-placeholder svg{width:2.25rem;height:2.25rem}.reading-card-body{flex-direction:column;gap:.7rem;min-width:0;padding:.85rem;display:flex}.reading-card-series{color:var(--pb-text-primary);text-overflow:ellipsis;white-space:nowrap;font-family:Syne,sans-serif;font-size:.78rem;font-weight:700;display:block;overflow:hidden}.reading-card-series:hover,.reading-card-series:focus-visible{color:var(--pb-interactive)}.reading-card-issue{min-width:0;color:var(--pb-text-tertiary);gap:.4rem;margin-top:.2rem;font-size:.72rem;display:flex}.reading-card-issue>span:first-child,.reading-card-state{font-family:JetBrains Mono,monospace}.reading-card-issue-title{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.reading-card-state{color:var(--pb-text-secondary);font-size:.67rem;font-weight:600}.reading-card-progress{background:var(--pb-border-subtle);border-radius:999px;height:3px;margin-top:.4rem;overflow:hidden}.reading-card-progress span{border-radius:inherit;background:var(--pb-interactive);height:100%;display:block}.reading-card-state-region{min-height:1.75rem}.reading-card-actions{grid-template-rows:repeat(2,minmax(2.25rem,auto));grid-template-columns:repeat(2,minmax(0,1fr));align-items:stretch;gap:.4rem;margin-top:auto;display:grid}.reading-card-view-read .reading-card-actions{grid-template-columns:minmax(0,2fr) minmax(0,3fr)}.reading-card-view-read .reading-card-primary-action,.reading-card-view-read .reading-card-completion-action{white-space:nowrap;padding-inline:.5rem}.reading-card-primary-action,.reading-card-completion-action,.reading-card-queue-action{width:100%;min-width:0}.reading-card-queue-action{grid-column:1/-1}.reading-card-queue-action-placeholder{visibility:hidden;pointer-events:none;grid-column:1/-1;min-height:2.25rem}.reading-card-mutation-status{min-height:1rem;color:var(--pb-error);font-size:.68rem}.reading-empty-state{border:1px dashed var(--pb-border);text-align:center;background:var(--pb-bg-card);border-radius:12px;flex-direction:column;justify-content:center;align-items:center;min-height:260px;padding:2.5rem 1.5rem;display:flex}@supports (color:color-mix(in lab, red, red)){.reading-empty-state{background:color-mix(in srgb, var(--pb-bg-card) 78%, transparent)}}.reading-empty-icon{width:2.75rem;height:2.75rem;color:var(--pb-text-dim)}.reading-empty-state h2{color:var(--pb-text-primary);margin-top:.85rem;font-family:Syne,sans-serif;font-size:.95rem;font-weight:700}.reading-empty-state p{max-width:34rem;color:var(--pb-text-tertiary);margin-top:.35rem;font-size:.78rem}.dashboard-reading-shelf{border:1px solid var(--pb-border);background:var(--pb-bg-card);border-radius:12px;padding:.85rem}@supports (color:color-mix(in lab, red, red)){.dashboard-reading-shelf{background:color-mix(in srgb, var(--pb-bg-card) 88%, transparent)}}.dashboard-reading-shelf{box-shadow:var(--pb-shadow-1)}.dashboard-reading-shelf-header h2{color:var(--pb-text-primary);margin-top:.1rem;font-family:Syne,sans-serif;font-size:1rem;font-weight:700}.dashboard-reading-shelf-link{color:var(--pb-interactive);font-size:.72rem;font-weight:700}.dashboard-reading-shelf-link:hover,.dashboard-reading-shelf-link:focus-visible{text-decoration:underline}.reading-card-dashboard{grid-template-columns:76px minmax(0,1fr)}.reading-card-dashboard .reading-card-cover-link,.reading-card-dashboard .reading-card-cover-frame{min-height:124px}.reading-card-dashboard .reading-card-body{gap:.5rem;padding:.7rem}@media (max-width:1180px){.dashboard-reading-card-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (max-width:640px){.reading-workspace-header{flex-direction:column;align-items:flex-start}.reading-card-grid,.dashboard-reading-card-grid{grid-template-columns:1fr}.reading-card{grid-template-columns:82px minmax(0,1fr)}.reading-card-cover-link,.reading-card-cover-frame{min-height:128px}}@property --tw-translate-x{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-y{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-z{syntax:"*";inherits:false;initial-value:0}@property --tw-scale-x{syntax:"*";inherits:false;initial-value:1}@property --tw-scale-y{syntax:"*";inherits:false;initial-value:1}@property --tw-scale-z{syntax:"*";inherits:false;initial-value:1}@property --tw-space-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-divide-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-border-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-leading{syntax:"*";inherits:false}@property --tw-font-weight{syntax:"*";inherits:false}@property --tw-tracking{syntax:"*";inherits:false}@property --tw-ordinal{syntax:"*";inherits:false}@property --tw-slashed-zero{syntax:"*";inherits:false}@property --tw-numeric-figure{syntax:"*";inherits:false}@property --tw-numeric-spacing{syntax:"*";inherits:false}@property --tw-numeric-fraction{syntax:"*";inherits:false}@property --tw-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-shadow-color{syntax:"*";inherits:false}@property --tw-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-inset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-shadow-color{syntax:"*";inherits:false}@property --tw-inset-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-ring-color{syntax:"*";inherits:false}@property --tw-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-ring-color{syntax:"*";inherits:false}@property --tw-inset-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-ring-inset{syntax:"*";inherits:false}@property --tw-ring-offset-width{syntax:"";inherits:false;initial-value:0}@property --tw-ring-offset-color{syntax:"*";inherits:false;initial-value:#fff}@property --tw-ring-offset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-outline-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-blur{syntax:"*";inherits:false}@property --tw-brightness{syntax:"*";inherits:false}@property --tw-contrast{syntax:"*";inherits:false}@property --tw-grayscale{syntax:"*";inherits:false}@property --tw-hue-rotate{syntax:"*";inherits:false}@property --tw-invert{syntax:"*";inherits:false}@property --tw-opacity{syntax:"*";inherits:false}@property --tw-saturate{syntax:"*";inherits:false}@property --tw-sepia{syntax:"*";inherits:false}@property --tw-drop-shadow{syntax:"*";inherits:false}@property --tw-drop-shadow-color{syntax:"*";inherits:false}@property --tw-drop-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-drop-shadow-size{syntax:"*";inherits:false}@property --tw-backdrop-blur{syntax:"*";inherits:false}@property --tw-backdrop-brightness{syntax:"*";inherits:false}@property --tw-backdrop-contrast{syntax:"*";inherits:false}@property --tw-backdrop-grayscale{syntax:"*";inherits:false}@property --tw-backdrop-hue-rotate{syntax:"*";inherits:false}@property --tw-backdrop-invert{syntax:"*";inherits:false}@property --tw-backdrop-opacity{syntax:"*";inherits:false}@property --tw-backdrop-saturate{syntax:"*";inherits:false}@property --tw-backdrop-sepia{syntax:"*";inherits:false}@property --tw-duration{syntax:"*";inherits:false}@property --tw-ease{syntax:"*";inherits:false}@keyframes spin{to{transform:rotate(360deg)}}@keyframes pulse{50%{opacity:.5}} +@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-scale-x:1;--tw-scale-y:1;--tw-scale-z:1;--tw-space-y-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-ordinal:initial;--tw-slashed-zero:initial;--tw-numeric-figure:initial;--tw-numeric-spacing:initial;--tw-numeric-fraction:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-outline-style:solid;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-backdrop-blur:initial;--tw-backdrop-brightness:initial;--tw-backdrop-contrast:initial;--tw-backdrop-grayscale:initial;--tw-backdrop-hue-rotate:initial;--tw-backdrop-invert:initial;--tw-backdrop-opacity:initial;--tw-backdrop-saturate:initial;--tw-backdrop-sepia:initial;--tw-duration:initial;--tw-ease:initial}}}@layer theme{:root,:host{--color-red-300:oklch(80.8% .114 19.571);--color-slate-950:oklch(12.9% .042 264.695);--color-black:#000;--color-white:#fff;--spacing:.25rem;--container-xs:20rem;--container-sm:24rem;--container-md:28rem;--container-lg:32rem;--container-xl:36rem;--container-2xl:42rem;--container-3xl:48rem;--container-4xl:56rem;--container-5xl:64rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-lg:1.125rem;--text-lg--line-height:calc(1.75 / 1.125);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--text-3xl:1.875rem;--text-3xl--line-height:calc(2.25 / 1.875);--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--font-weight-bold:700;--font-weight-extrabold:800;--font-weight-black:900;--tracking-tight:-.025em;--tracking-wide:.025em;--tracking-wider:.05em;--leading-tight:1.25;--leading-relaxed:1.625;--radius-xs:.125rem;--radius-sm:.25rem;--radius-md:.375rem;--radius-lg:.5rem;--radius-xl:.75rem;--radius-2xl:1rem;--radius-3xl:1.5rem;--ease-in:cubic-bezier(.4, 0, 1, 1);--ease-out:cubic-bezier(0, 0, .2, 1);--animate-spin:spin 1s linear infinite;--animate-pulse:pulse 2s cubic-bezier(.4, 0, .6, 1) infinite;--blur-xs:4px;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1)}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;font-feature-settings:normal;font-variation-settings:normal;-webkit-tap-highlight-color:transparent;font-family:DM Sans,sans-serif;line-height:1.5}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-feature-settings:normal;font-variation-settings:normal;font-family:JetBrains Mono,Fira Code,monospace;font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring:where(:not(iframe)){outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab, red, red)){::placeholder{color:color-mix(in oklab, currentcolor 50%, transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{padding-block:0}::-webkit-datetime-edit-month-field{padding-block:0}::-webkit-datetime-edit-day-field{padding-block:0}::-webkit-datetime-edit-hour-field{padding-block:0}::-webkit-datetime-edit-minute-field{padding-block:0}::-webkit-datetime-edit-second-field{padding-block:0}::-webkit-datetime-edit-millisecond-field{padding-block:0}::-webkit-datetime-edit-meridiem-field{padding-block:0}::-webkit-calendar-picker-indicator{line-height:1}:-moz-ui-invalid{box-shadow:none}button,input:where([type=button],[type=reset],[type=submit]){appearance:button}::file-selector-button{appearance:button}::-webkit-inner-spin-button{height:auto}::-webkit-outer-spin-button{height:auto}[hidden]:where(:not([hidden=until-found])){display:none!important}html{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}:root,[data-theme=dark]{--pb-surface-app:#101824;--pb-surface-shell:#141e2c;--pb-surface-card:#172231;--pb-surface-raised:#1d2a3d;--pb-surface-input:#132030;--pb-surface-overlay:#070b12b8;--pb-surface-selected:#8fb9ee29;--pb-text-primary:#f7f1e8;--pb-text-secondary:#cbd5e1;--pb-text-tertiary:#9aaaba;--pb-text-inverse:#101824;--pb-border-subtle:#cbd5e11f;--pb-border-default:#cbd5e133;--pb-border-strong:#cbd5e152;--pb-interactive:#8fb9ee;--pb-interactive-hover:#7da9df;--pb-interactive-active:#6a96c8;--pb-interactive-selected:#8fb9ee2e;--pb-brand:#c6a17b;--pb-brand-muted:#c6a17b29;--pb-status-success:#68b88b;--pb-status-warning:#d7a15b;--pb-status-danger:#e38473;--pb-status-info:#8fb9ee;--pb-focus-ring:#8fb9ee47;--pb-focus-outline:#8fb9ee85;--pb-selection:#8fb9ee38;--pb-shadow-0:none;--pb-shadow-1:0 1px 2px #070b1238, 0 10px 24px #070b1229;--pb-shadow-2:0 2px 6px #070b1247, 0 18px 40px #070b1238;--pb-shadow-overlay:0 20px 56px #00000075;--pb-brand-hover:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-hover:color-mix(in srgb, var(--pb-brand) 84%, black)}}:root,[data-theme=dark]{--pb-brand-dim:var(--pb-brand-muted);--pb-brand-border:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-border:color-mix(in srgb, var(--pb-brand) 28%, transparent)}}:root,[data-theme=dark]{--pb-brand-signal:var(--pb-status-warning);--pb-brand-signal-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-signal-dim:color-mix(in srgb, var(--pb-status-warning) 16%, transparent)}}:root,[data-theme=dark]{--pb-brand-signal-hover:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-signal-hover:color-mix(in srgb, var(--pb-status-warning) 22%, transparent)}}:root,[data-theme=dark]{--pb-brand-signal-border:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-brand-signal-border:color-mix(in srgb, var(--pb-status-warning) 28%, transparent)}}:root,[data-theme=dark]{--pb-interactive-dim:var(--pb-interactive-selected);--pb-interactive-border:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-interactive-border:color-mix(in srgb, var(--pb-interactive) 30%, transparent)}}:root,[data-theme=dark]{--pb-success:var(--pb-status-success);--pb-success-dim:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-success-dim:color-mix(in srgb, var(--pb-status-success) 18%, transparent)}}:root,[data-theme=dark]{--pb-warning:var(--pb-status-warning);--pb-warning-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-warning-dim:color-mix(in srgb, var(--pb-status-warning) 20%, transparent)}}:root,[data-theme=dark]{--pb-error:var(--pb-status-danger);--pb-error-dim:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-error-dim:color-mix(in srgb, var(--pb-status-danger) 18%, transparent)}}:root,[data-theme=dark]{--pb-info:var(--pb-status-info);--pb-info-dim:var(--pb-status-info)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-info-dim:color-mix(in srgb, var(--pb-status-info) 18%, transparent)}}:root,[data-theme=dark]{--pb-purple:#a88bda;--pb-purple-dim:#a88bda2e;--pb-bg-base:var(--pb-surface-app);--pb-bg-surface:var(--pb-surface-shell);--pb-bg-card:var(--pb-surface-card);--pb-bg-card-hover:var(--pb-surface-card)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-bg-card-hover:color-mix(in srgb, var(--pb-surface-card) 72%, var(--pb-surface-raised))}}:root,[data-theme=dark]{--pb-bg-input:var(--pb-surface-input);--pb-bg-overlay:var(--pb-surface-overlay);--pb-table-row-stripe:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-table-row-stripe:color-mix(in srgb, var(--pb-bg-card-hover) 68%, transparent)}}:root,[data-theme=dark]{--pb-table-row-hover:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){:root,[data-theme=dark]{--pb-table-row-hover:color-mix(in srgb, var(--pb-bg-card-hover) 88%, transparent)}}:root,[data-theme=dark]{--pb-table-cell-x:1.25rem;--pb-table-cell-x-tight:1rem;--pb-page-footer-clearance:1.5rem;--pb-text-dim:var(--pb-text-tertiary);--pb-border:var(--pb-border-default);--pb-border-hover:var(--pb-border-strong)}[data-theme=light]{--pb-surface-app:#f6f1e8;--pb-surface-shell:#fbf6ee;--pb-surface-card:#fffcf7;--pb-surface-raised:#fff;--pb-surface-input:#fffdfa;--pb-surface-overlay:#1a16136b;--pb-surface-selected:#2f5e8c1f;--pb-text-primary:#1e1a17;--pb-text-secondary:#51473d;--pb-text-tertiary:#74675b;--pb-text-inverse:#f7f1e8;--pb-border-subtle:#51473d24;--pb-border-default:#51473d33;--pb-border-strong:#51473d57;--pb-interactive:#2f5e8c;--pb-interactive-hover:#274f76;--pb-interactive-active:#21435f;--pb-interactive-selected:#2f5e8c24;--pb-brand:#855e3d;--pb-brand-muted:#8c68471f;--pb-status-success:#2e6a4f;--pb-status-warning:#8a5a1f;--pb-status-danger:#b24432;--pb-status-info:#2f5e8c;--pb-focus-ring:#2f5e8c42;--pb-focus-outline:#2f5e8cb8;--pb-selection:#2f5e8c33;--pb-shadow-0:none;--pb-shadow-1:0 1px 2px #1a161314, 0 10px 24px #1a16130f;--pb-shadow-2:0 2px 6px #1a16131a, 0 18px 40px #1a161314;--pb-shadow-overlay:0 20px 56px #1a161333;--pb-brand-hover:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-hover:color-mix(in srgb, var(--pb-brand) 84%, black)}}[data-theme=light]{--pb-brand-dim:var(--pb-brand-muted);--pb-brand-border:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-border:color-mix(in srgb, var(--pb-brand) 28%, transparent)}}[data-theme=light]{--pb-brand-signal:var(--pb-status-warning);--pb-brand-signal-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-signal-dim:color-mix(in srgb, var(--pb-status-warning) 12%, transparent)}}[data-theme=light]{--pb-brand-signal-hover:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-signal-hover:color-mix(in srgb, var(--pb-status-warning) 18%, transparent)}}[data-theme=light]{--pb-brand-signal-border:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-brand-signal-border:color-mix(in srgb, var(--pb-status-warning) 24%, transparent)}}[data-theme=light]{--pb-interactive-dim:var(--pb-interactive-selected);--pb-interactive-border:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-interactive-border:color-mix(in srgb, var(--pb-interactive) 28%, transparent)}}[data-theme=light]{--pb-success:var(--pb-status-success);--pb-success-dim:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-success-dim:color-mix(in srgb, var(--pb-status-success) 12%, transparent)}}[data-theme=light]{--pb-warning:var(--pb-status-warning);--pb-warning-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-warning-dim:color-mix(in srgb, var(--pb-status-warning) 14%, transparent)}}[data-theme=light]{--pb-error:var(--pb-status-danger);--pb-error-dim:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-error-dim:color-mix(in srgb, var(--pb-status-danger) 12%, transparent)}}[data-theme=light]{--pb-info:var(--pb-status-info);--pb-info-dim:var(--pb-status-info)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-info-dim:color-mix(in srgb, var(--pb-status-info) 12%, transparent)}}[data-theme=light]{--pb-purple:#7b5c91;--pb-purple-dim:#7b5c9124;--pb-bg-base:var(--pb-surface-app);--pb-bg-surface:var(--pb-surface-shell);--pb-bg-card:var(--pb-surface-card);--pb-bg-card-hover:var(--pb-surface-card)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-bg-card-hover:color-mix(in srgb, var(--pb-surface-card) 72%, var(--pb-surface-raised))}}[data-theme=light]{--pb-bg-input:var(--pb-surface-input);--pb-bg-overlay:var(--pb-surface-overlay);--pb-table-row-stripe:var(--pb-surface-shell)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-table-row-stripe:color-mix(in srgb, var(--pb-surface-shell) 40%, transparent)}}[data-theme=light]{--pb-table-row-hover:var(--pb-surface-shell)}@supports (color:color-mix(in lab, red, red)){[data-theme=light]{--pb-table-row-hover:color-mix(in srgb, var(--pb-surface-shell) 58%, transparent)}}[data-theme=light]{--pb-table-cell-x:1.25rem;--pb-table-cell-x-tight:1rem;--pb-text-dim:var(--pb-text-tertiary);--pb-border:var(--pb-border-default);--pb-border-hover:var(--pb-border-strong)}@media (prefers-color-scheme:light){:root:not([data-theme]){--pb-surface-app:#f6f1e8;--pb-surface-shell:#fbf6ee;--pb-surface-card:#fffcf7;--pb-surface-raised:#fff;--pb-surface-input:#fffdfa;--pb-surface-overlay:#1a16136b;--pb-surface-selected:#2f5e8c1f;--pb-text-primary:#1e1a17;--pb-text-secondary:#51473d;--pb-text-tertiary:#74675b;--pb-text-inverse:#f7f1e8;--pb-border-subtle:#51473d24;--pb-border-default:#51473d33;--pb-border-strong:#51473d57;--pb-interactive:#2f5e8c;--pb-interactive-hover:#274f76;--pb-interactive-active:#21435f;--pb-interactive-selected:#2f5e8c24;--pb-brand:#855e3d;--pb-brand-muted:#8c68471f;--pb-status-success:#2e6a4f;--pb-status-warning:#8a5a1f;--pb-status-danger:#b24432;--pb-status-info:#2f5e8c;--pb-focus-ring:#2f5e8c42;--pb-focus-outline:#2f5e8cb8;--pb-selection:#2f5e8c33;--pb-shadow-0:none;--pb-shadow-1:0 1px 2px #1a161314, 0 10px 24px #1a16130f;--pb-shadow-2:0 2px 6px #1a16131a, 0 18px 40px #1a161314;--pb-shadow-overlay:0 20px 56px #1a161333;--pb-brand-hover:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-hover:color-mix(in srgb, var(--pb-brand) 84%, black)}}:root:not([data-theme]){--pb-brand-dim:var(--pb-brand-muted);--pb-brand-border:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-border:color-mix(in srgb, var(--pb-brand) 28%, transparent)}}:root:not([data-theme]){--pb-brand-signal:var(--pb-status-warning);--pb-brand-signal-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-signal-dim:color-mix(in srgb, var(--pb-status-warning) 12%, transparent)}}:root:not([data-theme]){--pb-brand-signal-hover:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-signal-hover:color-mix(in srgb, var(--pb-status-warning) 18%, transparent)}}:root:not([data-theme]){--pb-brand-signal-border:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-brand-signal-border:color-mix(in srgb, var(--pb-status-warning) 24%, transparent)}}:root:not([data-theme]){--pb-interactive-dim:var(--pb-interactive-selected);--pb-interactive-border:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-interactive-border:color-mix(in srgb, var(--pb-interactive) 28%, transparent)}}:root:not([data-theme]){--pb-success:var(--pb-status-success);--pb-success-dim:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-success-dim:color-mix(in srgb, var(--pb-status-success) 12%, transparent)}}:root:not([data-theme]){--pb-warning:var(--pb-status-warning);--pb-warning-dim:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-warning-dim:color-mix(in srgb, var(--pb-status-warning) 14%, transparent)}}:root:not([data-theme]){--pb-error:var(--pb-status-danger);--pb-error-dim:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-error-dim:color-mix(in srgb, var(--pb-status-danger) 12%, transparent)}}:root:not([data-theme]){--pb-info:var(--pb-status-info);--pb-info-dim:var(--pb-status-info)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-info-dim:color-mix(in srgb, var(--pb-status-info) 12%, transparent)}}:root:not([data-theme]){--pb-purple:#7b5c91;--pb-purple-dim:#7b5c9124;--pb-bg-base:var(--pb-surface-app);--pb-bg-surface:var(--pb-surface-shell);--pb-bg-card:var(--pb-surface-card);--pb-bg-card-hover:var(--pb-surface-card)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-bg-card-hover:color-mix(in srgb, var(--pb-surface-card) 72%, var(--pb-surface-raised))}}:root:not([data-theme]){--pb-bg-input:var(--pb-surface-input);--pb-bg-overlay:var(--pb-surface-overlay);--pb-table-row-stripe:var(--pb-surface-shell)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-table-row-stripe:color-mix(in srgb, var(--pb-surface-shell) 40%, transparent)}}:root:not([data-theme]){--pb-table-row-hover:var(--pb-surface-shell)}@supports (color:color-mix(in lab, red, red)){:root:not([data-theme]){--pb-table-row-hover:color-mix(in srgb, var(--pb-surface-shell) 58%, transparent)}}:root:not([data-theme]){--pb-table-cell-x:1.25rem;--pb-table-cell-x-tight:1rem;--pb-text-dim:var(--pb-text-tertiary);--pb-border:var(--pb-border-default);--pb-border-hover:var(--pb-border-strong)}}html,body{overscroll-behavior:none;overflow:hidden}:focus-visible{outline:2px solid var(--pb-focus-outline);outline-offset:3px;box-shadow:0 0 0 4px var(--pb-focus-ring);border-radius:4px}::selection{background-color:var(--pb-selection)}:where(a[href]:not([aria-disabled=true]),button:not(:disabled):not([aria-disabled=true]),input[type=button]:not(:disabled),input[type=submit]:not(:disabled),input[type=reset]:not(:disabled),input[type=checkbox]:not(:disabled),input[type=radio]:not(:disabled),input[type=file]:not(:disabled),input[type=image]:not(:disabled),select:not(:disabled),summary,[role=button]:not([aria-disabled=true]),[role=link]:not([aria-disabled=true]),label:has(:is(input[type=checkbox],input[type=radio],input[type=file]):not(:disabled))){cursor:pointer}:where(button:disabled,input:disabled,select:disabled,[aria-disabled=true]){cursor:not-allowed}}@layer components{.control-size-sm,.btn-sm,.chip-btn-sm,.icon-btn-sm{--pb-control-min-height:2rem;--pb-control-radius:.5rem;--pb-control-gap:.5rem;--pb-control-px:.75rem;--pb-control-py:.4375rem;--pb-control-font-size:.875rem;--pb-control-line-height:1.25rem;--pb-control-icon-size:.875rem;--pb-control-panel-item-px:.75rem;--pb-control-panel-item-py:.5rem;--pb-control-search-icon-left:.625rem;--pb-control-search-padding-left:2rem;--pb-control-search-padding-right:1.875rem;--pb-control-clear-right:.5rem}.control-size-md,.btn-md,.chip-btn,.icon-btn,.dropdown-select,.search-field,.btn-primary,.btn-warning,.btn-ghost,.btn-danger{--pb-control-min-height:2.5rem;--pb-control-radius:.5rem;--pb-control-gap:.5rem;--pb-control-px:.75rem;--pb-control-py:.5rem;--pb-control-font-size:.875rem;--pb-control-line-height:1.25rem;--pb-control-icon-size:1rem;--pb-control-panel-item-px:.75rem;--pb-control-panel-item-py:.5rem;--pb-control-search-icon-left:.75rem;--pb-control-search-padding-left:2.5rem;--pb-control-search-padding-right:2.25rem;--pb-control-clear-right:.625rem}.control-size-lg,.btn-lg,.chip-btn-lg,.icon-btn-lg{--pb-control-min-height:2.875rem;--pb-control-radius:.625rem;--pb-control-gap:.625rem;--pb-control-px:.875rem;--pb-control-py:.625rem;--pb-control-font-size:.9375rem;--pb-control-line-height:1.375rem;--pb-control-icon-size:1rem;--pb-control-panel-item-px:.875rem;--pb-control-panel-item-py:.625rem;--pb-control-search-icon-left:.875rem;--pb-control-search-padding-left:2.75rem;--pb-control-search-padding-right:2.5rem;--pb-control-clear-right:.75rem}.btn-primary.btn-sm,.btn-warning.btn-sm,.btn-ghost.btn-sm,.btn-danger.btn-sm,.btn-primary.control-size-sm,.btn-warning.control-size-sm,.btn-ghost.control-size-sm,.btn-danger.control-size-sm{--pb-control-min-height:2rem;--pb-control-radius:.5rem;--pb-control-gap:.5rem;--pb-control-px:.75rem;--pb-control-py:.4375rem;--pb-control-font-size:.875rem;--pb-control-line-height:1.25rem;--pb-control-icon-size:.875rem}.btn-primary.btn-md,.btn-warning.btn-md,.btn-ghost.btn-md,.btn-danger.btn-md,.btn-primary.control-size-md,.btn-warning.control-size-md,.btn-ghost.control-size-md,.btn-danger.control-size-md{--pb-control-min-height:2.5rem;--pb-control-radius:.5rem;--pb-control-gap:.5rem;--pb-control-px:.75rem;--pb-control-py:.5rem;--pb-control-font-size:.875rem;--pb-control-line-height:1.25rem;--pb-control-icon-size:1rem}.btn-primary.btn-lg,.btn-warning.btn-lg,.btn-ghost.btn-lg,.btn-danger.btn-lg,.btn-primary.control-size-lg,.btn-warning.control-size-lg,.btn-ghost.control-size-lg,.btn-danger.control-size-lg{--pb-control-min-height:2.875rem;--pb-control-radius:.625rem;--pb-control-gap:.625rem;--pb-control-px:.875rem;--pb-control-py:.625rem;--pb-control-font-size:.9375rem;--pb-control-line-height:1.375rem;--pb-control-icon-size:1rem}.btn-primary{background-color:var(--pb-interactive);--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-weight:var(--font-weight-medium);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:10px;justify-content:center;align-items:center;display:inline-flex}@media (hover:hover){.btn-primary:hover{background-color:var(--pb-interactive-hover)}}.btn-primary:disabled{cursor:not-allowed;opacity:.5}.btn-primary{color:var(--pb-text-inverse);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.header-add-menu{flex-shrink:0;display:inline-flex;position:relative}.header-add-menu__primary{gap:calc(var(--spacing) * 2);white-space:nowrap;border-top-right-radius:0;border-bottom-right-radius:0;padding-right:.625rem}.header-add-menu__trigger{gap:calc(var(--spacing) * 2);white-space:nowrap;border-left:1px solid var(--pb-text-inverse);border-top-left-radius:0;border-bottom-left-radius:0;justify-content:space-between;min-width:7rem;padding-inline:.5rem}@supports (color:color-mix(in lab, red, red)){.header-add-menu__trigger{border-left:1px solid color-mix(in srgb, var(--pb-text-inverse) 26%, transparent)}}.header-add-menu__chevron{width:.875rem;height:.875rem;transition:transform .15s}.header-add-menu__panel{z-index:50;padding:calc(var(--spacing) * 1.5);border:1px solid var(--pb-border);background:var(--pb-bg-card);min-width:17rem;box-shadow:var(--pb-shadow-2);border-radius:.75rem;position:absolute;top:calc(100% + .5rem);left:0}.header-add-menu__item{align-items:center;gap:calc(var(--spacing) * 2.5);width:100%;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);text-align:left;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);min-height:2.5rem;color:var(--pb-text);border-radius:.5rem;transition:color .15s,background-color .15s;display:flex}.header-add-menu__item:hover,.header-add-menu__item:focus-visible,.header-add-menu__item--selected{color:var(--pb-interactive);background:var(--pb-interactive-dim)}.header-add-menu__item--disabled{color:var(--pb-text-ter);cursor:not-allowed;opacity:.72}.header-add-menu__item--disabled:hover{color:var(--pb-text-ter);background:0 0}.header-add-menu__item-label{flex:1;min-width:0}.header-add-menu__check{color:var(--pb-success)}.header-add-menu__soon{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-wide);letter-spacing:var(--tracking-wide);white-space:nowrap;text-transform:uppercase;color:var(--pb-text-ter);flex-shrink:0}.header-add-menu__item svg{height:calc(var(--spacing) * 4);width:calc(var(--spacing) * 4);flex-shrink:0}@media (prefers-reduced-motion:reduce){.header-add-menu__chevron,.header-add-menu__item{transition:none}}.btn-ghost{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background-color:#0000;border-radius:10px;justify-content:center;align-items:center;display:inline-flex}@media (hover:hover){.btn-ghost:hover{border-color:var(--pb-border-strong);color:var(--pb-text-primary)}}.btn-ghost:disabled{cursor:not-allowed;opacity:.5}.btn-ghost{min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.btn-warning{--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-weight:var(--font-weight-medium);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:10px;justify-content:center;align-items:center;display:inline-flex}.btn-warning:disabled{cursor:not-allowed;opacity:.5}.btn-warning{background-color:var(--pb-warning);color:var(--pb-text-inverse);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.btn-warning:hover{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.btn-warning:hover{background-color:color-mix(in srgb, var(--pb-warning) 86%, var(--pb-text-primary))}}.btn-danger{justify-content:center;align-items:center;gap:var(--pb-control-gap);background-color:var(--pb-error);--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-weight:var(--font-weight-medium);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:10px;display:inline-flex}@media (hover:hover){.btn-danger:hover{--tw-brightness:brightness(90%);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}}.btn-danger:disabled{cursor:not-allowed;opacity:.5}.btn-danger{color:var(--pb-text-inverse);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.pill,.badge-success,.badge-warning,.badge-error,.badge-info,.badge-neutral,.badge-muted,.badge-purple{justify-content:center;align-items:center;gap:var(--spacing);padding-inline:calc(var(--spacing) * 2);padding-block:calc(var(--spacing) * .5);text-align:center;--tw-font-weight:var(--font-weight-medium);font-family:DM Sans,sans-serif;font-size:11px;font-weight:var(--font-weight-medium);border-radius:3.40282e38px;display:inline-flex}.pill-success,.badge-success{background-color:var(--pb-success-dim);color:var(--pb-success)}.pill-warning,.badge-warning{background-color:var(--pb-warning-dim);color:var(--pb-warning)}.pill-error,.badge-error{background-color:var(--pb-error-dim);color:var(--pb-error)}.pill-info,.badge-info{background-color:var(--pb-info-dim);color:var(--pb-info)}.pill-neutral,.badge-neutral{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary)}.pill-muted,.badge-muted{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card-hover);color:var(--pb-text-dim)}.pill-purple,.badge-purple{background-color:var(--pb-purple-dim);color:var(--pb-purple)}.count-badge{min-width:1.25rem;min-height:1.25rem;padding-inline:calc(var(--spacing) * 1.5);--tw-leading:1;--tw-font-weight:var(--font-weight-bold);font-size:10px;line-height:1;font-weight:var(--font-weight-bold);--tw-numeric-spacing:tabular-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,);border-radius:3.40282e38px;justify-content:center;align-items:center;display:inline-flex}.count-badge-info{background-color:var(--pb-info-dim);color:var(--pb-info)}.count-badge-success{background-color:var(--pb-success-dim);color:var(--pb-success)}.count-badge-warning{background-color:var(--pb-warning-dim);color:var(--pb-warning)}.count-badge-error{background-color:var(--pb-error-dim);color:var(--pb-error)}.count-badge-neutral{background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary)}.table-shell{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card)}.table-shell-header{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4)}.table-shell-header-comfortable{padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4)}.table-shell-content{padding:calc(var(--spacing) * 5)}.table-shell-content-comfortable{padding:calc(var(--spacing) * 6)}.table-toolbar{gap:calc(var(--spacing) * 3);border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);flex-direction:column;display:flex}@media (min-width:64rem){.table-toolbar{flex-direction:row;justify-content:space-between;align-items:flex-end}}:where(.table-toolbar-title-block>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}.table-toolbar-eyebrow{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.table-toolbar-title{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.table-toolbar-title-compact{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.table-toolbar-copy{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.table-toolbar-actions{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;display:flex}.table-header-row{grid-template-columns:minmax(0,1fr) auto;align-items:center;gap:.75rem;display:grid}.table-header-aside{justify-self:end}.table-summary{align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));flex-wrap:wrap;display:flex}.table-scroll{overflow-x:auto}.table-base{border-collapse:collapse;width:100%;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.table-head{text-align:left;font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-tracking:var(--tracking-wider);letter-spacing:var(--tracking-wider);color:var(--pb-text-secondary);text-transform:uppercase}.table-head-row{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border)}.table-head-cell{padding-block:calc(var(--spacing) * 3);--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);padding-left:var(--pb-table-cell-x);padding-right:var(--pb-table-cell-x)}.table-head-cell-tight{padding-block:calc(var(--spacing) * 2.5);--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);padding-left:var(--pb-table-cell-x-tight);padding-right:var(--pb-table-cell-x-tight)}.table-head-cell-right{text-align:right}:where(.table-body>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)));border-color:var(--pb-border)}.table-body-striped>.table-row:nth-child(2n){background-color:var(--pb-table-row-stripe)}.table-row{color:var(--pb-text-secondary)}.table-row-hover>.table-row{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.table-row-hover>.table-row:hover{background-color:var(--pb-table-row-hover)}.table-row-hover>.table-row-detail{transition-property:none}.table-row-hover>.table-row-detail:hover{background-color:#0000}.table-cell{padding-block:calc(var(--spacing) * 3);padding-left:var(--pb-table-cell-x);padding-right:var(--pb-table-cell-x)}.table-cell-tight{padding-block:calc(var(--spacing) * 2.5);padding-left:var(--pb-table-cell-x-tight);padding-right:var(--pb-table-cell-x-tight)}.table-cell-right{text-align:right}.table-cell-muted{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary)}.table-cell-nowrap{white-space:nowrap}.table-footer{border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}.table-empty{padding:calc(var(--spacing) * 8);text-align:center}.table-empty-copy{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}.chip-btn{border-radius:var(--radius-lg);border-style:var(--tw-border-style);--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-width:1px;justify-content:center;align-items:center;display:inline-flex}.chip-btn:disabled{cursor:not-allowed;opacity:.5}.chip-btn{gap:var(--pb-control-gap);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.chip-btn-pill{padding-inline:calc(var(--spacing) * 3.5);padding-block:calc(var(--spacing) * 1.5);--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:.2em;letter-spacing:.2em;text-transform:uppercase;border-radius:9999px}.chip-btn-neutral{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary)}@media (hover:hover){.chip-btn-neutral:hover{border-color:var(--pb-border-strong);color:var(--pb-text-primary)}}.chip-btn-selected{color:var(--pb-info);background-color:var(--pb-info-dim);border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-selected{border-color:color-mix(in srgb, var(--pb-info) 26%, transparent)}}.chip-btn-selected:hover{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-selected:hover{background-color:color-mix(in srgb, var(--pb-info) 20%, var(--pb-bg-card))}}.chip-btn-selected:hover{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-selected:hover{border-color:color-mix(in srgb, var(--pb-info) 34%, transparent)}}.chip-btn-info{color:var(--pb-info);background-color:var(--pb-info-dim);border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-info{border-color:color-mix(in srgb, var(--pb-info) 24%, transparent)}}.chip-btn-info:hover{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-info:hover{background-color:color-mix(in srgb, var(--pb-info) 20%, var(--pb-bg-card))}}.chip-btn-info:hover{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.chip-btn-info:hover{border-color:color-mix(in srgb, var(--pb-info) 32%, transparent)}}.chip-btn-success{color:var(--pb-success);background-color:var(--pb-success-dim);border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.chip-btn-success{border-color:color-mix(in srgb, var(--pb-success) 24%, transparent)}}.chip-btn-success:hover{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.chip-btn-success:hover{background-color:color-mix(in srgb, var(--pb-success) 20%, var(--pb-bg-card))}}.chip-btn-success:hover{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.chip-btn-success:hover{border-color:color-mix(in srgb, var(--pb-success) 32%, transparent)}}.chip-btn-warning{color:var(--pb-warning);background-color:var(--pb-warning-dim);border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.chip-btn-warning{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.chip-btn-warning:hover{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.chip-btn-warning:hover{background-color:color-mix(in srgb, var(--pb-warning) 20%, var(--pb-bg-card))}}.chip-btn-warning:hover{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.chip-btn-warning:hover{border-color:color-mix(in srgb, var(--pb-warning) 32%, transparent)}}.chip-btn-error{color:var(--pb-error);background-color:var(--pb-error-dim);border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.chip-btn-error{border-color:color-mix(in srgb, var(--pb-error) 24%, transparent)}}.chip-btn-error:hover{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.chip-btn-error:hover{background-color:color-mix(in srgb, var(--pb-error) 20%, var(--pb-bg-card))}}.chip-btn-error:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.chip-btn-error:hover{border-color:color-mix(in srgb, var(--pb-error) 32%, transparent)}}.card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);border-radius:14px}.card-hover{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:14px}@media (hover:hover){.card-hover:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}}.auth-side-panel{border-right:1px solid var(--pb-border);background:linear-gradient(160deg, var(--pb-bg-surface) 0%, var(--pb-bg-surface) 42%, var(--pb-bg-card) 100%);position:relative;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.auth-side-panel{background:linear-gradient(160deg, color-mix(in srgb, var(--pb-bg-surface) 90%, var(--pb-brand) 10%) 0%, var(--pb-bg-surface) 42%, color-mix(in srgb, var(--pb-bg-card) 78%, var(--pb-interactive) 22%) 100%)}}.auth-side-panel:before,.auth-side-panel:after{content:"";pointer-events:none;filter:blur(12px);position:absolute;inset:auto}.auth-side-panel:before{background:radial-gradient(circle, var(--pb-brand) 0%, transparent 64%);width:24rem;height:24rem;top:-10%;left:-8%}@supports (color:color-mix(in lab, red, red)){.auth-side-panel:before{background:radial-gradient(circle, color-mix(in srgb, var(--pb-brand) 26%, transparent) 0%, transparent 64%)}}.auth-side-panel:after{background:radial-gradient(circle, var(--pb-interactive) 0%, transparent 66%);width:26rem;height:26rem;bottom:-14%;right:-12%}@supports (color:color-mix(in lab, red, red)){.auth-side-panel:after{background:radial-gradient(circle, color-mix(in srgb, var(--pb-interactive) 22%, transparent) 0%, transparent 66%)}}.auth-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-overlay);border-radius:1.75rem;position:relative;overflow:hidden}.auth-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.auth-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 40%, transparent), transparent)}}.auth-summary-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.auth-summary-card-emphasis{border-color:var(--pb-brand-border);background-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.auth-summary-card-emphasis{background-color:color-mix(in srgb, var(--pb-brand) 8%, var(--pb-bg-card))}}.auth-kicker{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.login-brand-mobile{justify-content:center;display:flex}.login-hero-stack{flex-direction:column;gap:1rem;max-width:29rem;padding-bottom:.25rem;display:flex}.login-hero-headline{letter-spacing:-.02em;color:var(--pb-text-primary);text-wrap:balance;padding-bottom:.1em;font-family:Syne,sans-serif;font-size:clamp(2.3rem,3.1vw,3.35rem);font-weight:800;line-height:1.08}.login-hero-copy{max-width:25rem;color:var(--pb-text-secondary);padding-bottom:.08em;font-size:.95rem;line-height:1.72}.login-side-footer{color:var(--pb-text-tertiary);flex-wrap:wrap;align-items:center;gap:.625rem 1.25rem;padding-bottom:.08em;font-family:JetBrains Mono,monospace;font-size:.68rem;display:flex}.login-side-footer-item{align-items:center;gap:.45rem;display:inline-flex}.login-side-footer-dot{background:var(--pb-success);width:.4rem;height:.4rem;box-shadow:0 0 4px var(--pb-success);border-radius:9999px}@supports (color:color-mix(in lab, red, red)){.login-side-footer-dot{box-shadow:0 0 4px color-mix(in srgb, var(--pb-success) 35%, transparent)}}.login-side-footer-dot{flex-shrink:0}.login-card-title{letter-spacing:-.02em;color:var(--pb-text-primary);padding-bottom:.08em;font-family:Syne,sans-serif;font-size:1.9rem;font-weight:800;line-height:1.08}.login-card-subtitle{color:var(--pb-text-secondary);padding-bottom:.05em;font-size:.875rem;line-height:1.62}.login-card-hint{color:var(--pb-text-tertiary);padding-bottom:.04em;font-size:.75rem;line-height:1.6}.floating-panel{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-overlay);overflow:hidden}.add-series-header-metrics{flex-wrap:wrap;align-items:flex-start;gap:1.5rem;display:flex}.add-series-results-shell{position:relative}.add-series-results-loading{z-index:12;background:var(--pb-bg-base);border-radius:.875rem;justify-content:center;align-items:flex-start;padding:1.25rem;display:flex;position:absolute;inset:0}@supports (color:color-mix(in lab, red, red)){.add-series-results-loading{background:color-mix(in srgb, var(--pb-bg-base) 55%, transparent)}}.add-series-results-loading{-webkit-backdrop-filter:blur(3px);backdrop-filter:blur(3px);visibility:hidden;pointer-events:none}.add-series-results-loading.htmx-request{opacity:1;visibility:visible}.add-series-results-loading-card{border:1px solid var(--pb-border-default);background:var(--pb-bg-card);border-radius:.875rem;align-items:center;gap:.75rem;max-width:24rem;padding:.875rem 1rem;display:flex}@supports (color:color-mix(in lab, red, red)){.add-series-results-loading-card{background:color-mix(in srgb, var(--pb-bg-card) 94%, transparent)}}.add-series-results-loading-card{color:var(--pb-text-primary);box-shadow:var(--pb-shadow-1)}.add-series-results-loading-card svg{color:var(--pb-interactive);flex-shrink:0}.add-series-results-loading-title{font-size:.82rem;font-weight:800;line-height:1.2}.add-series-results-loading-copy{color:var(--pb-text-secondary);margin-top:.125rem;font-size:.74rem;line-height:1.35}.add-series-preview-notice{border:1px solid var(--pb-interactive);border-radius:.875rem;flex-wrap:wrap;align-items:center;gap:.875rem;padding:.875rem 1rem;display:flex}@supports (color:color-mix(in lab, red, red)){.add-series-preview-notice{border:1px solid color-mix(in srgb, var(--pb-interactive) 32%, var(--pb-border-default))}}.add-series-preview-notice{background:linear-gradient(135deg, var(--pb-interactive), transparent 70%), var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.add-series-preview-notice{background:linear-gradient(135deg, color-mix(in srgb, var(--pb-interactive) 12%, transparent), transparent 70%), var(--pb-bg-card)}}.add-series-preview-notice{color:var(--pb-text-primary);box-shadow:var(--pb-shadow-0)}.add-series-preview-notice-icon{background:var(--pb-interactive);border-radius:999px;flex:none;justify-content:center;align-items:center;width:2rem;height:2rem;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.add-series-preview-notice-icon{background:color-mix(in srgb, var(--pb-interactive) 14%, transparent)}}.add-series-preview-notice-icon{color:var(--pb-interactive)}.add-series-preview-notice-body{flex:auto;min-width:0}.add-series-preview-notice-title{letter-spacing:.08em;text-transform:uppercase;font-size:.78rem;font-weight:850;line-height:1.2}.add-series-preview-notice-copy{color:var(--pb-text-secondary);margin-top:.125rem;font-size:.8rem;line-height:1.45}.add-series-preview-notice-action{white-space:nowrap;flex:none;margin-left:auto}@media (max-width:640px){.add-series-preview-notice{align-items:flex-start}.add-series-preview-notice-action{justify-content:center;width:100%;margin-left:0}}.add-series-results-list{gap:calc(var(--spacing) * 2);flex-direction:column;display:flex}.add-series-result-card{border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);border-radius:.875rem;align-items:flex-start;gap:.875rem;padding:.875rem;transition:border-color .14s,background-color .14s;display:flex}.add-series-result-card:hover{border-color:var(--pb-border-strong);background:var(--pb-surface-selected)}@supports (color:color-mix(in lab, red, red)){.add-series-result-card:hover{background:color-mix(in srgb, var(--pb-surface-selected) 35%, var(--pb-bg-card))}}.add-series-result-card-static:hover{border-color:var(--pb-border-subtle);background:var(--pb-bg-card)}.add-series-result-cover{aspect-ratio:2/3;border:1px solid var(--pb-border-subtle);background:var(--pb-surface-shell);width:3.5rem;min-width:3.5rem;color:var(--pb-text-tertiary);border-radius:.625rem;overflow:hidden}.add-series-result-cover-empty{justify-content:center;align-items:center;width:100%;height:100%;display:flex}.add-series-result-main{flex-wrap:wrap;justify-content:space-between;align-items:flex-start;gap:.875rem;display:flex}.add-series-result-title{color:var(--pb-text-primary);font-size:.92rem;font-weight:600;line-height:1.25}.add-series-result-meta{color:var(--pb-text-secondary);flex-wrap:wrap;gap:.25rem .75rem;margin-top:.25rem;font-size:.78rem;display:flex}.add-series-result-description{color:var(--pb-text-tertiary);-webkit-line-clamp:2;-webkit-box-orient:vertical;margin-top:.375rem;font-size:.78rem;line-height:1.55;display:-webkit-box;overflow:hidden}.add-series-result-actions{align-items:center;gap:calc(var(--spacing) * 2);display:flex}.add-series-result-add{white-space:nowrap;flex-shrink:0}.add-series-empty-state{text-align:center;border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);border-radius:.875rem;padding:2.5rem 1.25rem}.add-series-empty-title{text-transform:uppercase;letter-spacing:.04em;color:var(--pb-text-primary);margin-top:.75rem;font-family:Syne,sans-serif;font-size:.9rem;font-weight:700}.add-series-empty-copy{color:var(--pb-text-secondary);margin-top:.25rem;font-size:.82rem;line-height:1.55}.add-series-modal-title{color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:1rem;font-weight:800;line-height:1.1}.add-series-modal-subtitle{color:var(--pb-text-tertiary);font-size:.78rem}.add-series-modal-section-label{border-bottom:1px solid var(--pb-border-subtle);text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);padding-bottom:.375rem;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700}.add-series-modal-row{justify-content:space-between;align-items:center;gap:.75rem;display:flex}.add-series-modal-row-label{color:var(--pb-text-secondary);font-size:.85rem;font-weight:500}.add-series-modal-preview{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-shell);color:var(--pb-text-secondary);border-radius:.625rem;padding:.625rem .75rem;font-family:JetBrains Mono,monospace;font-size:.72rem;line-height:1.5}.add-series-root-display.input-pb:disabled,.add-series-root-display.input-pb[readonly]{opacity:1;color:var(--pb-text-primary);background:var(--pb-surface-input);cursor:default}.media-result-card{gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1);display:flex}.media-result-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.media-result-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.media-result-card:hover{box-shadow:var(--pb-shadow-2)}.media-result-card-static:hover{border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1)}.media-result-cover{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card-hover);flex-shrink:0;width:4.75rem;overflow:hidden}.media-result-cover-lg{width:6.25rem}.media-result-cover-inner{aspect-ratio:2/3;width:100%;height:100%}.media-result-empty{width:100%;height:100%;color:var(--pb-text-dim);justify-content:center;align-items:center;display:flex}.media-result-meta{margin-top:var(--spacing);align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary);flex-wrap:wrap;display:flex}.media-result-divider{color:var(--pb-text-dim)}.match-file-card{gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-direction:column;display:flex}@media (min-width:64rem){.match-file-card{flex-direction:row;align-items:flex-start}}.match-file-card{box-shadow:var(--pb-shadow-1)}.match-file-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.match-file-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 58%, transparent)}}.match-file-card:hover{box-shadow:var(--pb-shadow-2)}.match-file-format{height:calc(var(--spacing) * 14);width:calc(var(--spacing) * 14);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:.18em;letter-spacing:.18em;text-transform:uppercase;border-width:1px;flex-shrink:0;justify-content:center;align-items:center;display:flex}.match-file-format-cbz{color:var(--pb-interactive);border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.match-file-format-cbz{border-color:color-mix(in srgb, var(--pb-interactive) 24%, transparent)}}.match-file-format-cbz{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.match-file-format-cbz{background-color:color-mix(in srgb, var(--pb-interactive) 12%, transparent)}}.match-file-format-cbr{color:var(--pb-purple);border-color:var(--pb-purple)}@supports (color:color-mix(in lab, red, red)){.match-file-format-cbr{border-color:color-mix(in srgb, var(--pb-purple) 24%, transparent)}}.match-file-format-cbr{background-color:var(--pb-purple)}@supports (color:color-mix(in lab, red, red)){.match-file-format-cbr{background-color:color-mix(in srgb, var(--pb-purple) 12%, transparent)}}.match-file-format-pdf{color:var(--pb-warning);border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.match-file-format-pdf{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.match-file-format-pdf{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.match-file-format-pdf{background-color:color-mix(in srgb, var(--pb-warning) 12%, transparent)}}.match-file-format-other{color:var(--pb-text-secondary);border-color:var(--pb-border);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.match-file-format-other{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 82%, transparent)}}.match-file-card-actions{gap:calc(var(--spacing) * 2);flex-direction:column;width:100%;display:flex}@media (min-width:64rem){.match-file-card-actions{align-items:flex-end;width:auto;min-width:11rem}}.library-series-card{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);min-width:0;height:100%;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1);flex-direction:column;display:flex;overflow:hidden}.library-series-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.library-series-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 56%, var(--pb-bg-card))}}.library-series-card:hover{box-shadow:var(--pb-shadow-2)}.library-series-card-active{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-series-card-active{border-color:color-mix(in srgb, var(--pb-interactive) 30%, transparent)}}.library-series-card-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-series-card-active{background-color:color-mix(in srgb, var(--pb-interactive) 10%, var(--pb-bg-card))}}.library-series-cover-frame{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);background:linear-gradient(160deg, var(--pb-bg-card-hover) 0%, var(--pb-bg-card) 100%);position:relative;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.library-series-cover-frame{background:linear-gradient(160deg, color-mix(in srgb, var(--pb-bg-card-hover) 88%, transparent) 0%, color-mix(in srgb, var(--pb-bg-card) 82%, transparent) 100%)}}.library-series-cover-frame-compact{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border)}.library-series-placeholder{justify-content:center;align-items:center;gap:calc(var(--spacing) * 2);width:100%;height:100%;color:var(--pb-text-dim);flex-direction:column;display:flex}.library-series-meta{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.library-series-progress-track{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.library-series-progress-track{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 82%, transparent)}}.series-grid-card-meta{container-type:inline-size}.series-grid-card-publisher-year{grid-template-columns:8rem minmax(0,1fr);align-items:start;gap:.25rem .5rem;display:grid}.series-grid-card-publisher{overflow-wrap:anywhere;min-width:0}.series-grid-card-year{text-align:right;justify-self:end}.series-grid-card-progress-track{background-color:var(--pb-text-dim)}@supports (color:color-mix(in lab, red, red)){.series-grid-card-progress-track{background-color:color-mix(in srgb, var(--pb-text-dim) 72%, var(--pb-bg-card) 28%)}}@container (max-width:11rem){.series-grid-card-publisher-year{grid-template-columns:8rem}.series-grid-card-year{text-align:left;justify-self:start}}.admin-workspace-page{gap:calc(var(--spacing) * 4);flex-direction:column;min-height:0;display:flex}.admin-workspace-header{padding-block:0}.admin-workspace-header-copy{max-width:var(--container-3xl);min-width:0}.admin-workspace-tag-row{gap:calc(var(--spacing) * 2);padding-top:var(--spacing);flex-wrap:wrap;display:flex}.admin-workspace-header-aside{min-width:0}.admin-workspace-summary-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);height:100%;padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1)}.admin-workspace-body{gap:calc(var(--spacing) * 7);display:grid}@media (min-width:80rem){.admin-workspace-body{grid-template-columns:10.75rem minmax(0,1fr);align-items:flex-start}}.admin-workspace-body{padding-bottom:var(--pb-page-footer-clearance)}.admin-workspace-rail{gap:calc(var(--spacing) * 3);flex-direction:column;display:flex}@media (min-width:80rem){.admin-workspace-rail{top:calc(var(--spacing) * 24);position:sticky}}:where(.admin-workspace-rail-copy>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}.admin-workspace-rail-links{gap:var(--spacing);flex-direction:column;display:flex}.admin-workspace-content{flex:1;min-width:0}:where(.admin-workspace-content>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.admin-workspace-content{width:min(100%,58rem)}.section-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);overflow:hidden}.admin-workspace-content .space-y-6>.section-card:not(:has(~.section-card)){margin-bottom:0!important}.section-card-visible{overflow:visible}.system-version-banner{gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);flex-direction:column;padding-block:1.125rem;display:flex;position:relative;overflow:hidden}@media (min-width:64rem){.system-version-banner{flex-direction:row;justify-content:space-between;align-items:center}}.system-version-banner{box-shadow:var(--pb-shadow-1)}.system-version-banner:before{content:"";background:var(--pb-brand);width:3px;position:absolute;top:0;bottom:0;left:0}.system-version-banner-main{align-items:flex-start;gap:calc(var(--spacing) * 4);min-width:0;display:flex}.system-version-banner-icon{height:calc(var(--spacing) * 11);width:calc(var(--spacing) * 11);background:var(--pb-brand-muted);color:var(--pb-brand);border-radius:10px;flex-shrink:0;justify-content:center;align-items:center;display:flex}.system-version-banner-title{letter-spacing:-.02em;color:var(--pb-text-primary);flex-wrap:wrap;align-items:center;gap:.625rem;font-family:Bricolage Grotesque,sans-serif;font-size:1.125rem;font-weight:800;display:flex}.system-version-banner-meta{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.system-version-banner-meta span{font-family:JetBrains Mono,monospace;font-size:.6875rem}.system-version-banner-actions{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;flex-shrink:0;display:flex}:where(.system-detail-list>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)))}.system-detail-list{border-color:var(--pb-border-subtle)}.system-detail-row{justify-content:space-between;align-items:baseline;gap:calc(var(--spacing) * 5);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 2.5);display:flex}.system-detail-label{color:var(--pb-text-secondary);flex-shrink:0;font-size:13px}.system-detail-value{color:var(--pb-text-primary);text-align:right;word-break:break-word;font-family:JetBrains Mono,monospace;font-size:.78125rem}.system-detail-value-brand{color:var(--pb-brand)}.system-registry-filename{--tw-leading:calc(var(--spacing) * 5);font-family:JetBrains Mono,Fira Code,monospace;font-size:.8125rem;line-height:calc(var(--spacing) * 5);--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.system-links-grid{gap:calc(var(--spacing) * 2);padding:calc(var(--spacing) * 5);grid-template-columns:repeat(auto-fit,minmax(11rem,1fr));display:grid}.system-link-card{align-items:center;gap:calc(var(--spacing) * 2.5);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 3.5);padding-block:calc(var(--spacing) * 2.5);--tw-font-weight:var(--font-weight-medium);font-size:12.5px;font-weight:var(--font-weight-medium);color:var(--pb-text-primary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background:var(--pb-bg-card-hover);display:flex}@supports (color:color-mix(in lab, red, red)){.system-link-card{background:color-mix(in srgb, var(--pb-bg-card-hover) 20%, transparent)}}.system-link-card:hover{border-color:var(--pb-border-hover);background:var(--pb-bg-card-hover)}.system-link-card svg{height:calc(var(--spacing) * 3.5);width:calc(var(--spacing) * 3.5);color:var(--pb-text-dim);flex-shrink:0}.system-link-card-arrow{height:calc(var(--spacing) * 3);width:calc(var(--spacing) * 3);color:var(--pb-text-dim);margin-left:auto}.stat-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.stat-card-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.stat-card-value{margin-top:calc(var(--spacing) * 3);font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.stat-card-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5);color:var(--pb-text-secondary)}.info-panel{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1)}.info-panel-muted{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.info-panel-muted{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.dashboard-briefing-shell{border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-2);background:radial-gradient(circle at top right, var(--pb-interactive) 0%, transparent 38%), radial-gradient(circle at left bottom, var(--pb-brand) 0%, transparent 34%), var(--pb-bg-card);position:relative;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.dashboard-briefing-shell{background:radial-gradient(circle at top right, color-mix(in srgb, var(--pb-interactive) 14%, transparent) 0%, transparent 38%), radial-gradient(circle at left bottom, color-mix(in srgb, var(--pb-brand) 10%, transparent) 0%, transparent 34%), var(--pb-bg-card)}}.dashboard-briefing-shell:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), var(--pb-interactive), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.dashboard-briefing-shell:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 44%, transparent), color-mix(in srgb, var(--pb-interactive) 44%, transparent), transparent)}}.dashboard-briefing-meta{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);width:100%;padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}@media (min-width:64rem){.dashboard-briefing-meta{max-width:13rem}}.dashboard-briefing-meta{box-shadow:var(--pb-shadow-1);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.dashboard-briefing-meta{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 42%, transparent)}}.dashboard-status-pill,.dashboard-priority-badge,.dashboard-watch-tag,.dashboard-scorecard-delta{align-items:center;gap:var(--spacing);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.16em;letter-spacing:.16em;text-transform:uppercase;border-width:1px;border-color:currentColor;border-radius:3.40282e38px;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.dashboard-status-pill,.dashboard-priority-badge,.dashboard-watch-tag,.dashboard-scorecard-delta{border-color:color-mix(in srgb, currentColor 20%, transparent)}}.dashboard-priority-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.dashboard-priority-grid{grid-template-columns:repeat(3,minmax(0,1fr))}}.dashboard-priority-card-v2{align-items:flex-start;gap:calc(var(--spacing) * 4);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:14px;grid-template-columns:1fr auto;transition:box-shadow .14s;display:grid;position:relative;overflow:hidden}.dashboard-priority-card-v2:before{content:"";background-color:#0000;width:6px;position:absolute;inset:0 auto 0 0}.dashboard-priority-card-v2:hover{box-shadow:var(--pb-shadow-2)}.dashboard-priority-card-v2-tone-pill-error:before{background-color:var(--pb-error)}.dashboard-priority-card-v2-tone-pill-warning:before{background-color:var(--pb-warning)}.dashboard-priority-card-v2-tone-pill-info:before{background-color:var(--pb-info)}.dashboard-priority-card-v2-tone-pill-success:before{background-color:var(--pb-success)}.dashboard-priority-card-v2-tone-pill-neutral:before,.dashboard-priority-card-v2-tone-pill-muted:before{background-color:var(--pb-text-dim)}.dashboard-priority-fact{border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-base);padding-inline:calc(var(--spacing) * 2.5);padding-block:calc(var(--spacing) * .5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary);display:inline-flex}.dashboard-priority-card,.dashboard-decision-card{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-width:1px}.dashboard-priority-card-critical,.dashboard-decision-card-critical{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-critical,.dashboard-decision-card-critical{border-color:color-mix(in srgb, var(--pb-error) 24%, transparent)}}.dashboard-priority-card-critical,.dashboard-decision-card-critical{background:linear-gradient(135deg, var(--pb-error) 0%, var(--pb-bg-card-hover) 100%)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-critical,.dashboard-decision-card-critical{background:linear-gradient(135deg, color-mix(in srgb, var(--pb-error) 10%, var(--pb-bg-card)) 0%, color-mix(in srgb, var(--pb-bg-card-hover) 92%, transparent) 100%)}}.dashboard-priority-card-high,.dashboard-decision-card-high{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-high,.dashboard-decision-card-high{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.dashboard-priority-card-high,.dashboard-decision-card-high{background:linear-gradient(135deg, var(--pb-warning) 0%, var(--pb-bg-card-hover) 100%)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-high,.dashboard-decision-card-high{background:linear-gradient(135deg, color-mix(in srgb, var(--pb-warning) 10%, var(--pb-bg-card)) 0%, color-mix(in srgb, var(--pb-bg-card-hover) 92%, transparent) 100%)}}.dashboard-priority-card-watch,.dashboard-decision-card-watch{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-watch,.dashboard-decision-card-watch{border-color:color-mix(in srgb, var(--pb-warning) 18%, transparent)}}.dashboard-priority-card-watch,.dashboard-decision-card-watch{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-watch,.dashboard-decision-card-watch{background-color:color-mix(in srgb, var(--pb-warning) 7%, var(--pb-bg-card))}}.dashboard-priority-card-healthy,.dashboard-decision-card-healthy{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-healthy,.dashboard-decision-card-healthy{border-color:color-mix(in srgb, var(--pb-success) 20%, transparent)}}.dashboard-priority-card-healthy,.dashboard-decision-card-healthy{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-healthy,.dashboard-decision-card-healthy{background-color:color-mix(in srgb, var(--pb-success) 7%, var(--pb-bg-card))}}.dashboard-priority-card-info,.dashboard-decision-card-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-info,.dashboard-decision-card-info{border-color:color-mix(in srgb, var(--pb-info) 18%, transparent)}}.dashboard-priority-card-info,.dashboard-decision-card-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-priority-card-info,.dashboard-decision-card-info{background-color:color-mix(in srgb, var(--pb-info) 7%, var(--pb-bg-card))}}.dashboard-priority-facts{margin-top:calc(var(--spacing) * 5);gap:calc(var(--spacing) * 3);border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-top:calc(var(--spacing) * 4);display:grid}@media (min-width:40rem){.dashboard-priority-facts{grid-template-columns:repeat(3,minmax(0,1fr))}}.dashboard-priority-facts dt{--tw-font-weight:var(--font-weight-medium);font-size:10px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.dashboard-priority-facts dd{margin-top:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-primary)}.dashboard-quiet-note{align-items:center;gap:calc(var(--spacing) * 3);border-radius:var(--radius-xl);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-success);background:var(--pb-success);display:flex}@supports (color:color-mix(in lab, red, red)){.dashboard-quiet-note{background:color-mix(in srgb, var(--pb-success) 10%, var(--pb-bg-card))}}.dashboard-quiet-note{border:1px solid var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-quiet-note{border:1px solid color-mix(in srgb, var(--pb-success) 18%, transparent)}}.dashboard-pulse-tile{padding-block:calc(var(--spacing) * 2.5);flex-direction:column;align-items:flex-start;display:flex}.dashboard-pulse-value{margin-top:var(--spacing);font-size:var(--text-xl);line-height:var(--tw-leading,var(--text-xl--line-height));--tw-font-weight:var(--font-weight-bold);font-weight:var(--font-weight-bold)}.dashboard-pulse-tile-right{border-left-style:var(--tw-border-style);border-left-width:1px;border-color:var(--pb-border);padding-left:calc(var(--spacing) * 3)}.dashboard-pulse-tile-bottom{border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-top:calc(var(--spacing) * 2.5)}.dashboard-scoreboard-attention-grid{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.dashboard-scoreboard-attention-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.dashboard-scoreboard-attention-grid{grid-template-columns:repeat(4,minmax(0,1fr))}}.dashboard-scorecard{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);min-width:0;height:100%;padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-width:1px;flex-direction:column;justify-content:space-between;display:flex}.dashboard-scorecard-critical{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-critical{border-color:color-mix(in srgb, var(--pb-error) 22%, transparent)}}.dashboard-scorecard-critical{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-critical{background-color:color-mix(in srgb, var(--pb-error) 8%, var(--pb-bg-card))}}.dashboard-scorecard-high{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-high{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.dashboard-scorecard-high{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-high{background-color:color-mix(in srgb, var(--pb-warning) 9%, var(--pb-bg-card))}}.dashboard-scorecard-watch{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-watch{border-color:color-mix(in srgb, var(--pb-warning) 18%, transparent)}}.dashboard-scorecard-watch{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-watch{background-color:color-mix(in srgb, var(--pb-warning) 7%, var(--pb-bg-card))}}.dashboard-scorecard-healthy{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-healthy{border-color:color-mix(in srgb, var(--pb-success) 20%, transparent)}}.dashboard-scorecard-healthy{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-healthy{background-color:color-mix(in srgb, var(--pb-success) 7%, var(--pb-bg-card))}}.dashboard-scorecard-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-info{border-color:color-mix(in srgb, var(--pb-info) 18%, transparent)}}.dashboard-scorecard-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-scorecard-info{background-color:color-mix(in srgb, var(--pb-info) 7%, var(--pb-bg-card))}}.dashboard-scorecard-title{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.dashboard-scorecard-value{font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.dashboard-scorecard-value-watch{color:var(--pb-status-warning)}.dashboard-scorecard-value-critical{color:var(--pb-status-danger)}.dashboard-scorecard-copy{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.dashboard-healthy-strip{align-items:center;gap:calc(var(--spacing) * 2);column-gap:calc(var(--spacing) * 6);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 3.5);flex-wrap:wrap;display:flex}.dashboard-healthy-strip-label{align-items:center;gap:calc(var(--spacing) * 1.5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-success);display:flex}.dashboard-healthy-metric{align-items:baseline;gap:calc(var(--spacing) * 1.5);color:var(--pb-text-secondary);font-size:.82rem;display:flex}.dashboard-healthy-metric strong{--tw-font-weight:var(--font-weight-bold);font-weight:var(--font-weight-bold);color:var(--pb-text-primary)}.dashboard-inline-link{align-items:center;gap:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.dashboard-inline-link:hover{color:var(--pb-interactive-hover)}.dashboard-download-row{align-items:center;gap:calc(var(--spacing) * 3);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3.5);grid-template-columns:1fr 100px 80px;display:grid}.dashboard-watch-item,.dashboard-exception-row{border-radius:var(--radius-2xl)}.dashboard-watch-item-critical,.dashboard-exception-row-critical{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-critical,.dashboard-exception-row-critical{border-color:color-mix(in srgb, var(--pb-error) 20%, transparent)}}.dashboard-watch-item-critical,.dashboard-exception-row-critical{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-critical,.dashboard-exception-row-critical{background-color:color-mix(in srgb, var(--pb-error) 7%, var(--pb-bg-card))}}.dashboard-watch-item-high,.dashboard-exception-row-high{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-high,.dashboard-exception-row-high{border-color:color-mix(in srgb, var(--pb-warning) 22%, transparent)}}.dashboard-watch-item-high,.dashboard-exception-row-high{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-high,.dashboard-exception-row-high{background-color:color-mix(in srgb, var(--pb-warning) 8%, var(--pb-bg-card))}}.dashboard-watch-item-watch,.dashboard-exception-row-watch{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-watch,.dashboard-exception-row-watch{border-color:color-mix(in srgb, var(--pb-warning) 16%, transparent)}}.dashboard-watch-item-watch,.dashboard-exception-row-watch{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-watch,.dashboard-exception-row-watch{background-color:color-mix(in srgb, var(--pb-warning) 6%, var(--pb-bg-card))}}.dashboard-watch-item-healthy,.dashboard-exception-row-healthy{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-healthy,.dashboard-exception-row-healthy{border-color:color-mix(in srgb, var(--pb-success) 16%, transparent)}}.dashboard-watch-item-healthy,.dashboard-exception-row-healthy{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-healthy,.dashboard-exception-row-healthy{background-color:color-mix(in srgb, var(--pb-success) 6%, var(--pb-bg-card))}}.dashboard-watch-item-info,.dashboard-exception-row-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-info,.dashboard-exception-row-info{border-color:color-mix(in srgb, var(--pb-info) 16%, transparent)}}.dashboard-watch-item-info,.dashboard-exception-row-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-watch-item-info,.dashboard-exception-row-info{background-color:color-mix(in srgb, var(--pb-info) 6%, var(--pb-bg-card))}}.dashboard-mission-page{gap:calc(var(--spacing) * 4);width:100%;padding-bottom:var(--pb-page-footer-clearance);flex-direction:column;display:flex}.dashboard-mission-control{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 5);flex-wrap:wrap;display:flex}.dashboard-mission-control__summary{align-items:flex-start;gap:calc(var(--spacing) * 6);flex-wrap:wrap;flex:1;min-width:0;display:flex}.dashboard-mission-control__title-block{min-width:0}.dashboard-mission-control__title{letter-spacing:.04em;text-transform:uppercase;color:var(--pb-text);font-family:Syne,sans-serif;font-size:1.5rem;font-weight:800;line-height:1}.dashboard-mission-control__title span{color:var(--pb-brand)}.dashboard-mission-control__subtitle{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.dashboard-mission-control__gauges{align-items:flex-end;gap:calc(var(--spacing) * 4);flex-wrap:wrap;display:flex}.dashboard-mission-control__gauges-spacer{flex:none;width:224px;min-width:224px;min-height:72px}.dashboard-gauge{text-align:center;min-width:64px}.dashboard-gauge__ring{width:56px;height:56px;margin-inline:auto;position:relative}.dashboard-gauge__ring svg{width:56px;height:56px;transform:rotate(-90deg)}.dashboard-gauge__track{fill:none;stroke:var(--pb-text-dim)}@supports (color:color-mix(in lab, red, red)){.dashboard-gauge__track{stroke:color-mix(in srgb, var(--pb-text-dim) 14%, transparent)}}.dashboard-gauge__track{stroke-width:4.5px}.dashboard-gauge__fill{fill:none;stroke-width:4.5px;stroke-linecap:round}.dashboard-gauge__value{justify-content:center;align-items:center;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:700;display:flex;position:absolute;inset:0}.dashboard-gauge__label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);margin-top:.2rem;font-size:.52rem;font-weight:600;display:block}.dashboard-gauge--success .dashboard-gauge__fill,.dashboard-gauge--success .dashboard-gauge__value{stroke:var(--pb-success);color:var(--pb-success)}.dashboard-gauge--info .dashboard-gauge__fill,.dashboard-gauge--info .dashboard-gauge__value{stroke:var(--pb-info);color:var(--pb-info)}.dashboard-gauge--warning .dashboard-gauge__fill,.dashboard-gauge--warning .dashboard-gauge__value{stroke:var(--pb-warning);color:var(--pb-warning)}.dashboard-gauge--danger .dashboard-gauge__fill,.dashboard-gauge--danger .dashboard-gauge__value{stroke:var(--pb-error);color:var(--pb-error)}.dashboard-gauge--neutral .dashboard-gauge__fill,.dashboard-gauge--neutral .dashboard-gauge__value{stroke:var(--pb-text-dim);color:var(--pb-text-dim)}.dashboard-scoreboard{grid-template-columns:repeat(auto-fit,minmax(140px,1fr));gap:.5rem;display:grid}.dashboard-scoreboard__item{gap:calc(var(--spacing) * .5);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);min-width:0;padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);box-shadow:var(--pb-shadow-1);border-radius:10px;flex-direction:column;display:flex}.dashboard-scoreboard__label,.dashboard-section__label{letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);font-family:Syne,sans-serif;font-size:.54rem;font-weight:700}.dashboard-scoreboard__value{color:var(--pb-text);font-family:JetBrains Mono,monospace;font-size:1.05rem;font-weight:700}.dashboard-scoreboard__delta{color:var(--pb-text-dim);font-family:JetBrains Mono,monospace;font-size:.66rem}:where(.dashboard-section>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2.5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2.5) * calc(1 - var(--tw-space-y-reverse)))}.dashboard-table-wrap,.dashboard-activity-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:12px;overflow-x:auto}.dashboard-table{border-collapse:collapse;width:100%}.dashboard-table th{text-align:left;letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);background:var(--pb-bg-shell);border-bottom:1px solid var(--pb-border);padding:.45rem .9rem;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700}.dashboard-table th.c,.dashboard-table td.c{text-align:center}.dashboard-table th.r,.dashboard-table td.r{text-align:right}.dashboard-table td{border-bottom:1px solid var(--pb-border-subtle);padding:.6rem .9rem;font-size:.82rem;transition:background-color .12s}.dashboard-table tbody tr:last-child td{border-bottom:0}.dashboard-table tbody tr:hover td{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.dashboard-table tbody tr:hover td{background:color-mix(in srgb, var(--pb-interactive) 8%, var(--pb-bg-card))}}.dashboard-table__status-col{width:40px}.dashboard-table__name{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.dashboard-table__detail{margin-top:calc(var(--spacing) * .5);color:var(--pb-text-dim);font-size:.75rem}.table-mono-value,td.table-mono-value,.dashboard-table__mono,td.dashboard-table__mono,.health-table__mono,td.health-table__mono,.series-mission-control-owned,.series-mission-control-type,.utility-tool-table-mono,td.utility-tool-table-mono,.downloads-mono-cell,td.downloads-mono-cell{color:var(--pb-text-secondary);font-variant-numeric:tabular-nums;font-family:JetBrains Mono,monospace;font-size:.75rem;font-weight:400;line-height:1.2}.series-mission-control-reading{color:var(--pb-interactive);white-space:nowrap;margin-top:.22rem;font-family:JetBrains Mono,monospace;font-size:.58rem;font-weight:600;line-height:1.2;display:block}.dashboard-table__link{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.dashboard-table__link:hover{color:var(--pb-interactive-hover)}.dashboard-led{vertical-align:middle;border-radius:999px;flex-shrink:0;width:8px;height:8px;display:inline-block}.dashboard-led--sys{border:1px solid var(--pb-text);width:10px;height:10px}@supports (color:color-mix(in lab, red, red)){.dashboard-led--sys{border:1px solid color-mix(in srgb, var(--pb-text) 10%, transparent)}}.dashboard-led--green{background:var(--pb-success);box-shadow:0 0 5px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.dashboard-led--green{box-shadow:0 0 5px color-mix(in srgb, var(--pb-success) 35%, transparent)}}.dashboard-led--amber{background:var(--pb-warning);box-shadow:0 0 5px var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.dashboard-led--amber{box-shadow:0 0 5px color-mix(in srgb, var(--pb-warning) 35%, transparent)}}.dashboard-led--red{background:var(--pb-error);box-shadow:0 0 5px var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.dashboard-led--red{box-shadow:0 0 5px color-mix(in srgb, var(--pb-error) 35%, transparent)}}.dashboard-led--blue{background:var(--pb-info);box-shadow:0 0 5px var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.dashboard-led--blue{box-shadow:0 0 5px color-mix(in srgb, var(--pb-info) 35%, transparent)}}.dashboard-led--off{background:var(--pb-text-dim);opacity:.4}.dashboard-progress{align-items:center;gap:calc(var(--spacing) * 2);display:flex}.dashboard-progress__track{height:calc(var(--spacing) * 1.5);border-radius:var(--radius-xs);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background:var(--pb-text-dim);flex:1;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.dashboard-progress__track{background:color-mix(in srgb, var(--pb-text-dim) 10%, transparent)}}.dashboard-progress__fill{height:100%;position:relative}.dashboard-progress__fill:after{content:"";background:repeating-linear-gradient(90deg,#0000 0 2px,#ffffff1f 2px 4px);position:absolute;inset:0}.dashboard-progress__fill--green{background:linear-gradient(90deg, var(--pb-success), var(--pb-success))}@supports (color:color-mix(in lab, red, red)){.dashboard-progress__fill--green{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-success) 45%, transparent), var(--pb-success))}}.dashboard-progress__fill--blue{background:linear-gradient(90deg, var(--pb-info), var(--pb-info))}@supports (color:color-mix(in lab, red, red)){.dashboard-progress__fill--blue{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-info) 45%, transparent), var(--pb-info))}}.dashboard-progress__fill--pulse{opacity:.6;width:100%;animation:1.6s ease-in-out infinite pulse}.dashboard-progress__percent,.dashboard-activity-row__time,.dashboard-footer-strip{font-family:JetBrains Mono,monospace}.dashboard-progress__percent{text-align:right;min-width:32px;color:var(--pb-text-sec);font-size:.66rem;font-weight:600}.dashboard-activity-card{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2)}.dashboard-activity-row{align-items:flex-start;gap:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);border-bottom:1px solid var(--pb-border-subtle);display:flex}.dashboard-activity-row:last-child{border-bottom:0}.dashboard-activity-row__dot{margin-top:calc(var(--spacing) * 1.5);height:calc(var(--spacing) * 2);width:calc(var(--spacing) * 2);border-radius:3.40282e38px;flex-shrink:0}.dashboard-activity-row__dot--acquired{background:var(--pb-success)}.dashboard-activity-row__dot--imported{background:var(--pb-brand)}.dashboard-activity-row__dot--failed{background:var(--pb-error)}.dashboard-activity-row__body{flex:1}.dashboard-activity-row__summary{--tw-leading:calc(var(--spacing) * 6);font-size:.82rem;line-height:calc(var(--spacing) * 6);--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-primary)}.dashboard-activity-row__detail{--tw-leading:calc(var(--spacing) * 5);font-size:.78rem;line-height:calc(var(--spacing) * 5);color:var(--pb-text-secondary)}.dashboard-activity-row__meta{margin-top:var(--spacing);justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);display:flex}.dashboard-activity-row__time{color:var(--pb-text-dim);font-size:.66rem}.dashboard-activity-row__link{--tw-font-weight:var(--font-weight-semibold);font-size:.68rem;font-weight:var(--font-weight-semibold);color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.dashboard-activity-row__link:hover{color:var(--pb-interactive-hover)}.dashboard-activity-card__empty{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-dim)}.dashboard-footer-strip{align-items:center;column-gap:calc(var(--spacing) * 6);row-gap:calc(var(--spacing) * 2);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2.5);color:var(--pb-text-dim);box-shadow:var(--pb-shadow-1);border-radius:10px;flex-wrap:wrap;font-size:.72rem;display:flex}.dashboard-footer-strip strong{color:var(--pb-text);font-weight:600}.library-page-shell{flex:1;width:100%;min-height:0;overflow:hidden}.library-format-pills{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;display:flex}.library-scoreboard__item{text-decoration:none;transition:background-color .12s}.library-scoreboard__item--link:hover{background-color:var(--pb-bg-card-hover)}.library-scoreboard__value--info{color:var(--pb-info)}.library-scoreboard__value--success{color:var(--pb-success)}.library-scoreboard__value--warning{color:var(--pb-warning)}.library-matching-banner{justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));border-width:1px;border-color:var(--pb-warning);border-radius:12px;display:flex}@supports (color:color-mix(in lab, red, red)){.library-matching-banner{border-color:color-mix(in srgb, var(--pb-warning) 26%, transparent)}}.library-matching-banner{background:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.library-matching-banner{background:color-mix(in srgb, var(--pb-warning) 8%, var(--pb-bg-card))}}.library-matching-banner{box-shadow:var(--pb-shadow-1)}.library-matching-banner__body{align-items:center;column-gap:calc(var(--spacing) * 2);row-gap:var(--spacing);flex-wrap:wrap;min-width:0;display:flex}.library-matching-banner__title{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.library-matching-banner__copy{color:var(--pb-text-secondary)}.library-browser{--library-browser-header-height:calc(1.5rem + .75rem + 1px);border:1px solid var(--pb-border);background:var(--pb-bg-card);min-height:0;box-shadow:var(--pb-shadow-1);border-radius:14px;flex:1 1 0;grid-template-columns:minmax(220px,260px) minmax(0,1fr);display:grid;overflow:hidden}.library-browser__tree{overscroll-behavior:contain;z-index:2;border-right:1px solid var(--pb-border);background:var(--pb-bg-surface);flex-direction:column;min-height:0;display:flex;position:relative;overflow:hidden}.library-browser__tree-list{min-height:0;padding-block:var(--spacing);overscroll-behavior:contain;flex:1;overflow-y:auto}.library-browser__tree-header,.library-browser__toolbar{min-height:var(--library-browser-header-height);height:var(--library-browser-header-height)}.library-browser__tree-header{padding-inline:calc(var(--spacing) * 3);z-index:3;letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);border-bottom:1px solid var(--pb-border);background:var(--pb-bg-surface);align-items:center;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700;display:flex;position:sticky;top:0}@supports (color:color-mix(in lab, red, red)){.library-browser__tree-header{background:color-mix(in srgb, var(--pb-bg-surface) 72%, transparent)}}.library-browser__tree-node{min-height:calc(var(--spacing) * 6);border-radius:var(--radius-md);min-width:0;padding-inline:calc(var(--spacing) * 1.5);padding-block:var(--spacing);color:var(--pb-text-secondary);flex:1;align-items:center;font-size:.73rem;font-weight:400;line-height:1.2;text-decoration:none;display:flex}.library-browser__tree-node--active{background:var(--pb-interactive-dim);color:var(--pb-interactive);font-weight:500}.library-browser__tree-icon{height:calc(var(--spacing) * 3.5);width:calc(var(--spacing) * 3.5);color:var(--pb-text-dim);flex-shrink:0;transition:transform .14s,color .14s}.library-browser__tree-icon--folder{color:var(--pb-brand)}.library-browser__tree-icon--expanded{color:var(--pb-interactive);transform:rotate(90deg)}.library-browser__tree-item{padding-left:calc(var(--library-tree-level,0) * .8rem)}.library-browser__tree-row{min-height:calc(var(--spacing) * 6);align-items:center;gap:var(--spacing);min-width:0;padding-inline:calc(var(--spacing) * 2);display:flex}.library-browser__tree-row--context .library-browser__tree-node,.library-browser__tree-row--context .library-browser__tree-toggle,.library-browser__tree-row--context .library-browser__tree-leaf{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-browser__tree-row--context .library-browser__tree-node,.library-browser__tree-row--context .library-browser__tree-toggle,.library-browser__tree-row--context .library-browser__tree-leaf{background:color-mix(in srgb, var(--pb-interactive) 10%, transparent)}}.library-browser__tree-row--context .library-browser__tree-node,.library-browser__tree-row--context .library-browser__tree-toggle,.library-browser__tree-row--context .library-browser__tree-leaf{color:var(--pb-interactive)}.library-browser__tree-toggle,.library-browser__tree-leaf{height:calc(var(--spacing) * 6);width:calc(var(--spacing) * 6);color:var(--pb-text-dim);flex-shrink:0;justify-content:center;align-items:center;display:inline-flex}.library-browser__tree-toggle{border-radius:4px;transition:background-color .12s,color .12s}.library-browser__tree-toggle:hover{background:var(--pb-bg-card-hover);color:var(--pb-text)}:where(.library-browser__tree-children>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * .5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * .5) * calc(1 - var(--tw-space-y-reverse)))}.library-browser__tree-name{flex:1;min-width:0;display:block}.library-browser__panel{z-index:1;flex-direction:column;min-width:0;min-height:0;display:flex;position:relative;overflow:hidden}.library-browser__toolbar{align-items:center;gap:calc(var(--spacing) * 2);padding-inline:calc(var(--spacing) * 3);border-bottom:1px solid var(--pb-border);background:var(--pb-bg-surface);flex-shrink:0;display:flex}@supports (color:color-mix(in lab, red, red)){.library-browser__toolbar{background:color-mix(in srgb, var(--pb-bg-surface) 72%, transparent)}}.library-browser__toolbar-main{align-items:center;gap:calc(var(--spacing) * 1.5);flex:1;min-width:0;display:flex}.library-browser__up-btn{height:calc(var(--spacing) * 6);width:calc(var(--spacing) * 6);border-radius:var(--radius-xs);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-shrink:0;justify-content:center;align-items:center;text-decoration:none;display:inline-flex}.library-browser__up-btn:hover{background:var(--pb-bg-card-hover);color:var(--pb-text)}.library-browser__up-btn.is-disabled{pointer-events:none;opacity:.4}.library-browser__path-block{flex:1;align-items:center;min-width:0;display:flex}.library-browser__path{color:var(--pb-text-sec);white-space:nowrap;text-overflow:ellipsis;font-family:JetBrains Mono,monospace;font-size:.56rem;line-height:1.15;overflow:hidden}.library-browser__table-wrap{overscroll-behavior:contain;min-height:0;box-shadow:none;background:0 0;border:0;border-radius:0;flex:1;position:relative;overflow:auto}.library-browser-table{min-width:760px}.library-browser-table th{padding:.4rem .75rem;font-size:.54rem;font-weight:600}.library-browser-table td{padding:.5rem .75rem;font-size:.76rem;font-weight:400;line-height:1.3}.library-browser__table-row--file,.library-browser__table-row--file td,.library-browser__table-row--file .library-browser__name-label{cursor:pointer}.library-browser__table-row--context{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-browser__table-row--context{background:color-mix(in srgb, var(--pb-interactive) 8%, transparent)}}.library-context-menu{border-radius:var(--radius-xl);border-style:var(--tw-border-style);min-width:15.5rem;padding-block:calc(var(--spacing) * 1.5);z-index:80;border-width:1px;border-color:var(--pb-border);position:fixed;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.library-context-menu{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-context-menu{background:linear-gradient(180deg, var(--pb-bg-card) 0%, var(--pb-bg-surface) 100%)}@supports (color:color-mix(in lab, red, red)){.library-context-menu{background:linear-gradient(180deg, color-mix(in srgb, var(--pb-bg-card) 94%, transparent) 0%, color-mix(in srgb, var(--pb-bg-surface) 97%, transparent) 100%)}}.library-context-menu{-webkit-backdrop-filter:blur(12px);backdrop-filter:blur(12px);box-shadow:0 18px 48px #0f172a42,0 6px 18px #0f172a29}.library-context-menu__group{display:contents}.library-context-menu__item{align-items:center;gap:calc(var(--spacing) * 3);width:100%;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);text-align:left;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));color:var(--pb-text);display:flex}.library-context-menu__item:hover{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-context-menu__item:hover{background:color-mix(in srgb, var(--pb-interactive) 10%, transparent)}}.library-context-menu__item:hover{color:var(--pb-interactive)}.library-context-menu__item--danger:hover{background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-context-menu__item--danger:hover{background:color-mix(in srgb, var(--pb-error) 11%, transparent)}}.library-context-menu__item--danger:hover{color:var(--pb-error)}.library-context-menu__icon{height:calc(var(--spacing) * 4);width:calc(var(--spacing) * 4);color:currentColor;flex-shrink:0}.library-context-menu__sep{margin-block:var(--spacing);background:var(--pb-border-subtle);height:1px}@supports (color:color-mix(in lab, red, red)){.library-context-menu__sep{background:color-mix(in srgb, var(--pb-border-subtle) 80%, transparent)}}.modal-panel.library-browser-modal{border-radius:18px;width:min(92vw,48rem);max-height:min(86vh,52rem);overflow:hidden}.modal-panel.library-browser-modal--properties{width:min(84vw,28rem)}.modal-panel.library-browser-modal--compact,.modal-panel.library-delete-modal{width:min(92vw,34rem)}.library-delete-modal__header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 4);border-bottom-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);border-bottom-width:1px;border-color:var(--pb-border);display:flex}.library-delete-modal__eyebrow{letter-spacing:.14em;text-transform:uppercase;color:var(--pb-text-dim);font-size:11px;font-weight:500}.library-delete-modal__title{margin-top:var(--spacing);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.library-delete-modal__subtitle{margin-top:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}:where(.library-delete-modal__body>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.library-delete-modal__body{padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4)}.library-delete-modal__copy{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}.library-delete-modal__impact{border-radius:var(--radius-lg);border-style:var(--tw-border-style);padding:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border-hover)}@supports (color:color-mix(in lab, red, red)){.library-delete-modal__impact{border-color:color-mix(in srgb, var(--pb-border-hover) 88%, transparent)}}.library-delete-modal__impact{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-delete-modal__impact{background:color-mix(in srgb, var(--pb-bg-surface) 40%, transparent)}}.library-delete-modal__impact .library-browser-modal__details-table{margin-top:0}.series-delete-modal__options{gap:calc(var(--spacing) * 3);flex-direction:column;display:flex}.series-delete-modal__option{align-items:flex-start;gap:calc(var(--spacing) * 3);border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border);display:flex}@supports (color:color-mix(in lab, red, red)){.series-delete-modal__option{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.series-delete-modal__option{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.series-delete-modal__option{background:color-mix(in srgb, var(--pb-bg-surface) 82%, transparent)}}.series-delete-modal__checkbox{margin-top:calc(var(--spacing) * .5);border-color:var(--pb-border-hover);background:var(--pb-bg-card-hover);color:var(--pb-error);border-radius:.25rem}.series-delete-modal__checkbox:focus{--tw-ring-color:var(--pb-error);--tw-ring-offset-width:0px}.series-delete-modal__option-copy{min-width:0}.series-delete-modal__option-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-primary)}.series-delete-modal__option-text{margin-top:calc(var(--spacing) * .5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary);line-height:1.5}.library-delete-modal__footer{justify-content:flex-end;align-items:center;gap:calc(var(--spacing) * 2);border-top-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 3);border-top-width:1px;border-color:var(--pb-border);background:var(--pb-bg-card);display:flex}@supports (color:color-mix(in lab, red, red)){.library-delete-modal__footer{background:color-mix(in srgb, var(--pb-bg-card) 80%, transparent)}}.library-browser-modal__header{align-items:flex-start;gap:calc(var(--spacing) * 4);background:linear-gradient(180deg, var(--pb-bg-surface) 0%, var(--pb-bg-card) 100%)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__header{background:linear-gradient(180deg, color-mix(in srgb, var(--pb-bg-surface) 95%, transparent) 0%, color-mix(in srgb, var(--pb-bg-card) 96%, transparent) 100%)}}.library-browser-modal--properties .library-browser-modal__header,.library-browser-modal--properties .library-browser-modal__body{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}.library-browser-modal--properties .modal-footer{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);justify-content:flex-end}.library-browser-modal--form .library-browser-modal__body{padding-block:0;padding-inline:0}.library-browser-modal--form .modal-footer{padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 3)}.library-browser-modal .modal-footer{justify-content:flex-end;gap:calc(var(--spacing) * 2);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 3);background:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal .modal-footer{background:color-mix(in srgb, var(--pb-bg-card) 80%, transparent)}}.library-browser-modal__title-block{align-items:flex-start;gap:calc(var(--spacing) * 3);min-width:0;display:flex}.library-browser-modal__icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);color:var(--pb-interactive);border-width:1px;border-color:var(--pb-interactive);flex-shrink:0;justify-content:center;align-items:center;display:flex}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon{border-color:color-mix(in srgb, var(--pb-interactive) 24%, transparent)}}.library-browser-modal__icon{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon{background:color-mix(in srgb, var(--pb-interactive) 10%, transparent)}}.library-browser-modal__icon svg{height:calc(var(--spacing) * 5);width:calc(var(--spacing) * 5)}.library-browser-modal__icon--warning{color:var(--pb-warning);border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon--warning{border-color:color-mix(in srgb, var(--pb-warning) 24%, transparent)}}.library-browser-modal__icon--warning{background:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon--warning{background:color-mix(in srgb, var(--pb-warning) 10%, transparent)}}.library-browser-modal__icon--danger{color:var(--pb-error);border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon--danger{border-color:color-mix(in srgb, var(--pb-error) 24%, transparent)}}.library-browser-modal__icon--danger{background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__icon--danger{background:color-mix(in srgb, var(--pb-error) 10%, transparent)}}.library-browser-modal--properties .library-browser-modal__icon{height:calc(var(--spacing) * 9);width:calc(var(--spacing) * 9);border-radius:var(--radius-xl)}.library-browser-modal--properties .library-browser-modal__icon svg{width:1.125rem;height:1.125rem}.library-browser-modal__title{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.library-browser-modal__subtitle{margin-top:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}.library-browser-modal__close{border-radius:var(--radius-lg);padding:calc(var(--spacing) * 1.5);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.library-browser-modal__close:hover{background:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__close:hover{background:color-mix(in srgb, var(--pb-bg-card-hover) 88%, transparent)}}.library-browser-modal__close:hover{color:var(--pb-text)}.library-browser-modal__close:focus-visible{outline:2px solid var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__close:focus-visible{outline:2px solid color-mix(in srgb, var(--pb-interactive) 42%, transparent)}}.library-browser-modal__close:focus-visible{outline-offset:2px}.library-browser-modal__body{min-height:0}.library-browser-modal__form{flex-direction:column;flex:1;min-height:0;display:flex}.library-browser-modal__content{gap:calc(var(--spacing) * 4);flex-direction:column;display:flex}.library-browser-modal__section-label{margin-bottom:calc(var(--spacing) * 2);padding-bottom:calc(var(--spacing) * 2);letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);border-bottom:1px solid var(--pb-border-subtle);font-family:Syne,sans-serif;font-size:.64rem;font-weight:700;display:block}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__section-label{border-bottom:1px solid color-mix(in srgb, var(--pb-border-subtle) 80%, transparent)}}.library-browser-modal__section{padding-bottom:var(--spacing)}.library-browser-modal__details-table{border-collapse:collapse;table-layout:fixed;width:100%;margin-top:.625rem;font-size:.8rem}.library-browser-modal__details-table--flush{margin-top:0}.library-browser-modal__detail-label{vertical-align:top;width:6.875rem;color:var(--pb-text-dim);font-size:inherit;text-align:left;padding:.25rem .75rem .25rem 0;font-weight:500;line-height:1.35}.library-browser-modal__detail-value{vertical-align:top;min-width:0;color:var(--pb-text);font-size:inherit;overflow-wrap:anywhere;padding:.25rem 0;font-weight:500;line-height:1.4}.library-browser-modal__detail-value--mono{color:var(--pb-text-sec);font-family:JetBrains Mono,monospace;font-size:.76rem;line-height:1.45}.library-browser-modal__loading{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 6);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);border-width:1px;border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__loading{border-color:color-mix(in srgb, var(--pb-border) 86%, transparent)}}.library-browser-modal__loading{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__loading{background:color-mix(in srgb, var(--pb-bg-surface) 94%, transparent)}}.library-browser-modal__stats{gap:calc(var(--spacing) * 3);grid-template-columns:repeat(auto-fit,minmax(8.5rem,1fr));display:grid}.library-browser-modal__stat{justify-content:space-between;gap:calc(var(--spacing) * 2);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);min-height:5rem;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border);flex-direction:column;display:flex}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__stat{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__stat{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__stat{background:color-mix(in srgb, var(--pb-bg-surface) 94%, transparent)}}.library-browser-modal__stat-label,.library-browser-modal__field-label{letter-spacing:.11em;text-transform:uppercase;color:var(--pb-text-dim);font-family:Syne,sans-serif;font-size:.62rem;font-weight:700}.library-browser-modal__stat-value,.library-browser-modal__preview-value{min-width:0;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-primary);overflow-wrap:anywhere;font-weight:500;line-height:1.35}.library-browser-modal__meta{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__meta{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__meta{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__meta{background:color-mix(in srgb, var(--pb-bg-surface) 93%, transparent)}}.library-browser-modal__meta-row{gap:calc(var(--spacing) * 2);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);grid-template-columns:minmax(8rem,10rem) minmax(0,1fr);display:grid}.library-browser-modal__meta-row+.library-browser-modal__meta-row{border-top:1px solid var(--pb-border-subtle)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__meta-row+.library-browser-modal__meta-row{border-top:1px solid color-mix(in srgb, var(--pb-border-subtle) 84%, transparent)}}.library-browser-modal__meta-label{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-dim);letter-spacing:.08em;text-transform:uppercase}.library-browser-modal__meta-value{min-width:0;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);overflow-wrap:anywhere;font-family:JetBrains Mono,monospace;font-size:.76rem;line-height:1.45}.library-browser-modal__storage{border-radius:var(--radius-2xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__storage{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__storage{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__storage{background:color-mix(in srgb, var(--pb-bg-surface) 93%, transparent)}}.library-browser-modal__storage-head{margin-bottom:calc(var(--spacing) * 2);justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);display:flex}.library-browser-modal__storage-bar{height:calc(var(--spacing) * 2.5);background:var(--pb-bg-card-hover);border-radius:3.40282e38px;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__storage-bar{background:color-mix(in srgb, var(--pb-bg-card-hover) 72%, transparent)}}.library-browser-modal__storage-fill{background:linear-gradient(90deg, var(--pb-interactive) 0%, var(--pb-brand) 100%);border-radius:3.40282e38px;height:100%;display:block}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__storage-fill{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-interactive) 82%, white) 0%, color-mix(in srgb, var(--pb-brand) 74%, white) 100%)}}.library-browser-modal__field{gap:calc(var(--spacing) * 2);flex-direction:column;display:flex}.library-browser-modal__settings-rows{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border)}.library-browser-modal__note{padding-inline:calc(var(--spacing) * 6)}.library-browser-modal__body-inset{padding-inline:calc(var(--spacing) * 6);padding-bottom:calc(var(--spacing) * 4)}.library-browser-modal__body-inset--section{padding-bottom:0}.library-browser-modal__body-inset--compact-top{padding-top:calc(var(--spacing) * 3)}.library-browser-modal__body-inset--roomy-bottom{padding-bottom:calc(var(--spacing) * 4)}.library-browser-modal__section-stack{gap:calc(var(--spacing) * 4);padding-top:calc(var(--spacing) * 4);flex-direction:column;display:flex}.library-browser-modal__section-stack--delete{gap:calc(var(--spacing) * 4)}.library-browser-modal__section-panel{flex-direction:column;display:flex}.library-browser-modal__section-heading-row{padding-inline:calc(var(--spacing) * 6);padding-bottom:calc(var(--spacing) * 3);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-bottom:1px solid var(--pb-border-subtle);padding-top:0}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__section-heading-row{border-bottom:1px solid color-mix(in srgb, var(--pb-border-subtle) 80%, transparent)}}.library-browser-modal__preview-block{min-height:calc(var(--spacing) * 10);border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);border-width:1px;border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__preview-block{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__preview-block{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__preview-block{background:color-mix(in srgb, var(--pb-bg-surface) 94%, transparent)}}.library-browser-modal__preview-block{overflow-wrap:anywhere;line-height:1.45}.library-browser-modal__preview-block--mono{font-family:JetBrains Mono,monospace;font-size:.76rem}.library-browser-modal__detail-input{width:100%;min-height:2.5rem;font-size:.88rem;display:block}.library-browser-modal__helper,.library-browser-modal__context-value{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.library-browser-modal__context-value--mono{color:var(--pb-text-sec);overflow-wrap:anywhere;font-family:JetBrains Mono,monospace;font-size:.76rem;line-height:1.45}.library-browser-modal__error{border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-error);border-width:1px;border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__error{border-color:color-mix(in srgb, var(--pb-error) 24%, transparent)}}.library-browser-modal__error{background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__error{background:color-mix(in srgb, var(--pb-error) 8%, transparent)}}.library-browser-modal__actions{align-items:center;gap:calc(var(--spacing) * 2);display:flex}.library-browser-modal__preview-grid{gap:calc(var(--spacing) * 3);grid-template-columns:repeat(auto-fit,minmax(10rem,1fr));display:grid}.library-browser-modal__preview-card{justify-content:space-between;gap:calc(var(--spacing) * 2);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);min-height:5.25rem;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);border-width:1px;border-color:var(--pb-border);flex-direction:column;display:flex}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__preview-card{border-color:color-mix(in srgb, var(--pb-border) 88%, transparent)}}.library-browser-modal__preview-card{background:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.library-browser-modal__preview-card{background:color-mix(in srgb, var(--pb-bg-surface) 94%, transparent)}}.library-browser__name{align-items:center;gap:calc(var(--spacing) * 2);display:flex}.library-browser__name-icon{height:calc(var(--spacing) * 3.5);width:calc(var(--spacing) * 3.5);flex-shrink:0}.library-browser__name-icon--folder{color:var(--pb-brand)}.library-browser__name-icon--file{color:var(--pb-text-dim)}.library-browser__name-label{text-overflow:ellipsis;white-space:nowrap;min-width:0;font-size:.76rem;font-weight:400;overflow:hidden}.library-browser__name-label--folder{color:var(--pb-text-primary);font-weight:500;text-decoration:none}.library-browser__name-label--folder:hover{color:var(--pb-interactive)}.library-browser__empty{height:100%;min-height:280px;padding-inline:calc(var(--spacing) * 6);text-align:center;flex-direction:column;justify-content:center;align-items:center;display:flex}.library-browser__empty-icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);color:var(--pb-text-dim)}.library-browser__empty-title{margin-top:calc(var(--spacing) * 4);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.library-browser__empty-copy{margin-top:calc(var(--spacing) * 2);max-width:var(--container-xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.library-empty-shell{align-items:flex-start;gap:calc(var(--spacing) * 4);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 8);box-shadow:var(--pb-shadow-1);border-radius:14px;flex-direction:column;display:flex}.library-empty-shell__icon-wrap{height:calc(var(--spacing) * 16);width:calc(var(--spacing) * 16);background-color:var(--pb-bg-card-hover);color:var(--pb-text-dim);border-radius:3.40282e38px;justify-content:center;align-items:center;display:flex}.library-empty-shell__icon{height:calc(var(--spacing) * 8);width:calc(var(--spacing) * 8)}:where(.library-empty-shell__body>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}.library-empty-shell__copy{max-width:var(--container-2xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 7);line-height:calc(var(--spacing) * 7);color:var(--pb-text-secondary)}.library-empty-shell__actions{align-items:center;gap:calc(var(--spacing) * 3);flex-wrap:wrap;display:flex}@media (max-width:1024px){.library-browser{flex:none;grid-template-columns:1fr;height:auto;min-height:520px}.library-browser__tree{border-right:0;border-bottom:1px solid var(--pb-border);max-height:220px}}.health-section__label{letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);margin-block:.625rem;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700;display:block}.health-component-card__sub,.health-detail-card__stat-value,.health-history-table td{font-family:JetBrains Mono,monospace}:where(.health-section>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2.5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2.5) * calc(1 - var(--tw-space-y-reverse)))}.health-component-grid{gap:calc(var(--spacing) * 3);grid-template-columns:repeat(auto-fit,minmax(280px,1fr));display:grid}.health-component-card{cursor:pointer;gap:calc(var(--spacing) * 3);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-top:calc(var(--spacing) * 4);padding-bottom:calc(var(--spacing) * 3);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-width:1px;border-radius:14px;flex-direction:column;height:100%;display:flex}.health-component-card:hover{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.health-component-card:hover{background:color-mix(in srgb, var(--pb-interactive) 4%, var(--pb-bg-card))}}.health-component-card:hover{border-color:var(--pb-border-strong)}.health-component-card--success{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.health-component-card--success{border-color:color-mix(in srgb, var(--pb-success) 22%, var(--pb-border))}}.health-component-card--warning{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.health-component-card--warning{border-color:color-mix(in srgb, var(--pb-warning) 22%, var(--pb-border))}}.health-component-card--danger{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.health-component-card--danger{border-color:color-mix(in srgb, var(--pb-error) 22%, var(--pb-border))}}.health-component-card--neutral{border-color:var(--pb-border)}.health-component-card__header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 3);display:flex}.health-component-card__name{text-transform:uppercase;letter-spacing:.04em;color:var(--pb-text);font-family:Syne,sans-serif;font-size:.84rem;font-weight:700}.health-component-card__stats{align-items:flex-start;gap:calc(var(--spacing) * 2);grid-template-columns:repeat(2,minmax(0,1fr));display:grid}.health-detail-card__stats{gap:calc(var(--spacing) * 2);display:grid}@media (min-width:40rem){.health-detail-card__stats{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.health-detail-card__stats{grid-template-columns:repeat(4,minmax(0,1fr))}}.health-component-card__stat,.health-detail-card__stat{border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);border-width:1px;border-color:var(--pb-border-subtle);border-radius:10px;flex-direction:column;gap:0;display:flex}.health-component-card__stat{background:var(--pb-bg-base)}.health-detail-card__stat{padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 1.5);background:var(--pb-bg-base)}.health-component-card__stat-label,.health-detail-card__stat-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);font-family:Syne,sans-serif;font-size:.46rem;font-weight:700}.health-component-card__stat-value,.health-detail-card__stat-value{--tw-font-weight:var(--font-weight-semibold);font-size:.82rem;font-weight:var(--font-weight-semibold);color:var(--pb-text-primary);display:block}.health-component-card__stat-value{white-space:pre-line;line-height:1.25}.health-component-card__stat-value--danger,.health-detail-card__stat-value--danger{color:var(--pb-error)}.health-component-card__stat-value--warning,.health-detail-card__stat-value--warning{color:var(--pb-warning)}.health-component-card__message{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary);min-height:2.5rem}.health-component-card__sub{color:var(--pb-text-dim);margin-top:auto;font-size:.68rem}:where(.health-detail-shell>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}.health-detail-back{align-items:center;gap:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.health-detail-back:hover{color:var(--pb-interactive-hover)}:where(.health-detail-card>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.health-detail-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:14px}.health-detail-card__header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 3);flex-wrap:wrap;display:flex}.health-detail-card__title-row{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;min-width:0;display:flex}.health-detail-card__title{letter-spacing:.03em;text-transform:uppercase;color:var(--pb-text);font-family:Syne,sans-serif;font-size:1rem;font-weight:800}:where(.health-check-list>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}.health-check-row{align-items:center;gap:calc(var(--spacing) * 3);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);border-radius:10px;grid-template-columns:8px minmax(120px,140px) minmax(0,1fr) auto;display:grid}.health-check-row--database-metric{grid-template-columns:8px minmax(160px,180px) minmax(0,1fr) auto}.health-check-row__name{text-align:left;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-primary);justify-self:flex-start}.health-check-row__message{min-width:0;font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5);color:var(--pb-text-dim);flex:1}.health-check-list--database-metrics .health-check-row__name,.health-check-list--database-metrics .health-check-row__message{text-align:left;justify-self:flex-start}.health-check-list--database-metrics .health-led{justify-self:center}.health-table-wrap{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);border-radius:12px;overflow-x:auto}.health-table{border-collapse:collapse;width:100%}.health-table th{text-align:left;letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);background:var(--pb-bg-shell);border-bottom:1px solid var(--pb-border);padding:.45rem .85rem;font-family:Syne,sans-serif;font-size:.52rem;font-weight:700}.health-table td{border-bottom:1px solid var(--pb-border-subtle);color:var(--pb-text-sec);padding:.55rem .85rem;font-size:.76rem}.health-table tbody tr:last-child td{border-bottom:0}.health-table tbody tr:hover td{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.health-table tbody tr:hover td{background:color-mix(in srgb, var(--pb-interactive) 8%, var(--pb-bg-card))}}.health-table th.c,.health-table td.c{text-align:center}.health-table th.r,.health-table td.r{text-align:right}.health-table__status-col{width:40px}.health-table__name{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.health-table__detail{margin-top:calc(var(--spacing) * .5);color:var(--pb-text-dim);font-size:.72rem}.health-led{border-radius:999px;flex-shrink:0;width:8px;height:8px;display:inline-block}.health-led--header{width:10px;height:10px}.health-led--green{background:var(--pb-success);box-shadow:0 0 5px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.health-led--green{box-shadow:0 0 5px color-mix(in srgb, var(--pb-success) 35%, transparent)}}.health-led--amber{background:var(--pb-warning);box-shadow:0 0 5px var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.health-led--amber{box-shadow:0 0 5px color-mix(in srgb, var(--pb-warning) 35%, transparent)}}.health-led--red{background:var(--pb-error);box-shadow:0 0 5px var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.health-led--red{box-shadow:0 0 5px color-mix(in srgb, var(--pb-error) 35%, transparent)}}.health-led--off{background:var(--pb-text-dim);opacity:.4}.health-footer-strip{align-items:center;column-gap:calc(var(--spacing) * 6);row-gap:var(--spacing);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2);color:var(--pb-text-dim);border-radius:10px;flex-wrap:wrap;font-family:JetBrains Mono,monospace;font-size:.72rem;display:flex}.health-footer-strip strong{color:var(--pb-text);font-weight:600}.step-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1)}.step-badge{height:calc(var(--spacing) * 8);width:calc(var(--spacing) * 8);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-info);background-color:var(--pb-info);border-radius:3.40282e38px;justify-content:center;align-items:center;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.step-badge{background-color:color-mix(in srgb, var(--pb-info) 14%, transparent)}}.step-badge{border:1px solid var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.step-badge{border:1px solid color-mix(in srgb, var(--pb-info) 28%, transparent)}}.action-card{align-items:flex-start;gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);padding:calc(var(--spacing) * 4);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background-color:var(--pb-bg-card-hover);display:flex}@supports (color:color-mix(in lab, red, red)){.action-card{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 40%, transparent)}}.action-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.action-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 82%, transparent)}}.action-card-icon{height:calc(var(--spacing) * 11);width:calc(var(--spacing) * 11);border-radius:var(--radius-xl);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-shrink:0;justify-content:center;align-items:center;display:flex}.action-card-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.action-card-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5);color:var(--pb-text-dim)}.utility-launch-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);height:100%;padding:calc(var(--spacing) * 5);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;box-shadow:var(--pb-shadow-1);flex-direction:column;transition-duration:.15s;display:flex}.utility-launch-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.utility-launch-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 82%, transparent)}}.utility-launch-card:hover{box-shadow:var(--pb-shadow-2)}.utility-launch-icon-brand{background-color:var(--pb-brand-dim);color:var(--pb-brand)}.utility-launch-icon-interactive{background-color:var(--pb-interactive-dim);color:var(--pb-interactive)}.utility-launch-icon-info{background-color:var(--pb-info-dim);color:var(--pb-info)}.utility-launch-icon-success{background-color:var(--pb-success-dim);color:var(--pb-success)}.utility-launch-icon-warning{background-color:var(--pb-warning-dim);color:var(--pb-warning)}.utility-launch-kicker{margin-top:calc(var(--spacing) * 4);--tw-font-weight:var(--font-weight-medium);font-size:10px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.utility-launch-title{margin-top:calc(var(--spacing) * 2);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.utility-launch-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utility-launch-footer{justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);padding-top:calc(var(--spacing) * 4);margin-top:auto;display:flex}.utility-launch-tag{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.utility-launch-link{align-items:center;gap:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-interactive);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.utility-launch-link:hover{color:var(--pb-interactive-hover)}.utility-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.utility-hero-card{padding-inline:calc(var(--spacing) * 7)}}.utility-hero-card{box-shadow:var(--pb-shadow-2)}.utility-top-link{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 1.5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1);border-radius:3.40282e38px;align-items:center;display:inline-flex}.utility-top-link:hover{border-color:var(--pb-border-hover);color:var(--pb-text);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.utility-top-link:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 58%, transparent)}}.utility-top-link-muted{color:var(--pb-text-dim)}.utility-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.utility-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 42%, transparent), transparent)}}.utility-hero-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.utility-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.95fr);align-items:flex-start}}.utility-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.utility-hero-title{margin-top:calc(var(--spacing) * 2);font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.utility-hero-copy{margin-top:calc(var(--spacing) * 2);max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utility-hero-highlights{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:48rem){.utility-hero-highlights{grid-template-columns:repeat(3,minmax(0,1fr))}}.utility-hero-highlight{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.utility-hero-highlight-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.utility-hero-highlight-value{margin-top:calc(var(--spacing) * 2);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.utility-hero-highlight-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utility-hero-actions{gap:calc(var(--spacing) * 3);display:grid}.utility-hero-action{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1)}.utility-hero-action:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.utility-hero-action:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 58%, transparent)}}.utility-hero-action-primary{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.utility-hero-action-primary{border-color:color-mix(in srgb, var(--pb-interactive) 26%, transparent)}}.utility-hero-action-primary{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.utility-hero-action-primary{background-color:color-mix(in srgb, var(--pb-interactive) 8%, var(--pb-bg-card))}}.utility-hero-action-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.utility-hero-action-title{margin-top:calc(var(--spacing) * 2);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.utility-hero-action-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utility-hero-notes{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.utility-hero-notes{grid-template-columns:repeat(3,minmax(0,1fr))}}@media (min-width:80rem){.utility-hero-notes{grid-template-columns:repeat(1,minmax(0,1fr))}}.utility-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.utility-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.utility-hero-note-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.utilities-page{width:100%;max-width:1100px;padding-bottom:var(--pb-page-footer-clearance);margin-inline:auto}.utilities-shell{width:100%}:where(.utilities-content>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.utilities-header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 5);flex-wrap:wrap;display:flex}.utilities-header-copy{align-items:flex-start;gap:calc(var(--spacing) * 5);flex-wrap:wrap;display:flex}.utilities-header-title{letter-spacing:.04em;text-transform:uppercase;font-family:Syne,sans-serif;font-size:1.6rem;font-weight:800;line-height:1}.utilities-header-title span{color:var(--pb-brand)}.utilities-header-subtitle{color:var(--pb-text-dim);margin-top:4px;font-size:.78rem;font-weight:400;line-height:1.5}.utilities-gauges{align-items:flex-end;gap:calc(var(--spacing) * 3.5);display:flex}.utilities-gauge{text-align:center}.utilities-gauge-ring{height:calc(var(--spacing) * 12);width:calc(var(--spacing) * 12);border-style:var(--tw-border-style);background-color:var(--pb-bg-card);border-width:4px;border-radius:3.40282e38px;justify-content:center;align-items:center;margin-inline:auto;display:flex;position:relative}@supports (color:color-mix(in lab, red, red)){.utilities-gauge-ring{background-color:color-mix(in srgb, var(--pb-bg-card) 70%, var(--pb-bg-base))}}.utilities-gauge-ring-success{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.utilities-gauge-ring-success{border-color:color-mix(in srgb, var(--pb-success) 60%, var(--pb-border))}}.utilities-gauge-ring-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.utilities-gauge-ring-info{border-color:color-mix(in srgb, var(--pb-info) 60%, var(--pb-border))}}.utilities-gauge-ring-ink{border-color:var(--pb-text)}@supports (color:color-mix(in lab, red, red)){.utilities-gauge-ring-ink{border-color:color-mix(in srgb, var(--pb-text) 22%, var(--pb-border))}}.utilities-gauge-value{font-family:JetBrains Mono,monospace;font-size:.68rem;font-weight:700}.utilities-gauge-label{--tw-font-weight:var(--font-weight-medium);font-size:.52rem;font-weight:var(--font-weight-medium);color:var(--pb-text-dim);text-transform:uppercase;letter-spacing:.1em;margin-top:2px;display:block}.utilities-tabs{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);border-radius:10px;align-self:center;display:inline-flex;overflow:hidden}.utilities-tab{padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 2);--tw-font-weight:var(--font-weight-bold);font-family:Syne,sans-serif;font-size:.78rem;font-weight:var(--font-weight-bold);--tw-tracking:.08em;letter-spacing:.08em;color:var(--pb-text-dim);text-transform:uppercase;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.utilities-tab+.utilities-tab{border-left:1px solid var(--pb-border)}.utilities-tab:hover{color:var(--pb-text);background-color:var(--pb-text)}@supports (color:color-mix(in lab, red, red)){.utilities-tab:hover{background-color:color-mix(in srgb, var(--pb-text) 3%, transparent)}}.utilities-tab.is-active{background-color:var(--pb-interactive);color:var(--pb-text-inverse)}.utilities-section-label{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-bottom:calc(var(--spacing) * 1.5);--tw-font-weight:var(--font-weight-bold);font-family:Syne,sans-serif;font-size:.62rem;font-weight:var(--font-weight-bold);--tw-tracking:.12em;letter-spacing:.12em;color:var(--pb-text-dim);text-transform:uppercase}.utilities-tool-grid{grid-template-columns:repeat(auto-fill,minmax(240px,1fr));gap:.75rem;display:grid}.utilities-tool-grid-wide{grid-template-columns:repeat(auto-fill,minmax(280px,1fr))}.utility-launch-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);height:100%;padding:calc(var(--spacing) * 5);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;box-shadow:var(--pb-shadow-1);border-radius:14px;flex-direction:column;transition-duration:.15s;display:flex;overflow:hidden}.utility-launch-card:hover{border-color:var(--pb-border-strong);box-shadow:var(--pb-shadow-2);background-color:var(--pb-bg-card);transform:translateY(-2px)}.utility-launch-header{justify-content:space-between;align-items:flex-start;gap:calc(var(--spacing) * 3);display:flex}.utility-launch-arrow{color:var(--pb-text-dim);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;transition-duration:.15s}.utility-launch-card:hover .utility-launch-arrow{color:var(--pb-interactive);transform:translate(2px)}.utility-launch-icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:10px;justify-content:center;align-items:center;display:flex}.utility-launch-title{margin-top:calc(var(--spacing) * 3);--tw-font-weight:var(--font-weight-extrabold);font-family:Syne,sans-serif;font-size:.9rem;font-weight:var(--font-weight-extrabold);--tw-tracking:.04em;letter-spacing:.04em;color:var(--pb-text-primary);text-transform:uppercase}.utility-launch-copy{margin-top:calc(var(--spacing) * 1.5);color:var(--pb-text-secondary);flex:1;font-size:.78rem;font-weight:400;line-height:1.5}.utility-launch-footer{margin-top:calc(var(--spacing) * 4);justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 3);display:flex}.utility-launch-tag{border-radius:var(--radius-md);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 2);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-family:JetBrains Mono,monospace;font-weight:var(--font-weight-medium);color:var(--pb-text-dim);text-transform:uppercase;letter-spacing:.06em;align-items:center;font-size:.6rem;display:inline-flex}.utility-tool-page{width:100%;max-width:1100px;margin-inline:auto}:where(.utility-tool-header-block>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}.utility-tool-back-link{align-items:center;gap:var(--spacing);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));font-size:.82rem;display:inline-flex}@media (hover:hover){.utility-tool-back-link:hover{color:var(--pb-text-primary)}}.utility-tool-header-shell{justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 4);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:14px;flex-wrap:wrap;display:flex}.utility-tool-header-left{align-items:center;gap:calc(var(--spacing) * 3.5);display:flex}.utility-tool-header-icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:10px;justify-content:center;align-items:center;display:flex}.utility-tool-header-icon-converter{background-color:var(--pb-interactive-dim);color:var(--pb-interactive)}.utility-tool-header-icon-mass-convert{background-color:var(--pb-success-dim);color:var(--pb-success)}.utility-tool-header-icon-integrity{background-color:var(--pb-info-dim);color:var(--pb-info)}.utility-tool-header-icon-rename,.utility-tool-header-icon-db-check{background-color:var(--pb-warning-dim);color:var(--pb-warning)}.utility-tool-header-icon-export{background-color:var(--pb-brand-dim);color:var(--pb-brand)}.utility-tool-header-title{--tw-font-weight:var(--font-weight-extrabold);font-family:Syne,sans-serif;font-size:1.1rem;font-weight:var(--font-weight-extrabold);--tw-tracking:.03em;letter-spacing:.03em;color:var(--pb-text-primary);text-transform:uppercase}.utility-tool-header-title span{color:var(--pb-brand)}.utility-tool-header-subtitle{margin-top:calc(var(--spacing) * .5);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.utility-tool-header-tag{border-radius:var(--radius-md);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-semibold);font-family:JetBrains Mono,monospace;font-size:.62rem;font-weight:var(--font-weight-semibold);--tw-tracking:.08em;letter-spacing:.08em;color:var(--pb-text-dim);text-transform:uppercase;align-items:center;display:inline-flex}.system-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.system-hero-card{padding-inline:calc(var(--spacing) * 7)}}.system-hero-card{box-shadow:var(--pb-shadow-2)}.system-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.system-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 42%, transparent), transparent)}}.system-hero-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.system-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.95fr);align-items:flex-start}}.system-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.system-hero-title{font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.system-hero-copy{max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.system-hero-aside{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.system-hero-aside{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.system-hero-aside{grid-template-columns:repeat(1,minmax(0,1fr))}}.system-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.system-hero-note-emphasis{border-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.system-hero-note-emphasis{border-color:color-mix(in srgb, var(--pb-brand) 20%, transparent)}}.system-hero-note-emphasis{background-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.system-hero-note-emphasis{background-color:color-mix(in srgb, var(--pb-brand) 8%, var(--pb-bg-card))}}.system-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.system-hero-note-title{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.system-hero-note-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.system-shell-body{gap:calc(var(--spacing) * 6);display:grid}@media (min-width:80rem){.system-shell-body{grid-template-columns:280px minmax(0,1fr)}}@media (min-width:96rem){.system-shell-body{grid-template-columns:300px minmax(0,1fr)}}.system-rail-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;overflow:hidden}.system-rail-header{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 5)}.system-rail-eyebrow{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.system-rail-title{margin-top:var(--spacing);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.system-rail-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}:where(.system-rail-links>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}.system-rail-links{padding:calc(var(--spacing) * 3)}.system-rail-footer{border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.system-rail-footer{background-color:color-mix(in srgb, var(--pb-bg-card) 55%, transparent)}}.system-tab-card{align-items:flex-start;gap:calc(var(--spacing) * 3);border-radius:var(--radius-2xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;border-width:1px;transition-duration:.15s;display:flex}.system-tab-card-inactive{border-color:var(--pb-border);background-color:var(--pb-bg-surface)}.system-tab-card-inactive:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.system-tab-card-inactive:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 58%, transparent)}}.system-tab-card-inactive:hover{box-shadow:var(--pb-shadow-1)}.system-tab-card-active{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.system-tab-card-active{border-color:color-mix(in srgb, var(--pb-interactive) 30%, transparent)}}.system-tab-card-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.system-tab-card-active{background-color:color-mix(in srgb, var(--pb-interactive) 10%, var(--pb-bg-card))}}.system-tab-card-active{box-shadow:var(--pb-shadow-1)}.system-tab-icon{margin-top:calc(var(--spacing) * .5);height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:var(--radius-2xl);flex-shrink:0;justify-content:center;align-items:center;display:flex}.system-tab-copy{flex:1;min-width:0}.system-tab-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.system-tab-description{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5);color:var(--pb-text-secondary);display:block}.system-tab-arrow{margin-top:var(--spacing);color:var(--pb-text-dim);transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;transition-duration:.15s}.system-tab-card:hover .system-tab-arrow,.system-tab-card-active .system-tab-arrow{color:var(--pb-interactive);transform:translate(.125rem)}.system-focus-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;position:relative;overflow:hidden}.system-focus-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-interactive), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.system-focus-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-interactive) 34%, transparent), transparent)}}.system-focus-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.system-focus-grid{grid-template-columns:minmax(0,1.35fr) minmax(320px,.95fr);align-items:flex-start}}.system-focus-meta{align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim);flex-wrap:wrap;display:flex}.system-focus-chip{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.system-focus-actions{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.system-focus-actions{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.system-focus-actions{grid-template-columns:repeat(1,minmax(0,1fr))}}.review-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.review-hero-card{padding-inline:calc(var(--spacing) * 7)}}.review-hero-card{box-shadow:var(--pb-shadow-2)}.review-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-info), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.review-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-info) 42%, transparent), transparent)}}.review-hero-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.review-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.95fr);align-items:flex-start}}.review-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.review-hero-title{font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.review-hero-copy{max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.review-hero-notes{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.review-hero-notes{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.review-hero-notes{grid-template-columns:repeat(1,minmax(0,1fr))}}.review-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.review-hero-note-emphasis{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.review-hero-note-emphasis{border-color:color-mix(in srgb, var(--pb-info) 20%, transparent)}}.review-hero-note-emphasis{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.review-hero-note-emphasis{background-color:color-mix(in srgb, var(--pb-info) 10%, var(--pb-bg-card))}}.review-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.review-hero-note-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.review-workspace-tabs{align-items:center;gap:calc(var(--spacing) * 2);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);border-radius:1.25rem;flex-wrap:wrap;display:flex}@media (min-width:40rem){.review-workspace-tabs{padding-inline:calc(var(--spacing) * 5)}}.review-workspace-tabs{box-shadow:var(--pb-shadow-1)}.review-tab-chip-active{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary);border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.review-tab-chip-active{border-color:color-mix(in srgb, var(--pb-info) 34%, transparent)}}.review-tab-chip-active{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.review-tab-chip-active{background-color:color-mix(in srgb, var(--pb-info) 12%, var(--pb-bg-card))}}.review-tab-chip-active{box-shadow:var(--pb-shadow-1)}.review-tab-chip-inactive{color:var(--pb-text-dim)}.review-focus-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;position:relative;overflow:hidden}.review-focus-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.review-focus-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 34%, transparent), transparent)}}.review-focus-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.review-focus-grid{grid-template-columns:minmax(0,1.35fr) minmax(320px,.95fr);align-items:flex-start}}.review-focus-chip{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.review-focus-actions{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.review-focus-actions{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.review-focus-actions{grid-template-columns:repeat(1,minmax(0,1fr))}}.review-note-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:1.25rem}.workflow-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.workflow-hero-card{padding-inline:calc(var(--spacing) * 7)}}.workflow-hero-card{box-shadow:var(--pb-shadow-2)}.workflow-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-interactive), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.workflow-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-interactive) 40%, transparent), transparent)}}.workflow-hero-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.workflow-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.95fr);align-items:flex-start}}.workflow-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.workflow-hero-title{font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.workflow-hero-copy{max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.workflow-hero-notes{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.workflow-hero-notes{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.workflow-hero-notes{grid-template-columns:repeat(1,minmax(0,1fr))}}.workflow-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.workflow-hero-note-emphasis{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-hero-note-emphasis{border-color:color-mix(in srgb, var(--pb-interactive) 20%, transparent)}}.workflow-hero-note-emphasis{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-hero-note-emphasis{background-color:color-mix(in srgb, var(--pb-interactive) 10%, var(--pb-bg-card))}}.workflow-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.workflow-hero-note-copy{margin-top:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.series-page-shell{gap:calc(var(--spacing) * 6);flex-direction:column;min-height:100%;display:flex}.series-hero-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.series-hero-card{padding-inline:calc(var(--spacing) * 7)}}.series-hero-card{box-shadow:var(--pb-shadow-2)}.series-hero-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), var(--pb-interactive), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.series-hero-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 34%, transparent), color-mix(in srgb, var(--pb-interactive) 34%, transparent), transparent)}}.series-hero-grid{gap:calc(var(--spacing) * 5);display:grid}@media (min-width:96rem){.series-hero-grid{grid-template-columns:minmax(0,1.45fr) minmax(320px,.8fr);align-items:flex-start}}.series-hero-eyebrow{--tw-font-weight:var(--font-weight-semibold);font-size:11px;font-weight:var(--font-weight-semibold);--tw-tracking:.24em;letter-spacing:.24em;color:var(--pb-brand);text-transform:uppercase}.series-hero-title{font-family:Bricolage Grotesque,sans-serif;font-size:var(--text-3xl);line-height:var(--tw-leading,var(--text-3xl--line-height));--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold);--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight);color:var(--pb-text-primary)}.series-hero-copy{max-width:var(--container-3xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 7);line-height:calc(var(--spacing) * 7);color:var(--pb-text-secondary)}.series-hero-notes{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.series-hero-notes{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:96rem){.series-hero-notes{grid-template-columns:repeat(1,minmax(0,1fr))}}.series-hero-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1)}.series-hero-note-emphasis{border-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.series-hero-note-emphasis{border-color:color-mix(in srgb, var(--pb-brand) 22%, transparent)}}.series-hero-note-emphasis{background-color:var(--pb-brand)}@supports (color:color-mix(in lab, red, red)){.series-hero-note-emphasis{background-color:color-mix(in srgb, var(--pb-brand) 8%, var(--pb-bg-card))}}.series-hero-note-kicker{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.16em;letter-spacing:.16em;color:var(--pb-text-dim);text-transform:uppercase}.series-hero-note-title{margin-top:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.series-hero-note-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.page-context-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:1.25rem}.page-context-back-link{align-items:center;gap:calc(var(--spacing) * 1.5);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.page-context-back-link:hover{color:var(--pb-text-primary)}.page-context-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.page-context-grid{grid-template-columns:minmax(0,1.35fr) minmax(280px,.9fr);align-items:flex-start}}.page-context-breadcrumbs{align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);flex-wrap:wrap;display:flex}.page-context-link{align-items:center;gap:calc(var(--spacing) * 1.5);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:inline-flex}.page-context-link:hover{color:var(--pb-text-primary)}.page-context-separator{height:calc(var(--spacing) * 3.5);width:calc(var(--spacing) * 3.5);color:var(--pb-text-dim)}.page-context-summary{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.page-context-chip-row{gap:calc(var(--spacing) * 2);flex-wrap:wrap;display:flex}.page-context-note{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);padding:calc(var(--spacing) * 4);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.page-context-note{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 40%, transparent)}}.page-context-note-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.page-context-note-copy{margin-top:var(--spacing);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.detail-hero-shell{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);border-radius:1.75rem;position:relative;overflow:hidden}@media (min-width:40rem){.detail-hero-shell{padding-inline:calc(var(--spacing) * 7)}}.detail-hero-shell{box-shadow:var(--pb-shadow-2)}.detail-hero-shell:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.detail-hero-shell:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 40%, transparent), transparent)}}.detail-hero-grid{gap:calc(var(--spacing) * 6);display:grid}@media (min-width:80rem){.detail-hero-grid{grid-template-columns:220px minmax(0,1fr) 320px}}.detail-cover-panel{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card-hover);padding:calc(var(--spacing) * 4);box-shadow:var(--pb-shadow-1);border-radius:1.5rem}.detail-summary-panel{min-width:0}:where(.detail-summary-panel>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 5) * calc(1 - var(--tw-space-y-reverse)))}:where(.detail-header-stack>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.detail-status-row{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;display:flex}.detail-body-copy{max-width:56rem}.detail-stat-grid{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.detail-stat-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.detail-stat-grid{grid-template-columns:repeat(4,minmax(0,1fr))}}.detail-info-grid{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.detail-info-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.detail-info-grid{grid-template-columns:repeat(3,minmax(0,1fr))}}:where(.detail-aside>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.detail-action-panel{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:1.25rem}.series-domain-page{width:100%;max-width:1400px;padding-bottom:var(--pb-page-footer-clearance);margin-inline:auto}.series-domain-breadcrumb-row{flex-wrap:wrap;align-items:center;gap:.75rem;display:flex}.series-domain-back-link{color:var(--pb-interactive);align-items:center;gap:.25rem;font-size:.82rem;transition:color .14s;display:inline-flex}.series-domain-back-link:hover{color:var(--pb-interactive-hover)}.series-domain-breadcrumbs{color:var(--pb-text-tertiary);flex-wrap:wrap;align-items:center;gap:.35rem;font-size:.82rem;display:flex}.series-domain-breadcrumbs a{color:inherit}.series-domain-breadcrumbs a:hover{color:var(--pb-interactive)}.series-domain-breadcrumb-sep{font-size:.7rem}.series-domain-breadcrumbs .current{color:var(--pb-text-primary);font-weight:600}.detail-hero-shell{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 6);box-shadow:var(--pb-shadow-1);border-radius:14px;overflow:hidden}.detail-hero-shell:before{display:none}.series-domain-hero-inner{grid-template-columns:130px minmax(0,1fr) minmax(13rem,16rem);align-items:start;gap:1.25rem;display:grid}.story-arc-detail-hero-inner{grid-template-columns:130px minmax(0,1fr) minmax(18rem,19rem)}.issue-domain-hero-inner{grid-template-columns:120px minmax(0,1fr) minmax(13rem,16rem)}.series-domain-cover-column{flex-shrink:0;width:130px}.issue-domain-cover-column{width:120px}.series-domain-cover-frame{aspect-ratio:2/3;border:1px solid var(--pb-border-default);background:linear-gradient(155deg, var(--pb-surface-card) 0%, var(--pb-surface-app) 100%);color:var(--pb-text-tertiary);border-radius:12px;justify-content:center;align-items:center;transition:box-shadow .2s;display:flex;overflow:hidden}.series-domain-cover-frame:hover{box-shadow:var(--pb-shadow-2)}.issue-domain-cover-frame{width:120px}.series-domain-cover-image{object-fit:cover;object-position:top;cursor:pointer;width:100%;height:100%}.series-domain-cover-placeholder{justify-content:center;align-items:center;width:100%;height:100%;display:flex}.series-domain-hero-title{text-transform:uppercase;letter-spacing:.04em;color:var(--pb-text-primary);overflow-wrap:anywhere;font-family:Syne,sans-serif;font-size:1.6rem;font-weight:800;line-height:1.05}.series-domain-hero-title span{color:var(--pb-brand)}.series-domain-hero-title-link{color:inherit;text-underline-offset:.18em;text-decoration:none;text-decoration-thickness:2px;transition:color .14s,text-decoration-color .14s;display:inline}.series-domain-hero-title-link:hover,.series-domain-hero-title-link:focus-visible{color:var(--pb-interactive);text-decoration-line:underline;-webkit-text-decoration-color:var(--pb-interactive);-webkit-text-decoration-color:var(--pb-interactive);text-decoration-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-domain-hero-title-link:hover,.series-domain-hero-title-link:focus-visible{-webkit-text-decoration-color:color-mix(in srgb, var(--pb-interactive) 58%, transparent);-webkit-text-decoration-color:color-mix(in srgb, var(--pb-interactive) 58%, transparent);text-decoration-color:color-mix(in srgb, var(--pb-interactive) 58%, transparent)}}.series-domain-hero-title-link:hover,.series-domain-hero-title-link:focus-visible{outline:none}.series-domain-hero-title-sm{font-size:1.4rem}.series-domain-hero-description{max-width:35rem;color:var(--pb-text-secondary);margin-top:.5rem;font-size:.85rem;line-height:1.7}.series-domain-hero-subtitle{color:var(--pb-text-secondary);margin-top:.25rem;font-size:.88rem}.series-domain-status-row{flex-wrap:wrap;align-items:center;gap:.375rem;margin-top:.75rem;display:flex}.series-domain-led{border-radius:999px;width:8px;height:8px;display:inline-block}.series-domain-led-on{background:var(--pb-status-success);box-shadow:0 0 5px var(--pb-status-success), 0 0 10px var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.series-domain-led-on{box-shadow:0 0 5px color-mix(in srgb, var(--pb-status-success) 35%, transparent), 0 0 10px color-mix(in srgb, var(--pb-status-success) 12%, transparent)}}.series-domain-led-off{background:var(--pb-text-tertiary);opacity:.35}.app-progress{align-items:center;gap:.625rem;display:flex}.app-progress-track{background:var(--pb-text-secondary);border-radius:2px;flex:1;height:8px}@supports (color:color-mix(in lab, red, red)){.app-progress-track{background:color-mix(in srgb, var(--pb-text-secondary) 6%, transparent)}}.app-progress-track{border:1px solid var(--pb-border-subtle);position:relative;overflow:hidden}.app-progress-fill{background:linear-gradient(90deg, var(--pb-status-success), var(--pb-status-success));height:100%}@supports (color:color-mix(in lab, red, red)){.app-progress-fill{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-success) 45%, transparent), var(--pb-status-success))}}.app-progress-fill{position:relative}.app-progress-fill:after{content:"";background:repeating-linear-gradient(90deg,#0000 0 2px,#ffffff1f 2px 4px);position:absolute;inset:0}.app-progress-fill--interactive{background:linear-gradient(90deg, var(--pb-interactive), var(--pb-interactive))}@supports (color:color-mix(in lab, red, red)){.app-progress-fill--interactive{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-interactive) 45%, transparent), var(--pb-interactive))}}.app-progress-fill--error{background:linear-gradient(90deg, var(--pb-error), var(--pb-error))}@supports (color:color-mix(in lab, red, red)){.app-progress-fill--error{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-error) 45%, transparent), var(--pb-error))}}.app-progress-value{text-align:right;min-width:40px;color:var(--pb-status-success);font-family:JetBrains Mono,monospace;font-size:.82rem;font-weight:700}.app-progress-value--interactive{color:var(--pb-interactive)}.app-progress-value--error{color:var(--pb-error)}.app-progress-value-stack{flex-direction:column;align-items:flex-end;line-height:1.05;display:flex}.app-progress-value-secondary{letter-spacing:.01em;color:var(--pb-text-dim);white-space:nowrap;font-family:DM Sans,sans-serif;font-size:.62rem;font-weight:600}.series-domain-info-grid{grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:.625rem;margin-top:1rem;display:grid}.series-domain-info-box{background:var(--pb-surface-app);border:1px solid var(--pb-border-subtle);border-radius:10px;padding:.75rem .875rem}.series-domain-info-box-wide{grid-column:span 2}.series-domain-info-label{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.series-domain-info-value{color:var(--pb-text-secondary);margin-top:.25rem;font-size:.85rem}.series-domain-path{word-break:break-all;font-size:.75rem}.series-domain-actions-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);border-radius:14px;padding:1.125rem 1.5rem}.series-domain-actions-inner{flex-wrap:wrap;align-items:center;gap:1rem;display:flex}.series-domain-actions-panel{border-left:1px solid var(--pb-border-subtle);flex-direction:column;gap:.75rem;min-width:0;padding-left:1.25rem;display:flex}.series-domain-actions-panel-wide{min-width:18rem}.series-domain-actions-title{text-transform:uppercase;letter-spacing:.08em;color:var(--pb-text-primary);white-space:nowrap;font-family:Syne,sans-serif;font-size:.88rem;font-weight:800}.series-domain-actions-title span{color:var(--pb-brand)}.series-domain-actions-divider{background:var(--pb-border-default);align-self:stretch;width:1px}.series-domain-actions-buttons{flex-wrap:wrap;flex:1;align-items:center;gap:.5rem;display:flex}.series-domain-actions-panel .series-domain-actions-buttons{flex-direction:column;flex:initial;align-items:center;gap:.55rem}.series-domain-actions-panel .series-domain-actions-buttons>a,.series-domain-actions-panel .series-domain-actions-buttons>button,.series-domain-actions-panel .series-domain-actions-buttons .series-domain-inline-toggle{width:100%}.series-domain-actions-panel .series-domain-actions-buttons>a,.series-domain-actions-panel .series-domain-actions-buttons>button{gap:var(--pb-control-gap);justify-content:center}.series-domain-actions-panel .series-domain-actions-buttons .series-domain-inline-toggle{justify-content:space-between}.series-domain-action-form{width:100%}.series-domain-action-form>button{justify-content:center;width:100%}.series-domain-inline-toggle{border:1px solid var(--pb-border-default);background:var(--pb-surface-card);border-radius:10px;align-items:center;gap:.75rem;height:40px;padding:0 .75rem;display:inline-flex}.series-domain-inline-toggle-label{color:var(--pb-text-primary);font-size:.82rem;font-weight:600}.series-domain-section-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);border-radius:14px;padding:1rem 1.25rem}.series-domain-section-title{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:.82rem;font-weight:700}:where(.series-domain-alt-panel>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}.series-domain-alt-title{text-transform:uppercase;letter-spacing:.12em;color:var(--pb-text-dim);font-family:Syne,sans-serif;font-size:.72rem;font-weight:700}.series-domain-alt-strip{flex-direction:column;align-items:flex-start;gap:.5rem;display:flex}.series-domain-alt-list{flex-wrap:wrap;align-items:flex-start;gap:.35rem;min-height:0;display:flex}.series-domain-alt-pill{gap:.35rem}.series-domain-alt-pill-remove{color:var(--pb-text-tertiary);margin-left:.1rem;transition:color .14s}.series-domain-alt-pill-remove:hover{color:var(--pb-status-danger)}.series-domain-alt-form{flex-wrap:wrap;align-items:center;gap:.5rem;width:100%;display:flex}.series-domain-alt-input{border:1px solid var(--pb-border-default);background:var(--pb-surface-input);width:180px;color:var(--pb-text-primary);border-radius:8px;padding:.45rem .75rem;font-size:.78rem}.series-domain-alt-input:focus{border-color:var(--pb-interactive);outline:none}.series-domain-alt-add{padding:.3rem .75rem;font-size:.78rem}.series-domain-issues-wrap{overflow:visible}.series-domain-issues-card{padding-top:.875rem}.series-domain-issues-toolbar{flex-wrap:wrap;justify-content:space-between;align-items:flex-start;gap:.75rem;display:flex}.series-domain-issues-toolbar-left{flex-wrap:wrap;align-items:center;gap:.5rem;display:flex}.series-domain-issues-title{color:var(--pb-text-primary);font-size:.88rem;font-weight:700}.series-domain-issues-summary{flex-wrap:wrap;align-items:center;gap:.35rem;display:flex}.series-domain-issues-summary .pill{font-size:.62rem}.series-domain-issues-select.dropdown-select{--pb-control-min-height:34px;--pb-control-font-size:.78rem;width:auto}.series-domain-issues-select .dropdown-select-trigger{background:var(--pb-surface-input);padding-block:.4rem;padding-inline:.75rem .65rem}.series-domain-issues-progress{align-items:center;gap:.625rem;margin-top:.9rem;display:flex}.series-domain-issues-progress-track{background:var(--pb-text-secondary);border-radius:2px;flex:1;height:8px}@supports (color:color-mix(in lab, red, red)){.series-domain-issues-progress-track{background:color-mix(in srgb, var(--pb-text-secondary) 6%, transparent)}}.series-domain-issues-progress-track{border:1px solid var(--pb-border-subtle);overflow:hidden}.series-domain-issues-progress-fill{background:linear-gradient(90deg, var(--pb-status-success), var(--pb-status-success));height:100%}@supports (color:color-mix(in lab, red, red)){.series-domain-issues-progress-fill{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-success) 45%, transparent), var(--pb-status-success))}}.series-domain-issues-progress-label{text-align:right;min-width:36px;color:var(--pb-status-success);font-family:JetBrains Mono,monospace;font-size:.78rem;font-weight:700}.series-domain-issues-table-wrap{margin-top:.875rem;overflow-x:auto}.series-domain-issues-table{border-collapse:collapse;width:100%}.series-domain-issues-table th,.series-domain-issues-table td{border-bottom:1px solid var(--pb-border-subtle);padding:.75rem .875rem}.series-domain-issues-table thead th{text-align:left;text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);background:var(--pb-surface-shell);border-bottom:2px solid var(--pb-border-default);font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.series-domain-issues-table tbody tr:hover td{background:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.series-domain-issues-table tbody tr:hover td{background:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.series-domain-table-head.r{text-align:right}.series-domain-issue-num{white-space:nowrap;color:var(--pb-text-secondary);font-family:JetBrains Mono,monospace;font-size:.84rem}.series-domain-issue-title-cell{min-width:240px;color:var(--pb-text-secondary);font-size:.84rem}.series-domain-issue-title-link{color:var(--pb-text-primary);transition:color .14s;display:block}.series-domain-issue-title-link:hover{color:var(--pb-interactive)}.series-domain-issue-date{white-space:nowrap;color:var(--pb-text-tertiary);font-size:.8rem}.series-domain-issue-reading{min-width:9.5rem;color:var(--pb-text-secondary);font-family:JetBrains Mono,monospace;font-size:.7rem}.series-domain-issue-reading-mobile{color:var(--pb-interactive);margin-top:.3rem;font-family:JetBrains Mono,monospace;font-size:.65rem}.series-domain-reading-progress{gap:.28rem;max-width:8.5rem;display:grid}.series-domain-reading-progress-track{background:var(--pb-border-default);border-radius:999px;height:3px;display:block;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.series-domain-reading-progress-track{background:color-mix(in srgb, var(--pb-border-default) 68%, transparent)}}.series-domain-reading-progress-fill{border-radius:inherit;background:var(--pb-interactive);height:100%;display:block}.series-domain-reading-queue{color:var(--pb-text-tertiary);margin-top:.3rem;font-family:DM Sans,sans-serif;font-size:.62rem;display:block}.series-domain-issue-status{white-space:nowrap}.series-domain-issue-actions-cell{text-align:right}.series-domain-issue-actions{align-items:center;gap:.25rem;display:inline-flex}.series-domain-issue-action-btn{width:28px;height:28px;color:var(--pb-text-tertiary);background:0 0;border:1px solid #0000;border-radius:8px;justify-content:center;align-items:center;transition:all .14s;display:inline-flex}.series-domain-issue-action-btn:hover{color:var(--pb-interactive);background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-domain-issue-action-btn:hover{background:color-mix(in srgb, var(--pb-interactive) 6%, transparent)}}.series-domain-reading-menu{z-index:60;border:1px solid var(--pb-border-default);background:var(--pb-surface-raised);min-width:10.5rem;box-shadow:var(--pb-shadow-2);border-radius:10px;display:grid;position:absolute;top:calc(100% + .3rem);right:0;overflow:hidden}.series-domain-reading-menu button{text-align:left;min-height:2.5rem;color:var(--pb-text-secondary);padding:.55rem .75rem;font-size:.72rem;font-weight:600}.series-domain-reading-menu button:hover,.series-domain-reading-menu button:focus-visible{color:var(--pb-text-primary);background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-domain-reading-menu button:hover,.series-domain-reading-menu button:focus-visible{background:color-mix(in srgb, var(--pb-interactive) 9%, transparent)}}.series-domain-table-footer{margin-top:.75rem}.series-domain-copy-block{color:var(--pb-text-secondary);margin-top:.75rem}.issue-domain-stat-strip{grid-template-columns:repeat(4,minmax(0,1fr));gap:.625rem;margin-top:1rem;display:grid}.issue-domain-stat-box{background:var(--pb-surface-shell);border:1px solid var(--pb-border-subtle);border-radius:12px;padding:.8rem .9rem}.issue-domain-stat-label{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.issue-domain-stat-value{color:var(--pb-text-primary);margin-top:.3rem;font-family:JetBrains Mono,monospace;font-size:.85rem}.issue-domain-stat-value a{color:var(--pb-interactive)}.issue-domain-creators-grid{grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:.75rem;margin-top:.75rem;display:grid}.issue-domain-creator-card{background:var(--pb-surface-app);border:1px solid var(--pb-border-subtle);border-radius:10px;padding:.8rem .9rem}.issue-domain-creator-name{color:var(--pb-text-primary);font-size:.85rem;font-weight:600}.issue-domain-creator-role{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);margin-top:.2rem;font-size:.68rem}.issue-domain-file-grid{grid-template-columns:repeat(3,minmax(0,1fr));gap:.75rem;margin-top:.75rem;display:grid}.issue-domain-file-name{font-size:.78rem}.issue-domain-file-path-box{background:var(--pb-surface-app);border:1px solid var(--pb-border-subtle);border-radius:12px;justify-content:space-between;align-items:flex-start;gap:.75rem;margin-top:.75rem;padding:.875rem 1rem;display:flex}.issue-domain-file-path{word-break:break-all;color:var(--pb-text-secondary);margin-top:.3rem;font-family:JetBrains Mono,monospace;font-size:.75rem;line-height:1.7}.issue-domain-copy-btn{border:1px solid var(--pb-border-default);background:var(--pb-surface-card);width:30px;height:30px;color:var(--pb-text-tertiary);border-radius:8px;justify-content:center;align-items:center;transition:all .14s;display:inline-flex}.issue-domain-copy-btn:hover{border-color:var(--pb-interactive);color:var(--pb-interactive)}.series-domain-telemetry-strip{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);color:var(--pb-text-tertiary);box-shadow:var(--pb-shadow-1);border-radius:12px;flex-wrap:wrap;align-items:center;gap:.35rem 1rem;padding:.65rem 1rem;font-family:JetBrains Mono,monospace;font-size:.7rem;display:flex}.series-domain-telemetry-strip strong{color:var(--pb-text-primary);font-weight:700}.series-page-shell{gap:calc(var(--spacing) * 4);flex-direction:column;display:flex}.series-registry-header{flex-wrap:wrap;justify-content:space-between;align-items:flex-start;gap:1.25rem;display:flex}.series-registry-header-left{flex-wrap:wrap;align-items:flex-start;gap:1.5rem;display:flex}.series-registry-title{text-transform:uppercase;letter-spacing:.04em;color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:1.6rem;font-weight:800;line-height:1}.series-registry-title span{color:var(--pb-brand)}.series-registry-subtitle{color:var(--pb-text-tertiary);letter-spacing:.01em;margin-top:.25rem;font-size:.78rem}.series-registry-gauges{align-items:flex-end;gap:1rem;display:flex}.series-registry-gauge{text-align:center}.series-registry-gauge-ring{width:56px;height:56px;margin:0 auto;position:relative}.series-registry-gauge-ring svg{transform:rotate(-90deg)}.series-registry-gauge-bg{fill:none;stroke:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.series-registry-gauge-bg{stroke:color-mix(in srgb, var(--pb-text-secondary) 10%, transparent)}}.series-registry-gauge-bg{stroke-width:4.5px}.series-registry-gauge-fill{fill:none;stroke-width:4.5px;stroke-linecap:round;transition:stroke-dashoffset .6s}.series-registry-gauge-fill-success{stroke:var(--pb-status-success)}.series-registry-gauge-fill-info{stroke:var(--pb-status-info)}.series-registry-gauge-fill-warning{stroke:var(--pb-status-warning)}.series-registry-gauge-fill-danger{stroke:var(--pb-status-danger)}.series-registry-gauge-fill-default{stroke:var(--pb-text-secondary)}.series-registry-gauge-value{justify-content:center;align-items:center;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:700;display:flex;position:absolute;inset:0}.series-registry-gauge-label{text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);margin-top:.25rem;font-size:.6rem;font-weight:600}.series-registry-actions{flex-wrap:wrap;align-items:center;gap:.5rem;display:flex}.series-toolbar-shell{z-index:20;position:sticky;top:0}.series-toolbar-frame{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);overflow:visible}.series-toolbar-body{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}.series-toolbar-browse{flex-wrap:wrap;align-items:flex-end;gap:.5rem;display:flex}.series-toolbar-primary{flex-wrap:wrap;flex:1;align-items:flex-end;gap:.5rem;min-width:0;display:flex}.series-toolbar-actions{flex-wrap:wrap;align-items:flex-end;gap:.5rem;display:flex}.series-toolbar-field{flex-shrink:0;display:block}.series-toolbar-label{text-transform:uppercase;letter-spacing:.08em;color:var(--pb-text-tertiary);margin-bottom:.25rem;font-size:.68rem;font-weight:700;display:block}.series-registry-search{width:220px;max-width:100%}.add-series-search-field{width:275px}.series-registry-search.search-field{border-color:var(--pb-border-default);background:var(--pb-surface-input);min-height:38px;box-shadow:none;border-radius:8px}.series-registry-search .search-field-input{font-size:.82rem}.series-registry-dropdown.dropdown-select{--pb-control-min-height:38px;--pb-control-font-size:.78rem;--pb-control-radius:8px;width:fit-content;max-width:100%}.series-registry-dropdown .dropdown-select-trigger{border-color:var(--pb-border-default);background:var(--pb-surface-input);box-shadow:none;padding-inline:.75rem}.series-registry-dropdown .dropdown-select-trigger-label{color:var(--pb-text-primary);font-weight:500}.series-registry-dropdown .dropdown-select-panel{border-radius:10px}.series-toolbar-view-block{flex-direction:column;display:inline-flex}.series-view-toggle{border:1px solid var(--pb-border-default);background:0 0;border-radius:8px;display:inline-flex;overflow:visible}.series-view-toggle-item:first-child{border-top-left-radius:7px;border-bottom-left-radius:7px}.series-view-toggle-item:last-child{border-top-right-radius:7px;border-bottom-right-radius:7px}.series-view-toggle-item{width:38px;height:34px;color:var(--pb-text-tertiary);background:0 0;border:none;justify-content:center;align-items:center;transition:all .14s;display:inline-flex}.series-view-toggle-item+.series-view-toggle-item{border-left:1px solid var(--pb-border-default)}.series-view-toggle-item:hover{background:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.series-view-toggle-item:hover{background:color-mix(in srgb, var(--pb-text-secondary) 4%, transparent)}}.series-view-toggle-item:hover{color:var(--pb-text-primary)}html[data-series-view=list] .series-view-toggle-item[data-series-view-option=list],html[data-series-view=grid] .series-view-toggle-item[data-series-view-option=grid],html[data-story-arc-view=list] .series-view-toggle-item[data-story-arc-view-option=list],html[data-story-arc-view=grid] .series-view-toggle-item[data-story-arc-view-option=grid],html[data-whats-new-view=list] .whats-new-view-toggle-item[data-whats-new-view-option=list],html[data-whats-new-view=compact] .whats-new-view-toggle-item[data-whats-new-view-option=compact]{background:var(--pb-interactive);color:var(--pb-text-inverse)}.whats-new-pane-toggle{background:var(--pb-surface-input)}.whats-new-pane-toggle-item{min-height:34px;color:var(--pb-text-tertiary);letter-spacing:.06em;text-transform:uppercase;background:0 0;border:none;justify-content:center;align-items:center;padding:0 .875rem;font-size:.78rem;font-weight:700;transition:all .14s;display:inline-flex}.whats-new-pane-toggle-item:first-child{border-top-left-radius:7px;border-bottom-left-radius:7px}.whats-new-pane-toggle-item:last-child{border-top-right-radius:7px;border-bottom-right-radius:7px}.whats-new-pane-toggle-item+.whats-new-pane-toggle-item{border-left:1px solid var(--pb-border-default)}.whats-new-pane-toggle-item:hover{background:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.whats-new-pane-toggle-item:hover{background:color-mix(in srgb, var(--pb-text-secondary) 4%, transparent)}}.whats-new-pane-toggle-item:hover{color:var(--pb-text-primary)}.whats-new-pane-toggle-item.is-active,.whats-new-pane-toggle-item[aria-current=page]{background:var(--pb-interactive);color:var(--pb-text-inverse)}.whats-new-title-block{min-height:4.55rem}.whats-new-cache-badge{min-height:34px;color:var(--pb-text-inverse);align-items:center;display:inline-flex}.whats-new-cache-badge-success{background:#1f6f4a}.whats-new-cache-badge-error{background:#9f2f24}.whats-new-week-nav{border:1px solid var(--pb-border-default);background:var(--pb-surface-input);border-radius:10px;justify-content:center;justify-self:center;align-items:center;gap:.375rem;width:fit-content;max-width:min(100%,34rem);margin-inline:auto;padding:.375rem;display:inline-flex}.whats-new-week-nav-button{border:1px solid var(--pb-border-default);background:var(--pb-surface-card);min-width:32px;min-height:32px;color:var(--pb-text-secondary);border-radius:8px;justify-content:center;align-items:center;font-size:.9rem;font-weight:800;transition:all .14s;display:inline-flex}.whats-new-week-nav-button:hover{border-color:var(--pb-border-hover);background:var(--pb-interactive-selected);color:var(--pb-text-primary)}.whats-new-week-nav-button.is-disabled{opacity:.45;cursor:not-allowed}.whats-new-week-dropdown.dropdown-select{--pb-control-min-height:32px;--pb-control-font-size:.78rem;--pb-control-radius:8px;flex:0 auto;width:fit-content;max-width:min(52vw,14rem)}.whats-new-week-dropdown .dropdown-select-trigger{border-color:var(--pb-border-default);background:var(--pb-surface-card);min-width:9.5rem;max-width:14rem;box-shadow:none;padding-inline:.625rem}.whats-new-week-dropdown .dropdown-select-panel{border-radius:10px;min-width:11.5rem;max-width:min(90vw,16rem)}.whats-new-week-dropdown .dropdown-select-trigger-label,.whats-new-week-dropdown .dropdown-select-option-label{font-weight:700}.whats-new-week-position{color:var(--pb-text-tertiary);letter-spacing:.08em;text-transform:uppercase;flex:none;padding-inline:.25rem .375rem;font-size:.68rem;font-weight:700}@media (max-width:640px){.whats-new-week-nav{max-width:100%}.whats-new-week-dropdown.dropdown-select{max-width:min(58vw,13rem)}.whats-new-week-position{text-align:center;width:100%}}.whats-new-results-stack,.whats-new-panel{gap:1rem;display:grid}.whats-new-stale-banner{border:1px solid var(--pb-status-warning);justify-content:space-between;align-items:center;gap:1rem;display:flex}@supports (color:color-mix(in lab, red, red)){.whats-new-stale-banner{border:1px solid color-mix(in srgb, var(--pb-status-warning) 40%, transparent)}}.whats-new-stale-banner{background:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.whats-new-stale-banner{background:color-mix(in srgb, var(--pb-status-warning) 10%, transparent)}}.whats-new-stale-banner{color:var(--pb-text-primary);border-radius:12px;margin-bottom:1rem;padding:.75rem 1rem;font-size:.82rem}.whats-new-stale-copy{min-width:0}.whats-new-stale-actions{flex-direction:column;flex:none;align-items:flex-end;gap:.375rem;display:flex}.whats-new-stale-message{max-width:24rem;color:var(--pb-text-secondary);text-align:right;font-size:.72rem;line-height:1.35}@media (max-width:640px){.whats-new-stale-banner{flex-direction:column;align-items:stretch}.whats-new-stale-actions{align-items:flex-start}.whats-new-stale-message{text-align:left}}.whats-new-release-summary{margin-bottom:0}.whats-new-release-table-slot{min-width:0}html[data-whats-new-view=compact] .whats-new-release-cover,html[data-whats-new-view=compact] .whats-new-pulls-col{display:none}html[data-whats-new-view=compact] .whats-new-release-title-cell{gap:0}.select-btn{border:1px solid var(--pb-border-default);min-height:38px;color:var(--pb-text-secondary);background:0 0;border-radius:8px;justify-content:center;align-items:center;gap:.375rem;padding:.375rem .875rem;font-family:DM Sans,sans-serif;font-size:.78rem;font-weight:600;transition:all .14s;display:inline-flex}.select-btn:hover{border-color:var(--pb-interactive);color:var(--pb-interactive)}.select-btn:disabled{cursor:not-allowed;opacity:.45}.select-btn-success{color:var(--pb-status-success);border-color:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.select-btn-success{border-color:color-mix(in srgb, var(--pb-status-success) 30%, transparent)}}.select-btn-success{background:var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.select-btn-success{background:color-mix(in srgb, var(--pb-status-success) 8%, transparent)}}.select-btn-warning{color:var(--pb-status-warning);border-color:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.select-btn-warning{border-color:color-mix(in srgb, var(--pb-status-warning) 30%, transparent)}}.select-btn-warning{background:var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.select-btn-warning{background:color-mix(in srgb, var(--pb-status-warning) 8%, transparent)}}.select-btn-danger{color:var(--pb-status-danger);border-color:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){.select-btn-danger{border-color:color-mix(in srgb, var(--pb-status-danger) 30%, transparent)}}.select-btn-danger{background:var(--pb-status-danger)}@supports (color:color-mix(in lab, red, red)){.select-btn-danger{background:color-mix(in srgb, var(--pb-status-danger) 8%, transparent)}}.series-selection-shell{flex-direction:column;gap:.625rem;display:flex}.series-selection-inline{align-items:center;display:inline-flex}.series-selection-count{color:var(--pb-text-secondary);font-family:JetBrains Mono,monospace;font-size:.78rem;font-weight:600}.series-selection-controls-row{flex-wrap:wrap;justify-content:space-between;gap:.75rem;display:flex}.series-selection-bulk,.series-selection-actions{flex-wrap:wrap;align-items:center;gap:.5rem;display:flex}.series-results-shell{gap:calc(var(--spacing) * 4);min-height:16rem;padding-bottom:var(--pb-page-footer-clearance);flex-direction:column;flex:none;display:flex}.series-results-body-shell{min-height:14rem}:where(:is(.series-mission-control-shell,.series-collector-wall-shell)>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.series-mission-control-table-wrap{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);border-radius:14px;overflow:auto hidden}.series-mission-control-table{border-collapse:collapse;width:100%}.series-mission-control-table th{text-align:left;text-transform:uppercase;letter-spacing:.1em;color:var(--pb-text-tertiary);background:var(--pb-surface-shell);border-bottom:2px solid var(--pb-border-default);padding:.5625rem .875rem;font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.series-mission-control-table td{border-bottom:1px solid var(--pb-border-subtle);padding:.6875rem .875rem;font-size:.82rem;transition:background .1s}.series-mission-control-table tbody tr:last-child td{border-bottom:none}.series-mission-control-table tbody tr:not(.table-detail-row):hover td{background:var(--pb-surface-selected)}.series-mission-control-table th.c,.series-mission-control-table td.c{text-align:center}.series-mission-control-table th.r,.series-mission-control-table td.r{text-align:right}.series-mission-control-row-selected td{background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-mission-control-row-selected td{background:color-mix(in srgb, var(--pb-interactive) 10%, transparent)}}.series-led{border-radius:50%;width:8px;height:8px;display:inline-block}.series-led-green{background:var(--pb-status-success);box-shadow:0 0 5px var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.series-led-green{box-shadow:0 0 5px color-mix(in srgb, var(--pb-status-success) 35%, transparent)}}.series-led-amber{background:var(--pb-status-warning);box-shadow:0 0 5px var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.series-led-amber{box-shadow:0 0 5px color-mix(in srgb, var(--pb-status-warning) 35%, transparent)}}.series-led-off{background:var(--pb-text-tertiary);opacity:.35}.series-mission-control-name{color:var(--pb-text-primary);font-weight:700;transition:color .14s}.series-mission-control-name:hover{color:var(--pb-interactive)}.series-mission-control-publisher{color:var(--pb-text-tertiary);margin-top:1px;font-size:.72rem}.series-mission-control-year{color:var(--pb-text-tertiary);font-size:.78rem}.series-mission-control-bar-cell{width:220px}.series-mission-control-bar{align-items:center;gap:.5rem;display:flex}.series-mission-control-bar-track{background:var(--pb-text-secondary);border-radius:2px;flex:1;height:8px}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-track{background:color-mix(in srgb, var(--pb-text-secondary) 6%, transparent)}}.series-mission-control-bar-track{border:1px solid var(--pb-border-subtle);position:relative;overflow:hidden}.series-mission-control-bar-fill{height:100%;position:relative}.series-mission-control-bar-fill:after{content:"";background:repeating-linear-gradient(90deg,#0000 0 2px,#ffffff1f 2px 4px);position:absolute;inset:0}.series-mission-control-bar-fill-green{background:linear-gradient(90deg, var(--pb-status-success), var(--pb-status-success))}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-fill-green{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-success) 45%, transparent), var(--pb-status-success))}}.series-mission-control-bar-fill-amber{background:linear-gradient(90deg, var(--pb-status-warning), var(--pb-status-warning))}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-fill-amber{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-warning) 45%, transparent), var(--pb-status-warning))}}.series-mission-control-bar-fill-red{background:linear-gradient(90deg, var(--pb-status-danger), var(--pb-status-danger))}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-fill-red{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-danger) 45%, transparent), var(--pb-status-danger))}}.series-mission-control-bar-fill-blue{background:linear-gradient(90deg, var(--pb-status-info), var(--pb-status-info))}@supports (color:color-mix(in lab, red, red)){.series-mission-control-bar-fill-blue{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-status-info) 45%, transparent), var(--pb-status-info))}}.series-mission-control-bar-pct{text-align:right;font-variant-numeric:tabular-nums;min-width:36px;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:700}.series-mission-control-bar-pct-green{color:var(--pb-status-success)}.series-mission-control-bar-pct-amber{color:var(--pb-status-warning)}.series-mission-control-bar-pct-red{color:var(--pb-status-danger)}.series-mission-control-bar-pct-blue{color:var(--pb-status-info)}.series-mission-control-actions{justify-content:flex-end;gap:.25rem;display:flex}.series-mission-control-action-btn{border:1px solid var(--pb-border-subtle);width:30px;height:30px;color:var(--pb-text-tertiary);background:0 0;border-radius:8px;justify-content:center;align-items:center;transition:all .14s;display:inline-flex}.series-mission-control-action-btn:hover{border-color:var(--pb-interactive);color:var(--pb-interactive);background:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-mission-control-action-btn:hover{background:color-mix(in srgb, var(--pb-interactive) 4%, transparent)}}.series-mission-control-action-btn.disabled{opacity:.25;pointer-events:none}.series-mission-control-action-btn svg{width:14px;height:14px}.series-mission-control-footer{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);color:var(--pb-text-tertiary);border-radius:10px;flex-wrap:wrap;align-items:center;gap:.375rem 1.5rem;padding:.625rem 1rem;font-family:JetBrains Mono,monospace;font-size:.7rem;display:flex}.series-mission-control-footer strong{color:var(--pb-text-primary);font-weight:600}.series-collector-wall-grid{grid-template-columns:repeat(auto-fill,minmax(172px,1fr));gap:24px 18px;display:grid}.series-wall-card{min-width:0;transition:transform .22s;position:relative}.series-wall-card:hover{transform:translateY(-5px)}.series-wall-card-selected .series-wall-cover-wrap{outline:2px solid var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-wall-card-selected .series-wall-cover-wrap{outline:2px solid color-mix(in srgb, var(--pb-interactive) 28%, transparent)}}.series-wall-card-selected .series-wall-cover-wrap{outline-offset:2px}.series-wall-cover-wrap{aspect-ratio:2/3;border:1px solid var(--pb-border-subtle);background:var(--pb-surface-card);box-shadow:var(--pb-shadow-1);border-radius:14px;transition:box-shadow .22s;position:relative;overflow:hidden}.series-wall-card:hover .series-wall-cover-wrap{box-shadow:var(--pb-shadow-2), 0 0 20px var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.series-wall-card:hover .series-wall-cover-wrap{box-shadow:var(--pb-shadow-2), 0 0 20px color-mix(in srgb, var(--pb-interactive) 6%, transparent)}}.series-wall-cover-placeholder{width:100%;height:100%;color:var(--pb-text-tertiary);background:linear-gradient(155deg, var(--pb-surface-card) 0%, var(--pb-surface-app) 100%);justify-content:center;align-items:center;display:flex}.series-monitor-badge{width:1.5rem;height:1.5rem;color:var(--pb-text-inverse);background:var(--pb-brand-signal);border:2px solid var(--pb-surface-card);border-radius:999px;flex:none;justify-content:center;align-items:center;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.series-monitor-badge{border:2px solid color-mix(in srgb, var(--pb-surface-card) 92%, white)}}.series-monitor-badge{box-shadow:0 0 0 2px #0000006b,0 10px 22px #00000057}.series-monitor-badge svg{flex:none;width:.9rem;height:.9rem}.series-monitor-badge.is-paused{color:var(--pb-text-secondary);background:var(--pb-surface-raised);border-color:var(--pb-border-default);box-shadow:0 0 0 1px var(--pb-border-subtle)}.series-wall-monitor-dot{z-index:3;position:absolute;top:9px;left:9px}.story-arc-wall-review{z-index:4;border:1px solid var(--pb-border-default);background:var(--pb-surface-card);border-radius:999px;justify-content:center;align-items:center;gap:.2rem;min-width:1.75rem;height:1.75rem;padding-inline:.38rem;display:inline-flex;position:absolute;top:9px;right:9px}@supports (color:color-mix(in lab, red, red)){.story-arc-wall-review{background:color-mix(in srgb, var(--pb-surface-card) 92%, transparent)}}.story-arc-wall-review{box-shadow:var(--pb-shadow-1);font-family:JetBrains Mono,monospace;font-size:.65rem;font-weight:700}.story-arc-wall-review svg{width:.8rem;height:.8rem}.story-arc-wall-review-warning{color:var(--pb-warning);border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.story-arc-wall-review-warning{border-color:color-mix(in srgb, var(--pb-warning) 48%, transparent)}}.story-arc-wall-review-error{color:var(--pb-error);border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.story-arc-wall-review-error{border-color:color-mix(in srgb, var(--pb-error) 48%, transparent)}}.story-arc-review-indicator{border:1px solid var(--pb-border-default);border-radius:999px;justify-content:center;align-items:center;gap:.28rem;min-width:2.5rem;min-height:1.75rem;padding-inline:.45rem;font-family:JetBrains Mono,monospace;font-size:.68rem;font-weight:700;display:inline-flex}.story-arc-review-indicator svg{width:.82rem;height:.82rem}.story-arc-review-indicator-success{color:var(--pb-success);background:var(--pb-success-dim);border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.story-arc-review-indicator-success{border-color:color-mix(in srgb, var(--pb-success) 32%, transparent)}}.story-arc-review-indicator-warning{color:var(--pb-warning);background:var(--pb-warning-dim);border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.story-arc-review-indicator-warning{border-color:color-mix(in srgb, var(--pb-warning) 32%, transparent)}}.story-arc-review-indicator-error{color:var(--pb-error);background:var(--pb-error-dim);border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.story-arc-review-indicator-error{border-color:color-mix(in srgb, var(--pb-error) 32%, transparent)}}.series-wall-catalog-state-badge{z-index:4;border:1px solid var(--pb-warning);background:var(--pb-warning);border-radius:999px;justify-content:center;align-items:center;max-width:calc(100% - 18px);display:inline-flex;position:absolute;top:46px;right:9px}@supports (color:color-mix(in lab, red, red)){.series-wall-catalog-state-badge{background:color-mix(in srgb, var(--pb-warning) 92%, black)}}.series-wall-catalog-state-badge{color:var(--pb-text-inverse);letter-spacing:.04em;text-align:center;text-transform:uppercase;padding:.2rem .5rem;font-size:.58rem;font-weight:800;line-height:1.15;box-shadow:0 8px 18px #00000057}.series-wall-catalog-state-badge.is-failed{border-color:var(--pb-error);background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.series-wall-catalog-state-badge.is-failed{background:color-mix(in srgb, var(--pb-error) 92%, black)}}.pull-list-monitor-toggle{width:28px;height:28px;box-shadow:none;cursor:pointer;border-width:1px;border-radius:6px;transition:background-color .14s,border-color .14s,color .14s,box-shadow .14s}.pull-list-monitor-toggle svg{width:12px;height:12px}.pull-list-monitor-toggle:hover{border-color:var(--pb-brand-signal)}@supports (color:color-mix(in lab, red, red)){.pull-list-monitor-toggle:hover{border-color:color-mix(in srgb, var(--pb-brand-signal) 56%, transparent)}}.pull-list-monitor-toggle:hover{background:var(--pb-brand-signal);color:var(--pb-text-inverse);box-shadow:0 0 0 2px var(--pb-brand-signal)}@supports (color:color-mix(in lab, red, red)){.pull-list-monitor-toggle:hover{box-shadow:0 0 0 2px color-mix(in srgb, var(--pb-brand-signal) 14%, transparent)}}.pull-list-monitor-toggle:focus-visible{outline:2px solid var(--pb-focus-outline);outline-offset:3px}.series-wall-selection-control{z-index:4;border:1px solid var(--pb-border-default);background:var(--pb-surface-card);border-radius:10px;position:absolute;top:10px;right:10px}@supports (color:color-mix(in lab, red, red)){.series-wall-selection-control{background:color-mix(in srgb, var(--pb-surface-card) 92%, transparent)}}.series-wall-selection-control{box-shadow:var(--pb-shadow-1);padding:.25rem}.series-wall-ring{z-index:2;width:44px;height:44px;position:absolute;bottom:-6px;right:-6px}.series-wall-ring svg{transform:rotate(-90deg)}.series-wall-ring-bg{fill:none;stroke:var(--pb-surface-card);stroke-width:3.5px}.series-wall-ring-fill{fill:none;stroke-width:3.5px;stroke-linecap:round;transition:stroke-dashoffset .6s}.series-wall-ring-fill-green{stroke:var(--pb-status-success)}.series-wall-ring-fill-amber{stroke:var(--pb-status-warning)}.series-wall-ring-fill-red{stroke:var(--pb-status-danger)}.series-wall-ring-center{background:var(--pb-surface-card);color:var(--pb-text-primary);border-radius:999px;justify-content:center;align-items:center;margin:5px;font-family:JetBrains Mono,monospace;font-size:.58rem;font-weight:700;display:flex;position:absolute;inset:0}.series-wall-overlay{--series-wall-overlay-edge:#fff;z-index:3;-webkit-backdrop-filter:blur(6px);backdrop-filter:blur(6px);box-shadow:inset 0 0 0 4.5px var(--series-wall-overlay-edge);opacity:0;pointer-events:none;background:#1e1a17e0;border-radius:18px;flex-direction:column;justify-content:flex-end;padding:.875rem;transition:opacity .2s;display:flex;position:absolute;inset:-4px}[data-theme=light] .series-wall-overlay{--series-wall-overlay-edge:#1e1a17}@media (prefers-color-scheme:light){:root:not([data-theme]) .series-wall-overlay{--series-wall-overlay-edge:#1e1a17}}.series-wall-card:hover .series-wall-overlay{opacity:1}.series-wall-overlay-title{color:var(--pb-text-primary);font-size:.82rem;font-weight:700;line-height:1.3}.series-wall-overlay-meta{color:var(--pb-text-tertiary);gap:.25rem;margin-top:.45rem;font-size:.68rem;display:grid}.series-wall-overlay-meta-line,.series-wall-overlay-stat{grid-template-columns:minmax(4.25rem,max-content) minmax(0,1fr);align-items:baseline;column-gap:.5rem;line-height:1.25;display:grid}.series-wall-overlay-meta-label{color:var(--pb-text-tertiary);letter-spacing:.08em;text-transform:uppercase;font-family:JetBrains Mono,monospace;font-size:.56rem;font-weight:700}.series-wall-overlay-meta-value{overflow-wrap:anywhere;min-width:0;color:var(--pb-text-secondary)}.series-wall-overlay-stats{color:var(--pb-text-secondary);gap:.25rem;margin-top:.35rem;font-size:.68rem;display:grid}.series-wall-overlay-stats strong{color:var(--pb-text-primary);font-weight:600}.series-wall-overlay-actions{pointer-events:auto;gap:6px;margin-top:10px;display:flex}.series-wall-overlay-btn{color:var(--pb-text-secondary);text-align:center;background:#ffffff0f;border:1px solid #cbd5e140;border-radius:8px;flex:1;justify-content:center;padding:7px;font-family:DM Sans,sans-serif;font-size:.72rem;font-weight:600;transition:all .14s;display:inline-flex}.series-wall-overlay-btn:hover{background:var(--pb-interactive);color:var(--pb-text-primary);border-color:var(--pb-interactive)}.series-wall-card-title{text-overflow:ellipsis;white-space:nowrap;color:var(--pb-text-primary);margin-top:10px;font-size:.82rem;font-weight:600;line-height:1.3;display:block;overflow:hidden}.series-empty-state{justify-content:center;align-items:center;gap:calc(var(--spacing) * 3);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 12);text-align:center;box-shadow:var(--pb-shadow-1);border-radius:1.25rem;flex-direction:column;display:flex}.series-empty-state-title{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.series-empty-state-copy{max-width:var(--container-xl);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}@media (max-width:1180px){.series-domain-hero-inner{grid-template-columns:130px minmax(0,1fr)}.issue-domain-hero-inner{grid-template-columns:120px minmax(0,1fr)}.series-domain-actions-panel{border-top:1px solid var(--pb-border-subtle);border-left:0;grid-column:1/-1;padding-top:1rem;padding-left:0}.series-domain-actions-panel .series-domain-actions-buttons{grid-template-columns:repeat(auto-fit,minmax(12rem,1fr));align-items:stretch;display:grid}}@media (max-width:900px){.issue-domain-stat-strip{grid-template-columns:repeat(2,minmax(0,1fr))}.issue-domain-file-grid{grid-template-columns:1fr}}@media (max-width:640px){.detail-hero-shell,.series-domain-actions-card,.series-domain-section-card{padding:1rem}.series-domain-cover-column,.issue-domain-cover-column,.issue-domain-cover-frame{width:104px}.series-domain-hero-inner,.issue-domain-hero-inner{grid-template-columns:104px minmax(0,1fr)}.series-domain-hero-title{font-size:1.2rem}.series-domain-actions-inner{align-items:flex-start}.series-domain-actions-divider{display:none}.series-domain-actions-panel .series-domain-actions-buttons,.issue-domain-stat-strip{grid-template-columns:1fr}.series-domain-info-box-wide{grid-column:auto}.series-domain-issues-toolbar{flex-direction:column;align-items:stretch}}[data-series-page][data-series-toolbar-mode=browse] [data-testid=series-select-toolbar],[data-series-page][data-series-toolbar-mode=select] [data-testid=series-browse-toolbar]{display:none}[data-series-page][data-series-toolbar-mode=select] [data-testid=series-select-toolbar]{display:flex}[data-select-toolbar-page][data-toolbar-mode=browse] [data-select-toolbar],[data-select-toolbar-page][data-toolbar-mode=select] [data-browse-toolbar]{display:none}[data-select-toolbar-page][data-toolbar-mode=select] [data-select-toolbar]{display:flex}[data-series-page][data-series-toolbar-mode=browse] [data-series-selection-cell]{display:none}[data-series-page][data-series-toolbar-mode=select] [data-series-selection-cell]{display:table-cell}[data-series-page][data-series-toolbar-mode=browse] [data-series-selection-control]{display:none}[data-series-page][data-series-toolbar-mode=select] [data-series-selection-control]{display:block}[data-series-page][data-series-toolbar-mode=select] [data-series-selection-control=grid]{display:inline-flex}#page-footer-dock:empty{display:none}body:has(#page-footer-dock:not(:empty)) #content{padding-bottom:var(--pb-page-footer-clearance)}body:has(#page-footer-dock:not(:empty)) #content:has(.admin-workspace-page,.dashboard-mission-page,.downloads-view,.series-domain-page,.series-results-shell,.utilities-page){padding-bottom:0}#content:has(.admin-workspace-page){overflow-anchor:none}.page-dock-inner{--page-dock-height-status:2rem;--page-dock-height-pagination:2.5rem;max-width:85rem;height:var(--page-dock-height-status);flex-wrap:nowrap;justify-content:space-between;align-items:center;gap:.75rem;margin:0 auto;padding:0 1rem;display:flex;overflow:hidden}.page-dock-inner:has(.page-dock-pagination){height:var(--page-dock-height-pagination)}.page-footer-clearance{height:var(--pb-page-footer-clearance);flex:0 0 var(--pb-page-footer-clearance);pointer-events:none}.page-dock-inner-status-only{justify-content:flex-end}.page-dock-pagination{flex:none}.page-dock-pagination nav{gap:.125rem}.page-dock-pagination nav>a,.page-dock-pagination nav>button,.page-dock-pagination nav>span{text-align:center;border-radius:.375rem;min-width:1.75rem;padding:.25rem .625rem;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:600;line-height:1}.page-dock-pagination nav>a,.page-dock-pagination nav>button{appearance:none;color:var(--pb-text-secondary);cursor:pointer;background:0 0;border:0;margin:0}.page-dock-pagination nav>span.bg-pb-interactive{border-radius:.375rem}.page-dock-status{min-width:0;color:var(--pb-text-tertiary);scrollbar-width:none;flex-wrap:nowrap;align-items:center;gap:1rem;font-family:JetBrains Mono,monospace;font-size:.65rem;display:flex;overflow-x:auto}.page-dock-status::-webkit-scrollbar{display:none}.page-dock-status-item{white-space:nowrap;align-items:center;gap:.25rem;display:inline-flex}.page-dock-status-value{color:var(--pb-text-primary);font-weight:600}.page-dock-status-label{text-transform:uppercase;letter-spacing:.04em}.page-dock-led{border-radius:999px;width:6px;height:6px;margin-right:2px;display:inline-block}.page-dock-led-green{background:var(--pb-status-success);box-shadow:0 0 4px var(--pb-status-success)}@supports (color:color-mix(in lab, red, red)){.page-dock-led-green{box-shadow:0 0 4px color-mix(in srgb, var(--pb-status-success) 35%, transparent)}}.page-dock-led-amber{background:var(--pb-status-warning);box-shadow:0 0 4px var(--pb-status-warning)}@supports (color:color-mix(in lab, red, red)){.page-dock-led-amber{box-shadow:0 0 4px color-mix(in srgb, var(--pb-status-warning) 35%, transparent)}}.page-dock-led-off{background:var(--pb-text-tertiary);opacity:.35}@media (max-width:767px){.page-dock-inner{padding-inline:0}.page-dock-status{gap:.75rem}}.workflow-tabs-shell{align-items:center;gap:calc(var(--spacing) * 2);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);border-radius:1.25rem;flex-wrap:wrap;display:flex}@media (min-width:40rem){.workflow-tabs-shell{padding-inline:calc(var(--spacing) * 5)}}.workflow-tabs-shell{box-shadow:var(--pb-shadow-1)}.workflow-tab-chip-active{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary);border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-tab-chip-active{border-color:color-mix(in srgb, var(--pb-interactive) 34%, transparent)}}.workflow-tab-chip-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-tab-chip-active{background-color:color-mix(in srgb, var(--pb-interactive) 12%, var(--pb-bg-card))}}.workflow-tab-chip-active{box-shadow:var(--pb-shadow-1)}.workflow-tab-chip-inactive{color:var(--pb-text-dim)}.workflow-focus-card{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 5);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;position:relative;overflow:hidden}.workflow-focus-card:before{content:"";background:linear-gradient(90deg, transparent, var(--pb-brand), transparent);height:1px;position:absolute;inset:0 0 auto}@supports (color:color-mix(in lab, red, red)){.workflow-focus-card:before{background:linear-gradient(90deg, transparent, color-mix(in srgb, var(--pb-brand) 34%, transparent), transparent)}}.workflow-focus-grid{gap:calc(var(--spacing) * 4);display:grid}@media (min-width:80rem){.workflow-focus-grid{grid-template-columns:minmax(0,1.35fr) minmax(320px,.95fr);align-items:flex-start}}.workflow-focus-chip{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.workflow-focus-actions{gap:calc(var(--spacing) * 3);display:grid}@media (min-width:40rem){.workflow-focus-actions{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:80rem){.workflow-focus-actions{grid-template-columns:repeat(1,minmax(0,1fr))}}:where(.workflow-shell-region>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.workflow-shell-region{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 4);border-radius:1.5rem;overflow:visible}@media (min-width:40rem){.workflow-shell-region{padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 5)}}.workflow-shell-region{box-shadow:var(--pb-shadow-1)}.utility-workspace-shell{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:1.5rem;overflow:hidden}.utility-workspace-shell-visible{overflow:visible}:where(.utility-workspace-body>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.utility-workspace-body{padding:calc(var(--spacing) * 6)}@media (min-width:40rem){.utility-workspace-body{padding:calc(var(--spacing) * 7)}}:where(.utility-tool-body>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}.utility-tool-body{padding:calc(var(--spacing) * 5)}@media (min-width:40rem){.utility-tool-body{padding:calc(var(--spacing) * 5)}}.utility-workspace-shell .info-panel{border-color:var(--pb-border-subtle);background-color:var(--pb-bg-base);box-shadow:none}.utility-workspace-shell .info-panel-muted{background-color:var(--pb-bg-base)}.utility-empty-state{border-radius:var(--radius-xl);border-style:var(--tw-border-style);--tw-border-style:dashed;border-style:dashed;border-width:1px;border-color:var(--pb-border-subtle);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 8);text-align:center;background-color:var(--pb-bg-base)}.utility-path-pill{align-items:center;gap:calc(var(--spacing) * 2);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);display:flex}.utility-token{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 2.5);padding-block:var(--spacing);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);border-radius:3.40282e38px;align-items:center;display:inline-flex}.utility-step-grid{gap:calc(var(--spacing) * 3);display:grid}.utility-step-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);width:100%;padding:calc(var(--spacing) * 4);display:block}.utility-step-card-inline{align-items:center;gap:.75rem;display:flex}.utility-step-card-locked{opacity:.78}.utility-step-num{color:var(--pb-text-tertiary);min-width:1.25rem;font-family:JetBrains Mono,monospace;font-size:.62rem;font-weight:700}.utility-step-name{color:var(--pb-text-primary);flex:auto;min-width:0;font-size:.82rem;font-weight:600}.utility-step-tag{color:var(--pb-text-tertiary);white-space:nowrap;font-family:JetBrains Mono,monospace;font-size:.62rem}.utility-step-card-inline .utility-step-num,.utility-step-card-inline .utility-step-tag{color:var(--pb-text-secondary)}.utility-step-check{flex-shrink:0;margin-top:0}.utility-scan-mode-grid{gap:calc(var(--spacing) * 2);display:grid}@media (min-width:40rem){.utility-scan-mode-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}.utility-scan-mode{justify-content:center;align-items:flex-start;gap:var(--spacing);border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card);min-height:72px;padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);text-align:left;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-direction:column;display:flex}.utility-scan-mode:hover{border-color:var(--pb-border-hover)}.utility-scan-mode-active{border-color:var(--pb-interactive);background-color:var(--pb-surface-selected)}.utility-scan-mode-label{--tw-font-weight:var(--font-weight-semibold);font-size:.82rem;font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.utility-scan-mode-copy{--tw-leading:calc(var(--spacing) * 5);font-size:.72rem;line-height:calc(var(--spacing) * 5);color:var(--pb-text-dim)}.utility-scope-chip-row{flex-wrap:wrap;gap:.375rem;display:flex}.utility-scope-chip{border:1px solid var(--pb-border-default);color:var(--pb-text-secondary);background:0 0;border-radius:.5rem;justify-content:center;align-items:center;padding:.4375rem 1rem;font-size:.78rem;font-weight:600;transition:border-color .14s,background-color .14s,color .14s;display:inline-flex}.utility-scope-chip:hover{border-color:var(--pb-interactive);color:var(--pb-interactive)}.utility-scope-chip-active{background:var(--pb-interactive);color:var(--pb-text-inverse);border-color:var(--pb-interactive)}.utility-scope-chip-active:hover{color:var(--pb-text-inverse)}.utility-export-chip-row{flex-wrap:wrap;gap:.375rem;display:flex}.utility-export-chip{border:1px solid var(--pb-border-default);color:var(--pb-text-secondary);background:0 0;border-radius:999px;justify-content:center;align-items:center;padding:.4375rem .875rem;font-family:DM Sans,sans-serif;font-size:.76rem;font-weight:500;line-height:1.2;transition:border-color .14s,background-color .14s,color .14s;display:inline-flex}.utility-export-chip:hover{border-color:var(--pb-interactive);color:var(--pb-interactive)}.utility-export-chip-active{border-color:var(--pb-interactive);background:var(--pb-interactive);color:var(--pb-text-inverse)}.utility-export-chip-active:hover{color:var(--pb-text-inverse)}.utility-tool-section-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);margin-bottom:.5rem;font-family:Syne,sans-serif;font-size:.62rem;font-weight:700}.utility-tool-section-row{flex-wrap:wrap;justify-content:space-between;align-items:center;gap:.75rem;display:flex}.utility-tool-field-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);font-family:Syne,sans-serif;font-size:.58rem;font-weight:700;display:block}.utility-export-group-grid{gap:.75rem;display:grid}@media (min-width:1024px){.utility-export-group-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}.utility-export-group-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-app);border-radius:12px;flex-direction:column;gap:.5rem;padding:.875rem;display:flex}.utility-export-group-title{color:var(--pb-text-primary);font-size:.72rem;font-weight:600}.utility-export-group-fields{flex-direction:column;gap:.375rem;display:flex}.utility-export-group-fields .utility-step-card{padding:.5rem .75rem}.utility-export-field-label{color:var(--pb-text-primary);font-size:.8rem;font-weight:500;line-height:1.25}.utility-export-field-meta{justify-content:space-between;align-items:baseline;gap:.875rem;width:100%;min-width:0;display:flex}.utility-export-field-sub{color:var(--pb-text-tertiary);text-align:right;white-space:nowrap;text-overflow:ellipsis;flex:0 auto;min-width:0;font-family:JetBrains Mono,monospace;font-size:.66rem;line-height:1.2;overflow:hidden}.utility-export-options-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-app);border-radius:10px;padding:1rem}.utility-export-inline-label{color:var(--pb-text-primary);font-size:.72rem;font-weight:600}.utility-export-multi-grid{gap:.5rem;display:grid}@media (min-width:640px){.utility-export-multi-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}.utility-export-summary-grid{gap:.625rem;display:grid}@media (min-width:640px){.utility-export-summary-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}.utility-export-summary-card{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-app);border-radius:10px;padding:.875rem 1rem}.utility-export-summary-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);font-family:Syne,sans-serif;font-size:.58rem;font-weight:700}.utility-export-summary-value{color:var(--pb-text-primary);margin-top:.35rem;font-family:JetBrains Mono,monospace;font-size:.92rem;font-weight:600}.utility-tool-select.dropdown-select{--pb-control-font-size:.82rem;--pb-control-line-height:1.25rem;--pb-control-radius:8px;width:100%}.utility-tool-select .dropdown-select-trigger{border-color:var(--pb-border-default);background:var(--pb-surface-input);min-height:0;box-shadow:none;padding:.5rem .75rem}.utility-tool-select .dropdown-select-trigger-label{color:var(--pb-text-primary);font-weight:400}.utility-tool-select .dropdown-select-chevron{color:var(--pb-text-secondary)}.utility-tool-output-field{border:1px solid var(--pb-border-subtle);background:var(--pb-surface-app);color:var(--pb-text-secondary);border-radius:8px;padding:.5rem .75rem;font-family:DM Sans,sans-serif;font-size:.82rem;line-height:1.25rem}.utility-tool-browse-button{gap:.375rem;min-height:0;padding:.375rem .875rem;font-size:.78rem;font-weight:600;line-height:1.5}.utility-tool-browse-button svg{width:.875rem;height:.875rem}.utility-tool-table-wrap{border:1px solid var(--pb-border-subtle);border-radius:10px;overflow:hidden}.utility-tool-table{border-collapse:collapse;width:100%}.utility-tool-table th{text-align:left;letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);background:var(--pb-surface-shell);border-bottom:1px solid var(--pb-border-default);padding:.4375rem .75rem;font-family:Syne,sans-serif;font-size:.52rem;font-weight:700}.utility-tool-table th.r,.utility-tool-table td.r{text-align:right}.utility-tool-table td{color:var(--pb-text-secondary);border-bottom:1px solid var(--pb-border-subtle);padding:.5rem .75rem;font-size:.78rem}.utility-tool-table tbody tr:last-child td{border-bottom:none}.utility-tool-table tbody tr:hover td{background:var(--pb-surface-selected)}.utility-tool-table-output{color:var(--pb-status-success)}.utility-tool-action-footer{border-top:1px solid var(--pb-border-subtle);justify-content:flex-end;align-items:center;gap:.625rem;padding:.75rem 0 0;display:flex}.utility-tool-action-footer .settings-footer-actions{gap:.625rem;margin-left:auto}.utility-tool-action-footer .btn-ghost,.utility-tool-action-footer .btn-primary{min-height:0;padding:.5625rem 1.125rem;font-size:.82rem;line-height:1.25rem}.utility-tool-action-footer-card{margin-top:.25rem}.utility-step-card-active{border-color:var(--pb-interactive);background-color:var(--pb-surface-selected)}.utility-step-card-static{border-color:var(--pb-border-subtle);background-color:var(--pb-bg-card)}.utility-empty-state-compact{padding-top:1rem;padding-bottom:1rem}.utility-template-table-type{color:var(--pb-text-primary);font-size:.78rem;font-weight:600}.utility-template-meta-token{color:var(--pb-text-tertiary);font-family:JetBrains Mono,monospace;font-size:.72rem}.utility-template-meta-token strong{color:var(--pb-text-secondary);font-weight:600}.utilities-queue-table-wrap{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:14px;overflow:hidden}.utilities-queue-table{border-collapse:collapse;width:100%}.utilities-queue-table th{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-surface);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);text-align:left;--tw-font-weight:var(--font-weight-bold);font-family:Syne,sans-serif;font-size:.58rem;font-weight:var(--font-weight-bold);--tw-tracking:.1em;letter-spacing:.1em;color:var(--pb-text-dim);text-transform:uppercase}.utilities-queue-table th.c,.utilities-queue-table td.c{text-align:center}.utilities-queue-table th.r,.utilities-queue-table td.r{text-align:right}.utilities-queue-table td{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border-subtle);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);font-size:.82rem}.utilities-queue-table tbody:last-child tr:last-child td{border-bottom:0}.utilities-queue-row:hover td{background-color:var(--pb-surface-selected)}.utilities-queue-row>td{vertical-align:middle}.utilities-queue-led{height:calc(var(--spacing) * 2);width:calc(var(--spacing) * 2);background-color:var(--pb-text-dim);border-radius:3.40282e38px;display:inline-block}@supports (color:color-mix(in lab, red, red)){.utilities-queue-led{background-color:color-mix(in srgb, var(--pb-text-dim) 45%, transparent)}}.utilities-queue-led-green{background-color:var(--pb-success);box-shadow:0 0 8px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-led-green{box-shadow:0 0 8px color-mix(in srgb, var(--pb-success) 30%, transparent)}}.utilities-queue-led-amber{background-color:var(--pb-warning);box-shadow:0 0 8px var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-led-amber{box-shadow:0 0 8px color-mix(in srgb, var(--pb-warning) 28%, transparent)}}.utilities-queue-led-blue{background-color:var(--pb-info);box-shadow:0 0 8px var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-led-blue{box-shadow:0 0 8px color-mix(in srgb, var(--pb-info) 28%, transparent)}}.utilities-queue-job-name{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-primary)}.utilities-queue-job-type{margin-top:calc(var(--spacing) * .5);color:var(--pb-text-dim);font-size:.72rem}.utilities-queue-job-meta{margin-top:var(--spacing);color:var(--pb-text-secondary);font-size:.68rem}.utilities-queue-items{--tw-font-weight:var(--font-weight-semibold);font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:var(--font-weight-semibold);color:var(--pb-text-secondary)}.utilities-queue-progress-cell .series-mission-control-bar{gap:.625rem}.utilities-queue-progress-cell .series-mission-control-bar-track{min-width:120px}.utilities-queue-time{color:var(--pb-text-secondary);font-family:JetBrains Mono,monospace;font-size:.72rem}.utilities-queue-actions{justify-content:flex-end;align-items:center;gap:calc(var(--spacing) * 1.5);display:flex}.utilities-queue-act-btn{height:calc(var(--spacing) * 8);width:calc(var(--spacing) * 8);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:8px;justify-content:center;align-items:center;display:inline-flex}.utilities-queue-act-btn:hover{border-color:var(--pb-border-strong);color:var(--pb-text);background-color:var(--pb-bg-card-hover)}.utilities-queue-act-btn-danger{color:var(--pb-error)}.utilities-queue-act-btn-danger:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-act-btn-danger:hover{border-color:color-mix(in srgb, var(--pb-error) 36%, transparent)}}.utilities-queue-act-btn-danger:hover{background-color:var(--pb-error-dim);color:var(--pb-error)}.utilities-queue-detail-cell{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.utilities-queue-detail-cell{background-color:color-mix(in srgb, var(--pb-bg-surface) 72%, transparent)}}.utilities-queue-detail-summary{align-items:center;column-gap:calc(var(--spacing) * 6);row-gap:var(--spacing);border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3);color:var(--pb-text-dim);flex-wrap:wrap;font-size:.72rem;display:flex}.utilities-queue-detail-summary strong{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.utilities-queue-detail-label{--tw-tracking:.08em;letter-spacing:.08em;text-transform:uppercase}.utilities-queue-empty{justify-content:center;align-items:center;gap:calc(var(--spacing) * 2);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 12);text-align:center;flex-direction:column;display:flex}.utilities-queue-empty-compact{padding-block:calc(var(--spacing) * 8)}.utilities-queue-empty-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary)}.utilities-queue-empty-copy{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.queue-job-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1);overflow:hidden}.queue-job-card-expanded{border-color:var(--pb-border-hover);box-shadow:var(--pb-shadow-2)}.queue-progress-track{height:calc(var(--spacing) * 1.5);background-color:var(--pb-bg-card-hover);border-radius:3.40282e38px;width:100%;overflow:hidden}.queue-progress-fill{height:100%;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.5s;background-color:var(--pb-success);border-radius:3.40282e38px;transition-duration:.5s}.queue-progress-fill-paused{background-color:var(--pb-warning)}.selection-card{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);text-align:left;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));box-shadow:var(--pb-shadow-1)}.selection-card:not(.selection-card-active):hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.selection-card:not(.selection-card-active):hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 55%, transparent)}}.selection-card-active{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.selection-card-active{border-color:color-mix(in srgb, var(--pb-interactive) 42%, transparent)}}.selection-card-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.selection-card-active{background-color:color-mix(in srgb, var(--pb-interactive) 14%, var(--pb-bg-card))}}.selection-card-active{box-shadow:var(--pb-shadow-2)}.selection-card-icon{height:calc(var(--spacing) * 11);width:calc(var(--spacing) * 11);border-radius:var(--radius-xl);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex-shrink:0;justify-content:center;align-items:center;display:flex}.selection-card-icon-active{color:var(--pb-interactive);background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.selection-card-icon-active{background-color:color-mix(in srgb, var(--pb-interactive) 14%, transparent)}}.settings-theme-choice{padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 3)}.settings-theme-choice-icon{height:calc(var(--spacing) * 9);width:calc(var(--spacing) * 9);border-radius:var(--radius-lg)}.settings-media-preview-panel{background-color:var(--pb-surface-selected);border-color:var(--pb-border-hover)}@supports (color:color-mix(in lab, red, red)){.settings-media-preview-panel{border-color:color-mix(in srgb, var(--pb-border-hover) 85%, transparent)}}.settings-media-preview-panel{overflow-wrap:anywhere;min-height:4.625rem;transition:opacity .16s,border-color .16s}.settings-media-preview-panel.is-loading{opacity:.68;border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.settings-media-preview-panel.is-loading{border-color:color-mix(in srgb, var(--pb-interactive) 30%, var(--pb-border-hover))}}.workflow-step{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 3);box-shadow:var(--pb-shadow-1)}.workflow-step-active{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-step-active{border-color:color-mix(in srgb, var(--pb-interactive) 32%, transparent)}}.workflow-step-active{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-step-active{background-color:color-mix(in srgb, var(--pb-interactive) 10%, var(--pb-bg-card))}}.workflow-step-complete{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.workflow-step-complete{border-color:color-mix(in srgb, var(--pb-success) 28%, transparent)}}.workflow-step-complete{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.workflow-step-complete{background-color:color-mix(in srgb, var(--pb-success) 10%, var(--pb-bg-card))}}.workflow-step-badge{height:calc(var(--spacing) * 8);width:calc(var(--spacing) * 8);border-style:var(--tw-border-style);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);border-width:1px;border-color:var(--pb-border-hover);color:var(--pb-text-secondary);background-color:var(--pb-bg-card);border-radius:3.40282e38px;justify-content:center;align-items:center;display:flex}.workflow-step-active .workflow-step-badge{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-step-active .workflow-step-badge{border-color:color-mix(in srgb, var(--pb-interactive) 32%, transparent)}}.workflow-step-active .workflow-step-badge{color:var(--pb-interactive);background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.workflow-step-active .workflow-step-badge{background-color:color-mix(in srgb, var(--pb-interactive) 12%, transparent)}}.workflow-step-complete .workflow-step-badge{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.workflow-step-complete .workflow-step-badge{border-color:color-mix(in srgb, var(--pb-success) 28%, transparent)}}.workflow-step-complete .workflow-step-badge{color:var(--pb-success);background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.workflow-step-complete .workflow-step-badge{background-color:color-mix(in srgb, var(--pb-success) 12%, transparent)}}.workflow-step-label{--tw-font-weight:var(--font-weight-medium);font-size:10px;font-weight:var(--font-weight-medium);--tw-tracking:.12em;letter-spacing:.12em;color:var(--pb-text-dim);text-transform:uppercase}.workflow-step-title{margin-top:var(--spacing);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.checklist-item{align-items:center;gap:calc(var(--spacing) * 2);font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-secondary);display:flex}.section-body{padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4)}.section-header{gap:calc(var(--spacing) * 2);border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4);flex-direction:column;display:flex}@media (min-width:40rem){.section-header{flex-direction:row;justify-content:space-between;align-items:flex-start}}.section-header-copy{min-width:0}:where(.section-header-copy>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}.section-header-actions{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;flex-shrink:0;display:flex}.section-eyebrow{--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-dim);text-transform:uppercase}.section-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:.14em;letter-spacing:.14em;color:var(--pb-text-primary);text-transform:uppercase}.section-title-plain{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.section-description{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.settings-connection-list{gap:calc(var(--spacing) * 3);flex-direction:column;display:flex}.settings-connection-card{cursor:pointer;border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding:calc(var(--spacing) * 5);text-align:left;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.settings-connection-card:hover{border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.settings-connection-card-muted{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card-muted{background-color:color-mix(in srgb, var(--pb-bg-surface) 70%, transparent)}}.settings-connection-card-muted{opacity:.78}.settings-connection-card .downloads-action-btn.is-danger{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card .downloads-action-btn.is-danger{border-color:color-mix(in srgb, var(--pb-error) 28%, var(--pb-border-subtle))}}.settings-connection-card .downloads-action-btn.is-danger{background:var(--pb-error-dim);color:var(--pb-error)}.settings-connection-card .downloads-action-btn.is-danger:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card .downloads-action-btn.is-danger:hover{border-color:color-mix(in srgb, var(--pb-error) 44%, transparent)}}.settings-connection-card .downloads-action-btn.is-danger:hover{background:var(--pb-error-dim)}@supports (color:color-mix(in lab, red, red)){.settings-connection-card .downloads-action-btn.is-danger:hover{background:color-mix(in srgb, var(--pb-error-dim) 86%, transparent)}}.settings-connection-card .downloads-action-btn.is-danger:hover{color:var(--pb-error)}.admin-nav-link{align-items:center;gap:calc(var(--spacing) * 2.5);border-radius:var(--radius-lg);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 2.5);padding-block:calc(var(--spacing) * 2);--tw-font-weight:var(--font-weight-medium);font-size:.8125rem;font-weight:var(--font-weight-medium);color:var(--pb-text-dim);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-width:1px;border-color:#0000;display:flex}.admin-nav-link:hover{color:var(--pb-text-secondary);background-color:var(--pb-surface-selected)}.admin-nav-link-active{color:var(--pb-interactive);background-color:var(--pb-surface-selected)}.admin-nav-link-icon{height:calc(var(--spacing) * 4);width:calc(var(--spacing) * 4);flex-shrink:0;justify-content:center;align-items:center;display:flex}.admin-nav-link-active .admin-nav-link-icon{color:var(--pb-interactive)}.admin-nav-link-copy{min-width:0}.admin-nav-link-title{text-overflow:ellipsis;white-space:nowrap;--tw-font-weight:var(--font-weight-medium);font-size:.8125rem;font-weight:var(--font-weight-medium);overflow:hidden}:where(.settings-rows>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)));border-color:var(--pb-border)}.settings-row{gap:calc(var(--spacing) * 4);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4);grid-template-columns:repeat(1,minmax(0,1fr));display:grid}@media (min-width:40rem){.settings-row{grid-template-columns:220px minmax(0,1fr);align-items:flex-start}}.settings-row-meta{padding-top:var(--spacing)}.settings-row-label{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);color:var(--pb-text-secondary);display:block}.settings-row-help{margin-top:calc(var(--spacing) * .5);--tw-leading:calc(var(--spacing) * 5);font-size:11px;line-height:calc(var(--spacing) * 5);color:var(--pb-text-dim)}.settings-row-content{min-width:0}.settings-rows-align-end .settings-row-content{flex-direction:column;align-items:flex-end;display:flex}.settings-footer{gap:calc(var(--spacing) * 3);border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 3);flex-direction:column;display:flex}@media (min-width:40rem){.settings-footer{flex-direction:row;justify-content:space-between;align-items:center}}.settings-footer{background-color:var(--pb-bg-surface);overflow-anchor:none}@media (min-width:40rem){.settings-footer-end{justify-content:flex-end}}.settings-footer-copy{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));color:var(--pb-text-dim)}.settings-footer-actions{align-items:center;gap:calc(var(--spacing) * 3);flex-wrap:wrap;display:flex}.settings-footer-actions .btn-primary,.settings-footer-actions .btn-ghost,.settings-footer-actions .btn-danger,.settings-footer-actions .btn-warning{min-height:calc(var(--spacing) * 10);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.field-note{margin-top:calc(var(--spacing) * 1.5);align-items:flex-start;gap:calc(var(--spacing) * 1.5);--tw-leading:calc(var(--spacing) * 5);font-size:11px;line-height:calc(var(--spacing) * 5);display:flex}.field-note code{color:var(--pb-text-secondary)}.field-note-icon{margin-top:calc(var(--spacing) * .5);flex-shrink:0;width:.75rem;height:.75rem}.field-note-warning{color:var(--pb-warning)}.field-note-info{color:var(--pb-info)}.field-note-danger{color:var(--pb-error)}.field-note-success{color:var(--pb-success)}.alert-banner{gap:calc(var(--spacing) * 4);border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 6);padding-block:calc(var(--spacing) * 4);border-width:1px;flex-direction:column;display:flex}@media (min-width:40rem){.alert-banner{flex-direction:row;justify-content:space-between;align-items:flex-start}}.alert-banner{box-shadow:var(--pb-shadow-1)}.alert-banner-main{align-items:flex-start;gap:calc(var(--spacing) * 4);min-width:0;display:flex}.alert-banner-icon{height:calc(var(--spacing) * 10);width:calc(var(--spacing) * 10);border-radius:var(--radius-lg);flex-shrink:0;justify-content:center;align-items:center;display:flex}.alert-banner-copy{min-width:0}:where(.alert-banner-copy>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}.alert-banner-title{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);color:var(--pb-text-primary)}.alert-banner-description{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6);color:var(--pb-text-secondary)}.alert-banner-actions{align-items:center;gap:calc(var(--spacing) * 2);flex-wrap:wrap;flex-shrink:0;display:flex}.alert-banner-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.alert-banner-info{background-color:color-mix(in srgb, var(--pb-info) 10%, var(--pb-bg-card))}}.alert-banner-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.alert-banner-info{border-color:color-mix(in srgb, var(--pb-info) 18%, transparent)}}.alert-banner-info .alert-banner-icon{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.alert-banner-info .alert-banner-icon{background-color:color-mix(in srgb, var(--pb-info) 18%, transparent)}}.alert-banner-info .alert-banner-icon{color:var(--pb-info)}.alert-banner-warning{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.alert-banner-warning{background-color:color-mix(in srgb, var(--pb-warning) 10%, var(--pb-bg-card))}}.alert-banner-warning{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.alert-banner-warning{border-color:color-mix(in srgb, var(--pb-warning) 18%, transparent)}}.alert-banner-warning .alert-banner-icon{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.alert-banner-warning .alert-banner-icon{background-color:color-mix(in srgb, var(--pb-warning) 18%, transparent)}}.alert-banner-warning .alert-banner-icon{color:var(--pb-warning)}.alert-banner-danger{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.alert-banner-danger{background-color:color-mix(in srgb, var(--pb-error) 10%, var(--pb-bg-card))}}.alert-banner-danger{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.alert-banner-danger{border-color:color-mix(in srgb, var(--pb-error) 18%, transparent)}}.alert-banner-danger .alert-banner-icon{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.alert-banner-danger .alert-banner-icon{background-color:color-mix(in srgb, var(--pb-error) 18%, transparent)}}.alert-banner-danger .alert-banner-icon{color:var(--pb-error)}.alert-banner-success{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.alert-banner-success{background-color:color-mix(in srgb, var(--pb-success) 10%, var(--pb-bg-card))}}.alert-banner-success{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.alert-banner-success{border-color:color-mix(in srgb, var(--pb-success) 18%, transparent)}}.alert-banner-success .alert-banner-icon{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.alert-banner-success .alert-banner-icon{background-color:color-mix(in srgb, var(--pb-success) 18%, transparent)}}.alert-banner-success .alert-banner-icon{color:var(--pb-success)}.modal-shell{z-index:50;padding:calc(var(--spacing) * 4);padding-top:calc(var(--spacing) * 16);justify-content:center;align-items:flex-start;display:flex;position:fixed;inset:0}.modal-backdrop{z-index:0;background-color:var(--pb-bg-overlay);--tw-backdrop-blur:blur(var(--blur-xs));-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);position:fixed;inset:0}.modal-panel{z-index:10;border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);width:100%;box-shadow:var(--pb-shadow-overlay);flex-direction:column;display:flex;position:relative;overflow:hidden}.modal-header{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);justify-content:space-between;align-items:center;display:flex}.modal-body{flex:1;overflow-y:auto}.modal-footer{border-top-style:var(--tw-border-style);border-top-width:1px;border-color:var(--pb-border);background-color:var(--pb-bg-card);padding-inline:calc(var(--spacing) * 5);padding-block:calc(var(--spacing) * 4);flex-shrink:0;justify-content:space-between;align-items:center;display:flex}.donation-modal-shell{align-items:flex-start;padding-top:4.75rem}.donation-modal-panel{border-radius:1.5rem;width:min(100vw - 2rem,56rem);max-height:calc(100dvh - 6rem)}.donation-modal-header{border-bottom-color:var(--pb-border-subtle);background:radial-gradient(circle at top left, var(--pb-brand-dim), transparent 18rem), var(--pb-bg-card);align-items:flex-start;gap:1rem}.donation-modal-kicker{letter-spacing:.18em;text-transform:uppercase;color:var(--pb-brand);font-family:JetBrains Mono,monospace;font-size:.68rem;font-weight:700}.donation-modal-title{color:var(--pb-text);letter-spacing:-.03em;margin-top:.2rem;font-family:Syne,sans-serif;font-size:clamp(1.35rem,2.5vw,2rem);font-weight:800;line-height:1.05}.donation-modal-copy{max-width:38rem;color:var(--pb-text-sec);margin-top:.45rem;font-size:.9rem;line-height:1.55}.donation-modal-body{background:linear-gradient(135deg, transparent, var(--pb-surface-selected)), var(--pb-bg-surface);gap:1rem;padding:1rem;display:grid}.donation-option-card{border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);box-shadow:var(--pb-shadow-1);border-radius:1.25rem;grid-template-columns:minmax(0,1fr) 10.5rem;align-items:stretch;gap:1rem;padding:1rem;display:grid}.donation-option-card--coffee{border-color:#fd0}@supports (color:color-mix(in lab, red, red)){.donation-option-card--coffee{border-color:color-mix(in srgb, #fd0 38%, var(--pb-border-subtle))}}.donation-option-card--liberapay{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.donation-option-card--liberapay{border-color:color-mix(in srgb, var(--pb-success) 32%, var(--pb-border-subtle))}}.donation-option-main{flex-direction:column;justify-content:space-between;gap:1rem;min-width:0;display:flex}.donation-option-eyebrow{letter-spacing:.16em;text-transform:uppercase;color:var(--pb-text-sec);font-family:JetBrains Mono,monospace;font-size:.64rem;font-weight:700}.donation-option-title{color:var(--pb-text);margin-top:.25rem;font-family:Syne,sans-serif;font-size:1.1rem;font-weight:800}.donation-option-copy{max-width:28rem;color:var(--pb-text-sec);margin-top:.35rem;font-size:.82rem;line-height:1.5}.donation-option-button{border-radius:999px;justify-content:center;align-items:center;width:fit-content;padding:.62rem 1rem;font-size:.82rem;font-weight:800;line-height:1;text-decoration:none;transition:transform .16s,box-shadow .16s,filter .16s;display:inline-flex}.donation-option-button:hover,.donation-option-button:focus-visible{filter:brightness(1.02);transform:translateY(-1px)}.donation-option-button:focus-visible{outline:2px solid var(--pb-focus-outline);outline-offset:3px}.donation-option-button--coffee{color:#16130a;background:#fd0;box-shadow:0 10px 20px #fd03}.donation-option-button--liberapay{background:var(--pb-success);color:#f7f1e8;box-shadow:0 10px 20px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.donation-option-button--liberapay{box-shadow:0 10px 20px color-mix(in srgb, var(--pb-success) 20%, transparent)}}[data-theme=dark] .donation-option-button--liberapay{color:#16130a}.donation-qr-frame{border:1px solid var(--pb-border-subtle);background:#fff;border-radius:1rem;place-items:center;min-height:10.5rem;padding:.625rem;display:grid}.donation-qr{object-fit:contain;width:9.25rem;height:9.25rem}@media (max-width:720px){.donation-modal-shell{padding-top:3.25rem}.donation-modal-panel{max-height:calc(100dvh - 4rem)}.donation-option-card{grid-template-columns:1fr}.donation-option-button{width:100%}}.issue-search-modal-panel{border-radius:1.25rem;width:min(100vw - 2rem,78rem);max-width:none;max-height:90vh}.issue-search-modal-header{border-bottom-color:var(--pb-border-subtle);align-items:flex-start;padding:1.125rem 1.5rem}.issue-search-modal-title{letter-spacing:.03em;text-transform:uppercase;color:var(--pb-text);font-family:Syne,sans-serif;font-size:1.05rem;font-weight:800}.issue-search-modal-title span{color:var(--pb-brand)}.issue-search-modal-subtitle{color:var(--pb-text-dim);margin-top:.1875rem;font-size:.78rem}.issue-search-modal-stats{flex-wrap:wrap;gap:.375rem;margin-top:.5rem;display:flex}.issue-search-modal-close{border-radius:var(--radius-lg);--tw-border-style:none;color:var(--pb-text-dim);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background-color:#0000;border-style:none;justify-content:center;align-items:center;display:inline-flex}@media (hover:hover){.issue-search-modal-close:hover{color:var(--pb-text-primary)}}.issue-search-modal-close{width:1.875rem;height:1.875rem}.issue-search-modal-close:hover{background:var(--pb-surface-selected)}.issue-search-modal-body{padding:0}.issue-search-loading-state{min-height:24rem}.issue-search-loading-title{color:var(--pb-text);font-size:.82rem;font-weight:600}.issue-search-loading-copy{color:var(--pb-text-dim);font-size:.72rem}.issue-search-dc-status{border-top:1px solid var(--pb-border-subtle);color:var(--pb-text-dim);align-items:center;gap:.625rem;padding:.75rem 1rem;font-size:.78rem;display:flex}.issue-search-dc-results{border-top:1px solid var(--pb-border-subtle)}.issue-search-dc-results-heading{align-items:center;gap:.5rem;padding:.75rem 1rem;display:flex}.issue-search-modal-footer{border-top-color:var(--pb-border-subtle);justify-content:flex-end;padding:.75rem 1.5rem}.issue-search-modal-footer-meta{color:var(--pb-text-dim);font-family:JetBrains Mono,monospace;font-size:.68rem}.issue-search-modal-footer-meta:empty{display:none}.issue-search-modal-footer-meta strong{color:var(--pb-text);font-weight:700}.issue-search-modal-footer-close{margin-left:auto}.inline-alert{align-items:flex-start;gap:calc(var(--spacing) * 2);border-radius:var(--radius-lg);border-style:var(--tw-border-style);padding-inline:calc(var(--spacing) * 4);padding-block:calc(var(--spacing) * 2.5);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));border-width:1px;display:flex}.inline-alert-icon{margin-top:calc(var(--spacing) * .5);flex-shrink:0}.inline-alert-danger{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.inline-alert-danger{background-color:color-mix(in srgb, var(--pb-error) 12%, var(--pb-bg-card))}}.inline-alert-danger{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.inline-alert-danger{border-color:color-mix(in srgb, var(--pb-error) 22%, transparent)}}.inline-alert-danger{color:var(--pb-error)}.inline-alert-warning{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.inline-alert-warning{background-color:color-mix(in srgb, var(--pb-warning) 12%, var(--pb-bg-card))}}.inline-alert-warning{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.inline-alert-warning{border-color:color-mix(in srgb, var(--pb-warning) 22%, transparent)}}.inline-alert-warning{color:var(--pb-warning)}.inline-alert-info{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.inline-alert-info{background-color:color-mix(in srgb, var(--pb-info) 12%, var(--pb-bg-card))}}.inline-alert-info{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.inline-alert-info{border-color:color-mix(in srgb, var(--pb-info) 22%, transparent)}}.inline-alert-info{color:var(--pb-info)}.inline-alert-success{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.inline-alert-success{background-color:color-mix(in srgb, var(--pb-success) 12%, var(--pb-bg-card))}}.inline-alert-success{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.inline-alert-success{border-color:color-mix(in srgb, var(--pb-success) 22%, transparent)}}.inline-alert-success{color:var(--pb-success)}.field-invalid{border-color:var(--pb-error)!important}@supports (color:color-mix(in lab, red, red)){.field-invalid{border-color:color-mix(in srgb, var(--pb-error) 52%, transparent)!important}}.icon-btn{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));justify-content:center;align-items:center;display:inline-flex}@media (hover:hover){.icon-btn:hover{border-color:var(--pb-border-strong);color:var(--pb-text-primary)}}.icon-btn:disabled{cursor:not-allowed;opacity:.5}.icon-btn{min-width:var(--pb-control-min-height);width:var(--pb-control-min-height);min-height:var(--pb-control-min-height);border-radius:var(--pb-control-radius);padding:0}.toggle-switch{cursor:pointer;background-color:var(--pb-bg-card-hover);width:2.25rem;height:1.25rem;box-shadow:inset 0 0 0 1px var(--pb-border-hover);border-radius:999px;flex-shrink:0;transition:background-color .18s,box-shadow .18s;display:inline-flex;position:relative}label:has(>.toggle-input+.toggle-switch){position:relative}.toggle-input{clip:rect(0, 0, 0, 0);clip-path:inset(50%);white-space:nowrap;border:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.toggle-input-overlay{opacity:0;clip:auto;clip-path:none;white-space:normal;z-index:1;width:100%;height:100%;margin:0;inset:0}.toggle-switch:after{content:"";background-color:var(--pb-brand);width:1rem;height:1rem;box-shadow:0 1px 2px var(--pb-text-primary);border-radius:999px;position:absolute;top:2px;left:2px}@supports (color:color-mix(in lab, red, red)){.toggle-switch:after{box-shadow:0 1px 2px color-mix(in srgb, var(--pb-text-primary) 14%, transparent)}}.toggle-switch:after{transition:transform .18s}.peer:checked+.toggle-switch{background-color:var(--pb-interactive);box-shadow:inset 0 0 0 1px var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.peer:checked+.toggle-switch{box-shadow:inset 0 0 0 1px color-mix(in srgb, var(--pb-interactive) 34%, transparent)}}.peer:checked+.toggle-switch:after{background-color:var(--pb-surface-shell);transform:translate(1rem)}.peer:focus-visible+.toggle-switch{outline:2px solid var(--pb-focus-outline);outline-offset:3px;box-shadow:0 0 0 4px var(--pb-focus-ring)}.strength-segment{height:calc(var(--spacing) * 1.5);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.3s;border-radius:3.40282e38px;flex:1;transition-duration:.3s}.strength-segment-neutral{background-color:var(--pb-bg-card-hover)}.strength-segment-danger{background-color:var(--pb-error)}.strength-segment-warning{background-color:var(--pb-warning)}.strength-segment-success{background-color:var(--pb-success)}.input-pb{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-input);width:100%;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);font-family:DM Sans,sans-serif;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-primary);border-radius:10px}.input-pb::placeholder{color:var(--pb-text-dim)}.input-pb:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;border-color:#0000;outline-style:none}@media (forced-colors:active){.input-pb:focus{outline-offset:2px;outline:2px solid #0000}}.checkbox-pb{height:calc(var(--spacing) * 4);width:calc(var(--spacing) * 4);border-color:var(--pb-border-hover);background-color:var(--pb-bg-input);color:var(--pb-interactive);border-radius:.25rem}.checkbox-pb:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-ring-offset-width:0px;--tw-ring-offset-shadow:var(--tw-ring-inset,) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color)}.select-pb{appearance:none;border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-input);width:100%;padding-block:calc(var(--spacing) * 2);padding-right:calc(var(--spacing) * 8);padding-left:calc(var(--spacing) * 3);font-family:DM Sans,sans-serif;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-primary);border-radius:10px}.select-pb:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;border-color:#0000;outline-style:none}@media (forced-colors:active){.select-pb:focus{outline-offset:2px;outline:2px solid #0000}}.select-pb{background-image:url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 20 20'%3E%3Cpath stroke='%237A8599' stroke-linecap='round' stroke-linejoin='round' stroke-width='1.5' d='M6 8l4 4 4-4'/%3E%3C/svg%3E");background-position:right .5rem center;background-repeat:no-repeat;background-size:1.25rem}.dropdown-select{min-width:0;position:relative}.dropdown-select-trigger{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);width:100%;color:var(--pb-text-primary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));justify-content:space-between;align-items:center;display:flex}@media (hover:hover){.dropdown-select-trigger:hover{border-color:var(--pb-border-strong)}}.dropdown-select-trigger:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;border-color:#0000;outline-style:none}@media (forced-colors:active){.dropdown-select-trigger:focus{outline-offset:2px;outline:2px solid #0000}}.dropdown-select-trigger:disabled{cursor:not-allowed;opacity:.6}.dropdown-select-trigger{gap:var(--pb-control-gap);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-px);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius)}.dropdown-select-trigger-label{text-overflow:ellipsis;white-space:nowrap;text-align:left;flex:1;min-width:0;overflow:hidden}.dropdown-select-chevron{color:var(--pb-text-secondary);transition-property:transform,translate,scale,rotate;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));width:var(--pb-control-icon-size);height:var(--pb-control-icon-size);flex-shrink:0}.import-advanced-disclosure[open]>summary .import-advanced-disclosure-chevron{rotate:90deg}.dropdown-select-panel{z-index:80;max-height:calc(var(--spacing) * 60);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card);padding-block:var(--spacing);--tw-shadow:0 25px 50px -12px var(--tw-shadow-color,#00000040);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-shadow-color:#0003;position:fixed;top:0;left:0;overflow-y:auto}@supports (color:color-mix(in lab, red, red)){.dropdown-select-panel{--tw-shadow-color:color-mix(in oklab, color-mix(in oklab, var(--color-black) 20%, transparent) var(--tw-shadow-alpha), transparent)}}.dropdown-select-panel{max-height:var(--pb-dropdown-panel-max-height,15rem);will-change:top, left}.dropdown-select-panel[data-ready=false]{visibility:hidden;pointer-events:none}.dropdown-select-option{text-align:left;width:100%;color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));justify-content:space-between;align-items:center;gap:var(--pb-control-gap);min-height:var(--pb-control-min-height);padding-inline:var(--pb-control-panel-item-px);padding-block:var(--pb-control-panel-item-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);display:flex}.dropdown-select-option-label{text-align:left;white-space:nowrap;overflow-wrap:normal;flex:1;min-width:0}.import-reconcile-issue-dropdown .dropdown-select-trigger{align-items:flex-start;height:auto}.import-reconcile-issue-dropdown .dropdown-select-trigger-label{white-space:normal;overflow-wrap:anywhere;text-overflow:clip;overflow:visible}.dropdown-select-panel-wrap .dropdown-select-option{align-items:flex-start}.dropdown-select-panel-wrap .dropdown-select-option-label{white-space:normal;overflow-wrap:anywhere;word-break:normal}.dropdown-select-option:hover,.dropdown-select-option-active{background-color:var(--pb-info-dim);color:var(--pb-text-primary)}.dropdown-select-option-active{transition-duration:0s}.dropdown-select-option-selected{background-color:var(--pb-info-dim);color:var(--pb-text-primary);font-weight:500}.dropdown-select-option-check{width:var(--pb-control-icon-size);height:var(--pb-control-icon-size);flex-shrink:0}.direct-provider-uri-control{position:relative}.direct-provider-uri-input{padding-right:calc(var(--spacing) * 10);font-family:JetBrains Mono,Fira Code,monospace}.direct-provider-uri-toggle{width:calc(var(--spacing) * 10);border-top-right-radius:var(--radius-lg);border-bottom-right-radius:var(--radius-lg);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));inset-block:0;justify-content:center;align-items:center;display:flex;position:absolute;right:0}@media (hover:hover){.direct-provider-uri-toggle:hover{color:var(--pb-text-primary)}}.direct-provider-uri-toggle:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;outline-style:none}@media (forced-colors:active){.direct-provider-uri-toggle:focus{outline-offset:2px;outline:2px solid #0000}}.direct-provider-uri-toggle:focus{--tw-ring-inset:inset}.direct-provider-uri-options{z-index:90;margin-top:var(--spacing);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card);padding-block:var(--spacing);--tw-shadow:0 25px 50px -12px var(--tw-shadow-color,#00000040);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-shadow-color:#0003;position:absolute;top:100%;left:0;right:0;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.direct-provider-uri-options{--tw-shadow-color:color-mix(in oklab, color-mix(in oklab, var(--color-black) 20%, transparent) var(--tw-shadow-alpha), transparent)}}.direct-provider-uri-option{justify-content:space-between;align-items:center;gap:calc(var(--spacing) * 2);width:100%;padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);text-align:left;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));display:flex}@media (hover:hover){.direct-provider-uri-option:hover{background-color:var(--pb-info-dim);color:var(--pb-text-primary)}}.direct-provider-uri-option:focus{background-color:var(--pb-info-dim);color:var(--pb-text-primary);--tw-outline-style:none;outline-style:none}@media (forced-colors:active){.direct-provider-uri-option:focus{outline-offset:2px;outline:2px solid #0000}}.direct-provider-uri-option-selected{background-color:var(--pb-info-dim);color:var(--pb-text-primary)}.search-field{min-width:0;position:relative}.search-field-icon{pointer-events:none;z-index:10;--tw-translate-y:calc(calc(1 / 2 * 100%) * -1);translate:var(--tw-translate-x) var(--tw-translate-y);color:var(--pb-text-secondary);top:50%;left:var(--pb-control-search-icon-left);width:var(--pb-control-icon-size);height:var(--pb-control-icon-size);position:absolute}.search-field-input{border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card-hover);width:100%;color:var(--pb-text-primary)}.search-field-input::placeholder{color:var(--pb-text-secondary)}.search-field-input:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--pb-interactive);--tw-outline-style:none;border-color:#0000;outline-style:none}@media (forced-colors:active){.search-field-input:focus{outline-offset:2px;outline:2px solid #0000}}.search-field-input{min-height:var(--pb-control-min-height);padding-inline-start:var(--pb-control-search-padding-left);padding-inline-end:var(--pb-control-search-padding-right);padding-block:var(--pb-control-py);font-size:var(--pb-control-font-size);line-height:var(--pb-control-line-height);border-radius:var(--pb-control-radius);appearance:none;background-image:none}.search-field-editor{white-space:nowrap;min-height:42px;overflow:hidden}.search-field-editor[data-empty=true]:before{content:attr(data-placeholder);color:var(--pb-text-secondary)}.search-field-input::-webkit-search-decoration{appearance:none;display:none}.search-field-input::-webkit-search-cancel-button{appearance:none;display:none}.search-field-input::-webkit-search-results-button{appearance:none;display:none}.search-field-input::-webkit-search-results-decoration{appearance:none;display:none}.search-field-input::-ms-clear{width:0;height:0;display:none}.search-field-input::-ms-reveal{width:0;height:0;display:none}.search-field-clear{z-index:10;height:calc(var(--spacing) * 6);width:calc(var(--spacing) * 6);--tw-translate-y:calc(calc(1 / 2 * 100%) * -1);translate:var(--tw-translate-x) var(--tw-translate-y);padding:calc(var(--spacing) * .5);color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:.25rem;justify-content:center;align-items:center;display:inline-flex;position:absolute;top:50%}@media (hover:hover){.search-field-clear:hover{color:var(--pb-text-primary)}}.search-field-clear{right:var(--pb-control-clear-right)}.search-history-panel{z-index:30;margin-top:var(--spacing);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--pb-border-hover);background-color:var(--pb-bg-card);--tw-shadow:0 20px 25px -5px var(--tw-shadow-color,#0000001a), 0 8px 10px -6px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-shadow-color:#0006;position:absolute;top:100%;left:0;right:0;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.search-history-panel{--tw-shadow-color:color-mix(in oklab, color-mix(in oklab, var(--color-black) 40%, transparent) var(--tw-shadow-alpha), transparent)}}.search-history-panel-header{border-bottom-style:var(--tw-border-style);border-bottom-width:1px;border-color:var(--pb-border);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);justify-content:space-between;align-items:center;display:flex}.search-history-panel-title{--tw-font-weight:var(--font-weight-semibold);font-size:10px;font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-wider);letter-spacing:var(--tracking-wider);color:var(--pb-text-dim);text-transform:uppercase}.search-history-panel-clear{color:var(--pb-text-dim);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));font-size:10px}@media (hover:hover){.search-history-panel-clear:hover{color:var(--pb-error)}}.search-history-list{max-height:280px;padding-block:var(--spacing);overflow-y:auto}.search-history-item{align-items:center;display:flex}.search-history-item-button{align-items:center;gap:calc(var(--spacing) * 2.5);padding-inline:calc(var(--spacing) * 3);padding-block:calc(var(--spacing) * 2);text-align:left;font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));color:var(--pb-text-secondary);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));flex:1;display:flex}@media (hover:hover){.search-history-item-button:hover{color:var(--pb-text-primary)}}.search-history-item-button:hover{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.search-history-item-button:hover{background-color:color-mix(in srgb, var(--pb-bg-card-hover) 60%, transparent)}}.search-history-item-remove{padding-inline:calc(var(--spacing) * 2);padding-block:calc(var(--spacing) * 2);color:var(--pb-text-dim);opacity:0;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}@media (hover:hover){.search-history-item-remove:is(:where(.group):hover *){opacity:1}.search-history-item-remove:hover{color:var(--pb-error)}}.touch-target{min-width:44px;min-height:44px}.progress-track{height:var(--spacing);background-color:var(--pb-border);border-radius:3.40282e38px}.progress-fill{background-color:var(--pb-success);height:100%;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:3.40282e38px}.progress-fill-partial{background-color:var(--pb-interactive);height:100%;transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));border-radius:3.40282e38px}.callout-brand{align-items:center;gap:calc(var(--spacing) * 3);border-radius:var(--radius-xl);border-style:var(--tw-border-style);padding:calc(var(--spacing) * 4);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));background-color:var(--pb-brand-signal-dim);border-width:1px;border-color:var(--pb-brand-signal-border);display:flex}.callout-brand:hover{background-color:var(--pb-brand-signal-hover);border-color:var(--pb-brand-signal-border)}.callout-brand-icon{border-radius:var(--radius-lg);padding:calc(var(--spacing) * 2);background-color:var(--pb-brand-signal-dim);color:var(--pb-brand-signal);flex-shrink:0}.callout-brand-title{color:var(--pb-brand-signal)}.callout-brand-body{color:var(--pb-brand-signal);opacity:.7}.callout-brand-chevron{color:var(--pb-brand-signal);opacity:.5}}@layer utilities{.pointer-events-none{pointer-events:none}.visible{visibility:visible}.sr-only{clip-path:inset(50%);white-space:nowrap;border-width:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.absolute{position:absolute}.fixed{position:fixed}.relative{position:relative}.sticky{position:sticky}.inset-0{inset:0}.inset-y-0{inset-block:0}.-top-3{top:calc(var(--spacing) * -3)}.top-0{top:0}.top-1\/2{top:50%}.top-2{top:calc(var(--spacing) * 2)}.top-4{top:calc(var(--spacing) * 4)}.-right-1{right:calc(var(--spacing) * -1)}.-right-3{right:calc(var(--spacing) * -3)}.right-0{right:0}.right-2{right:calc(var(--spacing) * 2)}.right-4{right:calc(var(--spacing) * 4)}.-bottom-1{bottom:calc(var(--spacing) * -1)}.left-0{left:0}.left-3{left:calc(var(--spacing) * 3)}.z-10{z-index:10}.z-20{z-index:20}.z-30{z-index:30}.z-40{z-index:40}.z-50{z-index:50}.z-\[1\]{z-index:1}.z-\[60\]{z-index:60}.z-\[100\]{z-index:100}.z-\[110\]{z-index:110}.col-span-2{grid-column:span 2/span 2}.col-span-full{grid-column:1/-1}.container{width:100%}@media (min-width:40rem){.container{max-width:40rem}}@media (min-width:48rem){.container{max-width:48rem}}@media (min-width:64rem){.container{max-width:64rem}}@media (min-width:80rem){.container{max-width:80rem}}@media (min-width:96rem){.container{max-width:96rem}}.mx-auto{margin-inline:auto}.my-2{margin-block:calc(var(--spacing) * 2)}.\!mt-0{margin-top:0!important}.\!mt-2{margin-top:calc(var(--spacing) * 2)!important}.mt-0\.5{margin-top:calc(var(--spacing) * .5)}.mt-1{margin-top:var(--spacing)}.mt-1\.5{margin-top:calc(var(--spacing) * 1.5)}.mt-2{margin-top:calc(var(--spacing) * 2)}.mt-3{margin-top:calc(var(--spacing) * 3)}.mt-4{margin-top:calc(var(--spacing) * 4)}.mt-5{margin-top:calc(var(--spacing) * 5)}.mt-6{margin-top:calc(var(--spacing) * 6)}.mt-8{margin-top:calc(var(--spacing) * 8)}.mr-1{margin-right:var(--spacing)}.\!mb-0{margin-bottom:0!important}.mb-1{margin-bottom:var(--spacing)}.mb-1\.5{margin-bottom:calc(var(--spacing) * 1.5)}.mb-2{margin-bottom:calc(var(--spacing) * 2)}.mb-3{margin-bottom:calc(var(--spacing) * 3)}.mb-4{margin-bottom:calc(var(--spacing) * 4)}.mb-5{margin-bottom:calc(var(--spacing) * 5)}.-ml-2{margin-left:calc(var(--spacing) * -2)}.ml-0\.5{margin-left:calc(var(--spacing) * .5)}.ml-1{margin-left:var(--spacing)}.ml-2{margin-left:calc(var(--spacing) * 2)}.ml-4{margin-left:calc(var(--spacing) * 4)}.ml-\[140px\]{margin-left:140px}.ml-auto{margin-left:auto}.line-clamp-2{-webkit-line-clamp:2;-webkit-box-orient:vertical;display:-webkit-box;overflow:hidden}.block{display:block}.contents{display:contents}.flex{display:flex}.grid{display:grid}.hidden{display:none}.inline{display:inline}.inline-flex{display:inline-flex}.table{display:table}.table-cell{display:table-cell}.table-row{display:table-row}.aspect-\[2\/3\]{aspect-ratio:2/3}.h-1\.5{height:calc(var(--spacing) * 1.5)}.h-2{height:calc(var(--spacing) * 2)}.h-2\.5{height:calc(var(--spacing) * 2.5)}.h-3{height:calc(var(--spacing) * 3)}.h-3\.5{height:calc(var(--spacing) * 3.5)}.h-4{height:calc(var(--spacing) * 4)}.h-4\.5{height:calc(var(--spacing) * 4.5)}.h-5{height:calc(var(--spacing) * 5)}.h-6{height:calc(var(--spacing) * 6)}.h-7{height:calc(var(--spacing) * 7)}.h-8{height:calc(var(--spacing) * 8)}.h-9{height:calc(var(--spacing) * 9)}.h-10{height:calc(var(--spacing) * 10)}.h-11{height:calc(var(--spacing) * 11)}.h-12{height:calc(var(--spacing) * 12)}.h-14{height:calc(var(--spacing) * 14)}.h-16{height:calc(var(--spacing) * 16)}.h-28{height:calc(var(--spacing) * 28)}.h-32{height:calc(var(--spacing) * 32)}.h-96{height:calc(var(--spacing) * 96)}.h-\[18px\]{height:18px}.h-\[100dvh\]{height:100dvh}.h-full{height:100%}.max-h-64{max-height:calc(var(--spacing) * 64)}.max-h-72{max-height:calc(var(--spacing) * 72)}.max-h-80{max-height:calc(var(--spacing) * 80)}.max-h-\[28rem\]{max-height:28rem}.max-h-\[62vh\]{max-height:62vh}.max-h-\[70vh\]{max-height:70vh}.max-h-\[85vh\]{max-height:85vh}.max-h-\[90vh\]{max-height:90vh}.max-h-\[95vh\]{max-height:95vh}.max-h-\[calc\(100vh-5rem\)\]{max-height:calc(100vh - 5rem)}.\!min-h-8{min-height:calc(var(--spacing) * 8)!important}.\!min-h-10{min-height:calc(var(--spacing) * 10)!important}.min-h-0{min-height:0}.min-h-8{min-height:calc(var(--spacing) * 8)}.min-h-56{min-height:calc(var(--spacing) * 56)}.min-h-\[36px\]{min-height:36px}.min-h-\[200px\]{min-height:200px}.\!w-8{width:calc(var(--spacing) * 8)!important}.\!w-10{width:calc(var(--spacing) * 10)!important}.w-1\.5{width:calc(var(--spacing) * 1.5)}.w-1\/3{width:33.3333%}.w-2{width:calc(var(--spacing) * 2)}.w-2\.5{width:calc(var(--spacing) * 2.5)}.w-3{width:calc(var(--spacing) * 3)}.w-3\.5{width:calc(var(--spacing) * 3.5)}.w-4{width:calc(var(--spacing) * 4)}.w-4\.5{width:calc(var(--spacing) * 4.5)}.w-5{width:calc(var(--spacing) * 5)}.w-6{width:calc(var(--spacing) * 6)}.w-7{width:calc(var(--spacing) * 7)}.w-8{width:calc(var(--spacing) * 8)}.w-9{width:calc(var(--spacing) * 9)}.w-10{width:calc(var(--spacing) * 10)}.w-11{width:calc(var(--spacing) * 11)}.w-12{width:calc(var(--spacing) * 12)}.w-16{width:calc(var(--spacing) * 16)}.w-20{width:calc(var(--spacing) * 20)}.w-24{width:calc(var(--spacing) * 24)}.w-28{width:calc(var(--spacing) * 28)}.w-32{width:calc(var(--spacing) * 32)}.w-36{width:calc(var(--spacing) * 36)}.w-40{width:calc(var(--spacing) * 40)}.w-44{width:calc(var(--spacing) * 44)}.w-48{width:calc(var(--spacing) * 48)}.w-56{width:calc(var(--spacing) * 56)}.w-64{width:calc(var(--spacing) * 64)}.w-80{width:calc(var(--spacing) * 80)}.w-96{width:calc(var(--spacing) * 96)}.w-\[10\%\]{width:10%}.w-\[12\%\]{width:12%}.w-\[14\%\]{width:14%}.w-\[18px\]{width:18px}.w-\[22rem\]{width:22rem}.w-\[24\%\]{width:24%}.w-\[26\%\]{width:26%}.w-\[28px\]{width:28px}.w-\[28rem\]{width:28rem}.w-\[30rem\]{width:30rem}.w-\[32px\]{width:32px}.w-\[52px\]{width:52px}.w-\[60px\]{width:60px}.w-\[64px\]{width:64px}.w-\[68px\]{width:68px}.w-\[72px\]{width:72px}.w-\[76px\]{width:76px}.w-\[80px\]{width:80px}.w-\[82px\]{width:82px}.w-\[84px\]{width:84px}.w-\[88px\]{width:88px}.w-\[90px\]{width:90px}.w-\[92px\]{width:92px}.w-\[96px\]{width:96px}.w-\[98px\]{width:98px}.w-\[100px\]{width:100px}.w-\[104px\]{width:104px}.w-\[108px\]{width:108px}.w-\[110px\]{width:110px}.w-\[112px\]{width:112px}.w-\[116px\]{width:116px}.w-\[118px\]{width:118px}.w-\[120px\]{width:120px}.w-\[128px\]{width:128px}.w-\[136px\]{width:136px}.w-\[140px\]{width:140px}.w-\[148px\]{width:148px}.w-\[150px\]{width:150px}.w-\[152px\]{width:152px}.w-\[156px\]{width:156px}.w-\[168px\]{width:168px}.w-\[180px\]{width:180px}.w-\[240px\]{width:240px}.w-auto{width:auto}.w-full{width:100%}.max-w-0{max-width:0}.max-w-2xl{max-width:var(--container-2xl)}.max-w-3xl{max-width:var(--container-3xl)}.max-w-4xl{max-width:var(--container-4xl)}.max-w-5xl{max-width:var(--container-5xl)}.max-w-64{max-width:calc(var(--spacing) * 64)}.max-w-\[16rem\]{max-width:16rem}.max-w-\[140px\]{max-width:140px}.max-w-\[150px\]{max-width:150px}.max-w-\[160px\]{max-width:160px}.max-w-\[220px\]{max-width:220px}.max-w-\[240px\]{max-width:240px}.max-w-\[260px\]{max-width:260px}.max-w-\[280px\]{max-width:280px}.max-w-\[300px\]{max-width:300px}.max-w-\[320px\]{max-width:320px}.max-w-\[calc\(100vw-2rem\)\]{max-width:calc(100vw - 2rem)}.max-w-full{max-width:100%}.max-w-lg{max-width:var(--container-lg)}.max-w-md{max-width:var(--container-md)}.max-w-none{max-width:none}.max-w-sm{max-width:var(--container-sm)}.max-w-xl{max-width:var(--container-xl)}.max-w-xs{max-width:var(--container-xs)}.min-w-0{min-width:0}.min-w-5{min-width:calc(var(--spacing) * 5)}.min-w-7{min-width:calc(var(--spacing) * 7)}.min-w-40{min-width:calc(var(--spacing) * 40)}.min-w-48{min-width:calc(var(--spacing) * 48)}.min-w-\[11\.5rem\]{min-width:11.5rem}.min-w-\[20px\]{min-width:20px}.min-w-\[48rem\]{min-width:48rem}.min-w-\[54rem\]{min-width:54rem}.min-w-\[60px\]{min-width:60px}.min-w-\[440px\]{min-width:440px}.min-w-\[620px\]{min-width:620px}.min-w-\[640px\]{min-width:640px}.min-w-\[720px\]{min-width:720px}.min-w-\[760px\]{min-width:760px}.min-w-\[860px\]{min-width:860px}.min-w-\[880px\]{min-width:880px}.min-w-\[920px\]{min-width:920px}.min-w-\[980px\]{min-width:980px}.min-w-\[1040px\]{min-width:1040px}.flex-1{flex:1}.flex-shrink{flex-shrink:1}.shrink-0{flex-shrink:0}.table-fixed{table-layout:fixed}.-translate-x-full{--tw-translate-x:-100%;translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-x-0{--tw-translate-x:0px;translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-x-full{--tw-translate-x:100%;translate:var(--tw-translate-x) var(--tw-translate-y)}.-translate-y-1{--tw-translate-y:calc(var(--spacing) * -1);translate:var(--tw-translate-x) var(--tw-translate-y)}.-translate-y-1\/2{--tw-translate-y:calc(calc(1 / 2 * 100%) * -1);translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-y-0{--tw-translate-y:0px;translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-y-1{--tw-translate-y:var(--spacing);translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-y-2{--tw-translate-y:calc(var(--spacing) * 2);translate:var(--tw-translate-x) var(--tw-translate-y)}.scale-95{--tw-scale-x:95%;--tw-scale-y:95%;--tw-scale-z:95%;scale:var(--tw-scale-x) var(--tw-scale-y)}.scale-100{--tw-scale-x:100%;--tw-scale-y:100%;--tw-scale-z:100%;scale:var(--tw-scale-x) var(--tw-scale-y)}.rotate-90{rotate:90deg}.rotate-180{rotate:180deg}.animate-pulse{animation:var(--animate-pulse)}.animate-spin{animation:var(--animate-spin)}.cursor-default{cursor:default}.cursor-help{cursor:help}.cursor-not-allowed{cursor:not-allowed}.cursor-pointer{cursor:pointer}.cursor-wait{cursor:wait}.resize{resize:both}.list-decimal{list-style-type:decimal}.list-disc{list-style-type:disc}.list-none{list-style-type:none}.grid-cols-1{grid-template-columns:repeat(1,minmax(0,1fr))}.grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.grid-cols-\[1fr_auto\]{grid-template-columns:1fr auto}.flex-col{flex-direction:column}.flex-col-reverse{flex-direction:column-reverse}.flex-nowrap{flex-wrap:nowrap}.flex-wrap{flex-wrap:wrap}.items-baseline{align-items:baseline}.items-center{align-items:center}.items-end{align-items:flex-end}.items-start{align-items:flex-start}.justify-between{justify-content:space-between}.justify-center{justify-content:center}.justify-end{justify-content:flex-end}.justify-start{justify-content:flex-start}.gap-0{gap:0}.gap-0\.5{gap:calc(var(--spacing) * .5)}.gap-1{gap:var(--spacing)}.gap-1\.5{gap:calc(var(--spacing) * 1.5)}.gap-2{gap:calc(var(--spacing) * 2)}.gap-2\.5{gap:calc(var(--spacing) * 2.5)}.gap-3{gap:calc(var(--spacing) * 3)}.gap-4{gap:calc(var(--spacing) * 4)}.gap-5{gap:calc(var(--spacing) * 5)}.gap-6{gap:calc(var(--spacing) * 6)}.gap-8{gap:calc(var(--spacing) * 8)}:where(.space-y-0>:not(:last-child)){--tw-space-y-reverse:0;margin-block:0}:where(.space-y-0\.5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * .5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * .5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-1>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-1\.5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 1.5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 1.5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-2>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-3>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-4>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-6>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.gap-x-1{column-gap:var(--spacing)}.gap-x-3{column-gap:calc(var(--spacing) * 3)}.gap-x-4{column-gap:calc(var(--spacing) * 4)}.gap-x-5{column-gap:calc(var(--spacing) * 5)}.gap-x-6{column-gap:calc(var(--spacing) * 6)}.gap-y-1{row-gap:var(--spacing)}.gap-y-1\.5{row-gap:calc(var(--spacing) * 1.5)}.gap-y-2{row-gap:calc(var(--spacing) * 2)}:where(.divide-y>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)))}:where(.divide-pb-border>:not(:last-child)){border-color:var(--pb-border)}:where(.divide-pb-border-subtle>:not(:last-child)){border-color:var(--pb-border-subtle)}:where(.divide-pb-border\/40>:not(:last-child)){border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){:where(.divide-pb-border\/40>:not(:last-child)){border-color:color-mix(in oklab, var(--pb-border) 40%, transparent)}}.self-start{align-self:flex-start}.truncate{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.overflow-auto{overflow:auto}.overflow-hidden{overflow:hidden}.overflow-visible{overflow:visible}.overflow-x-auto{overflow-x:auto}.overflow-x-hidden{overflow-x:hidden}.overflow-y-auto{overflow-y:auto}.overscroll-contain{overscroll-behavior:contain}.\!rounded-none{border-radius:0!important}.rounded{border-radius:.25rem}.rounded-2xl{border-radius:var(--radius-2xl)}.rounded-3xl{border-radius:var(--radius-3xl)}.rounded-\[var\(--radius-card\)\]{border-radius:var(--radius-card)}.rounded-full{border-radius:3.40282e38px}.rounded-lg{border-radius:var(--radius-lg)}.rounded-md{border-radius:var(--radius-md)}.rounded-sm{border-radius:var(--radius-sm)}.rounded-xl{border-radius:var(--radius-xl)}.rounded-t-xl{border-top-left-radius:var(--radius-xl);border-top-right-radius:var(--radius-xl)}.rounded-b-xl{border-bottom-right-radius:var(--radius-xl);border-bottom-left-radius:var(--radius-xl)}.\!border-0{border-style:var(--tw-border-style)!important;border-width:0!important}.border{border-style:var(--tw-border-style);border-width:1px}.border-0{border-style:var(--tw-border-style);border-width:0}.border-\[3px\]{border-style:var(--tw-border-style);border-width:3px}.border-t{border-top-style:var(--tw-border-style);border-top-width:1px}.border-b{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.border-l{border-left-style:var(--tw-border-style);border-left-width:1px}.border-l-2{border-left-style:var(--tw-border-style);border-left-width:2px}.border-dashed{--tw-border-style:dashed;border-style:dashed}.\!border-pb-warning\/35{border-color:var(--pb-warning)!important}@supports (color:color-mix(in lab, red, red)){.\!border-pb-warning\/35{border-color:color-mix(in oklab, var(--pb-warning) 35%, transparent)!important}}.border-current\/15{border-color:currentColor}@supports (color:color-mix(in lab, red, red)){.border-current\/15{border-color:color-mix(in oklab, currentcolor 15%, transparent)}}.border-pb-border{border-color:var(--pb-border)}.border-pb-border-hover,.border-pb-border-hover\/50{border-color:var(--pb-border-hover)}@supports (color:color-mix(in lab, red, red)){.border-pb-border-hover\/50{border-color:color-mix(in oklab, var(--pb-border-hover) 50%, transparent)}}.border-pb-border-subtle{border-color:var(--pb-border-subtle)}.border-pb-border\/50{border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.border-pb-border\/50{border-color:color-mix(in oklab, var(--pb-border) 50%, transparent)}}.border-pb-border\/70{border-color:var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.border-pb-border\/70{border-color:color-mix(in oklab, var(--pb-border) 70%, transparent)}}.border-pb-error,.border-pb-error\/30{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.border-pb-error\/30{border-color:color-mix(in oklab, var(--pb-error) 30%, transparent)}}.border-pb-error\/35{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.border-pb-error\/35{border-color:color-mix(in oklab, var(--pb-error) 35%, transparent)}}.border-pb-error\/40{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.border-pb-error\/40{border-color:color-mix(in oklab, var(--pb-error) 40%, transparent)}}.border-pb-info\/30{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.border-pb-info\/30{border-color:color-mix(in oklab, var(--pb-info) 30%, transparent)}}.border-pb-info\/40{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.border-pb-info\/40{border-color:color-mix(in oklab, var(--pb-info) 40%, transparent)}}.border-pb-interactive,.border-pb-interactive\/25{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.border-pb-interactive\/25{border-color:color-mix(in oklab, var(--pb-interactive) 25%, transparent)}}.border-pb-interactive\/30{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.border-pb-interactive\/30{border-color:color-mix(in oklab, var(--pb-interactive) 30%, transparent)}}.border-pb-interactive\/40{border-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.border-pb-interactive\/40{border-color:color-mix(in oklab, var(--pb-interactive) 40%, transparent)}}.border-pb-success\/30{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.border-pb-success\/30{border-color:color-mix(in oklab, var(--pb-success) 30%, transparent)}}.border-pb-success\/35{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.border-pb-success\/35{border-color:color-mix(in oklab, var(--pb-success) 35%, transparent)}}.border-pb-success\/40{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.border-pb-success\/40{border-color:color-mix(in oklab, var(--pb-success) 40%, transparent)}}.border-pb-surface{border-color:var(--pb-bg-surface)}.border-pb-warning\/20{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/20{border-color:color-mix(in oklab, var(--pb-warning) 20%, transparent)}}.border-pb-warning\/25{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/25{border-color:color-mix(in oklab, var(--pb-warning) 25%, transparent)}}.border-pb-warning\/30{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/30{border-color:color-mix(in oklab, var(--pb-warning) 30%, transparent)}}.border-pb-warning\/35{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/35{border-color:color-mix(in oklab, var(--pb-warning) 35%, transparent)}}.border-pb-warning\/40{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/40{border-color:color-mix(in oklab, var(--pb-warning) 40%, transparent)}}.border-pb-warning\/50{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.border-pb-warning\/50{border-color:color-mix(in oklab, var(--pb-warning) 50%, transparent)}}.border-transparent{border-color:#0000}.border-t-pb-interactive{border-top-color:var(--pb-interactive)}.\!bg-pb-warning-dim\/45{background-color:var(--pb-warning-dim)!important}@supports (color:color-mix(in lab, red, red)){.\!bg-pb-warning-dim\/45{background-color:color-mix(in oklab, var(--pb-warning-dim) 45%, transparent)!important}}.\!bg-transparent{background-color:#0000!important}.bg-black\/80{background-color:#000c}@supports (color:color-mix(in lab, red, red)){.bg-black\/80{background-color:color-mix(in oklab, var(--color-black) 80%, transparent)}}.bg-pb-base{background-color:var(--pb-bg-base)}.bg-pb-card{background-color:var(--pb-bg-card)}.bg-pb-card-hover,.bg-pb-card-hover\/20{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/20{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 20%, transparent)}}.bg-pb-card-hover\/25{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/25{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 25%, transparent)}}.bg-pb-card-hover\/35{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/35{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 35%, transparent)}}.bg-pb-card-hover\/40{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/40{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 40%, transparent)}}.bg-pb-card-hover\/45{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/45{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 45%, transparent)}}.bg-pb-card-hover\/50{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/50{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 50%, transparent)}}.bg-pb-card-hover\/60{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card-hover\/60{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 60%, transparent)}}.bg-pb-card\/40{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/40{background-color:color-mix(in oklab, var(--pb-bg-card) 40%, transparent)}}.bg-pb-card\/50{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/50{background-color:color-mix(in oklab, var(--pb-bg-card) 50%, transparent)}}.bg-pb-card\/60{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/60{background-color:color-mix(in oklab, var(--pb-bg-card) 60%, transparent)}}.bg-pb-card\/70{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/70{background-color:color-mix(in oklab, var(--pb-bg-card) 70%, transparent)}}.bg-pb-card\/80{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/80{background-color:color-mix(in oklab, var(--pb-bg-card) 80%, transparent)}}.bg-pb-card\/85{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/85{background-color:color-mix(in oklab, var(--pb-bg-card) 85%, transparent)}}.bg-pb-card\/95{background-color:var(--pb-bg-card)}@supports (color:color-mix(in lab, red, red)){.bg-pb-card\/95{background-color:color-mix(in oklab, var(--pb-bg-card) 95%, transparent)}}.bg-pb-error{background-color:var(--pb-error)}.bg-pb-error-dim,.bg-pb-error-dim\/70{background-color:var(--pb-error-dim)}@supports (color:color-mix(in lab, red, red)){.bg-pb-error-dim\/70{background-color:color-mix(in oklab, var(--pb-error-dim) 70%, transparent)}}.bg-pb-error\/10{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.bg-pb-error\/10{background-color:color-mix(in oklab, var(--pb-error) 10%, transparent)}}.bg-pb-error\/15{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.bg-pb-error\/15{background-color:color-mix(in oklab, var(--pb-error) 15%, transparent)}}.bg-pb-info{background-color:var(--pb-info)}.bg-pb-info-dim{background-color:var(--pb-info-dim)}.bg-pb-info\/10{background-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.bg-pb-info\/10{background-color:color-mix(in oklab, var(--pb-info) 10%, transparent)}}.bg-pb-input{background-color:var(--pb-bg-input)}.bg-pb-interactive{background-color:var(--pb-interactive)}.bg-pb-interactive-dim{background-color:var(--pb-interactive-dim)}.bg-pb-interactive\/15{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.bg-pb-interactive\/15{background-color:color-mix(in oklab, var(--pb-interactive) 15%, transparent)}}.bg-pb-overlay,.bg-pb-overlay\/45{background-color:var(--pb-bg-overlay)}@supports (color:color-mix(in lab, red, red)){.bg-pb-overlay\/45{background-color:color-mix(in oklab, var(--pb-bg-overlay) 45%, transparent)}}.bg-pb-purple-dim{background-color:var(--pb-purple-dim)}.bg-pb-success{background-color:var(--pb-success)}.bg-pb-success-dim,.bg-pb-success-dim\/70{background-color:var(--pb-success-dim)}@supports (color:color-mix(in lab, red, red)){.bg-pb-success-dim\/70{background-color:color-mix(in oklab, var(--pb-success-dim) 70%, transparent)}}.bg-pb-success\/10{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.bg-pb-success\/10{background-color:color-mix(in oklab, var(--pb-success) 10%, transparent)}}.bg-pb-success\/15{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.bg-pb-success\/15{background-color:color-mix(in oklab, var(--pb-success) 15%, transparent)}}.bg-pb-surface,.bg-pb-surface\/30{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/30{background-color:color-mix(in oklab, var(--pb-bg-surface) 30%, transparent)}}.bg-pb-surface\/35{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/35{background-color:color-mix(in oklab, var(--pb-bg-surface) 35%, transparent)}}.bg-pb-surface\/40{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/40{background-color:color-mix(in oklab, var(--pb-bg-surface) 40%, transparent)}}.bg-pb-surface\/50{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/50{background-color:color-mix(in oklab, var(--pb-bg-surface) 50%, transparent)}}.bg-pb-surface\/60{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/60{background-color:color-mix(in oklab, var(--pb-bg-surface) 60%, transparent)}}.bg-pb-surface\/80{background-color:var(--pb-bg-surface)}@supports (color:color-mix(in lab, red, red)){.bg-pb-surface\/80{background-color:color-mix(in oklab, var(--pb-bg-surface) 80%, transparent)}}.bg-pb-text-dim{background-color:var(--pb-text-dim)}.bg-pb-warning{background-color:var(--pb-warning)}.bg-pb-warning-dim,.bg-pb-warning-dim\/25{background-color:var(--pb-warning-dim)}@supports (color:color-mix(in lab, red, red)){.bg-pb-warning-dim\/25{background-color:color-mix(in oklab, var(--pb-warning-dim) 25%, transparent)}}.bg-pb-warning-dim\/45{background-color:var(--pb-warning-dim)}@supports (color:color-mix(in lab, red, red)){.bg-pb-warning-dim\/45{background-color:color-mix(in oklab, var(--pb-warning-dim) 45%, transparent)}}.bg-pb-warning\/5{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.bg-pb-warning\/5{background-color:color-mix(in oklab, var(--pb-warning) 5%, transparent)}}.bg-pb-warning\/10{background-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.bg-pb-warning\/10{background-color:color-mix(in oklab, var(--pb-warning) 10%, transparent)}}.bg-slate-950\/70{background-color:#020618b3}@supports (color:color-mix(in lab, red, red)){.bg-slate-950\/70{background-color:color-mix(in oklab, var(--color-slate-950) 70%, transparent)}}.bg-transparent{background-color:#0000}.object-contain{object-fit:contain}.object-cover{object-fit:cover}.object-top{object-position:top}.\!p-0{padding:0!important}.p-0{padding:0}.p-1{padding:var(--spacing)}.p-2{padding:calc(var(--spacing) * 2)}.p-2\.5{padding:calc(var(--spacing) * 2.5)}.p-3{padding:calc(var(--spacing) * 3)}.p-4{padding:calc(var(--spacing) * 4)}.p-5{padding:calc(var(--spacing) * 5)}.p-6{padding:calc(var(--spacing) * 6)}.p-8{padding:calc(var(--spacing) * 8)}.\!px-0{padding-inline:0!important}.\!px-2\.5{padding-inline:calc(var(--spacing) * 2.5)!important}.\!px-3{padding-inline:calc(var(--spacing) * 3)!important}.\!px-4{padding-inline:calc(var(--spacing) * 4)!important}.px-0{padding-inline:0}.px-1{padding-inline:var(--spacing)}.px-1\.5{padding-inline:calc(var(--spacing) * 1.5)}.px-2{padding-inline:calc(var(--spacing) * 2)}.px-2\.5{padding-inline:calc(var(--spacing) * 2.5)}.px-3{padding-inline:calc(var(--spacing) * 3)}.px-3\.5{padding-inline:calc(var(--spacing) * 3.5)}.px-4{padding-inline:calc(var(--spacing) * 4)}.px-5{padding-inline:calc(var(--spacing) * 5)}.px-6{padding-inline:calc(var(--spacing) * 6)}.\!py-0{padding-block:0!important}.\!py-1\.5{padding-block:calc(var(--spacing) * 1.5)!important}.\!py-2{padding-block:calc(var(--spacing) * 2)!important}.\!py-2\.5{padding-block:calc(var(--spacing) * 2.5)!important}.py-0\.5{padding-block:calc(var(--spacing) * .5)}.py-1{padding-block:var(--spacing)}.py-1\.5{padding-block:calc(var(--spacing) * 1.5)}.py-2{padding-block:calc(var(--spacing) * 2)}.py-2\.5{padding-block:calc(var(--spacing) * 2.5)}.py-3{padding-block:calc(var(--spacing) * 3)}.py-4{padding-block:calc(var(--spacing) * 4)}.py-5{padding-block:calc(var(--spacing) * 5)}.py-6{padding-block:calc(var(--spacing) * 6)}.py-8{padding-block:calc(var(--spacing) * 8)}.py-10{padding-block:calc(var(--spacing) * 10)}.py-12{padding-block:calc(var(--spacing) * 12)}.py-\[3px\]{padding-block:3px}.pt-0\.5{padding-top:calc(var(--spacing) * .5)}.pt-1{padding-top:var(--spacing)}.pt-1\.5{padding-top:calc(var(--spacing) * 1.5)}.pt-2{padding-top:calc(var(--spacing) * 2)}.pt-3{padding-top:calc(var(--spacing) * 3)}.pt-4{padding-top:calc(var(--spacing) * 4)}.pt-5{padding-top:calc(var(--spacing) * 5)}.pt-6{padding-top:calc(var(--spacing) * 6)}.pt-12{padding-top:calc(var(--spacing) * 12)}.pr-1{padding-right:var(--spacing)}.pr-3{padding-right:calc(var(--spacing) * 3)}.pr-4{padding-right:calc(var(--spacing) * 4)}.\!pb-0{padding-bottom:0!important}.pb-2{padding-bottom:calc(var(--spacing) * 2)}.pb-4{padding-bottom:calc(var(--spacing) * 4)}.pl-5{padding-left:calc(var(--spacing) * 5)}.pl-6{padding-left:calc(var(--spacing) * 6)}.\!text-left{text-align:left!important}.text-center{text-align:center}.text-left{text-align:left}.text-right{text-align:right}.align-top{vertical-align:top}.font-display{font-family:Bricolage Grotesque,sans-serif}.font-mono{font-family:JetBrains Mono,Fira Code,monospace}.font-sans{font-family:DM Sans,sans-serif}.text-2xl{font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height))}.text-base{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height))}.text-lg{font-size:var(--text-lg);line-height:var(--tw-leading,var(--text-lg--line-height))}.text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.text-xl{font-size:var(--text-xl);line-height:var(--tw-leading,var(--text-xl--line-height))}.text-xs{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height))}.text-\[0\.72rem\]{font-size:.72rem}.text-\[0\.78rem\]{font-size:.78rem}.text-\[0\.82rem\]{font-size:.82rem}.text-\[0\.88rem\]{font-size:.88rem}.text-\[9px\]{font-size:9px}.text-\[10px\]{font-size:10px}.text-\[11px\]{font-size:11px}.text-\[12px\]{font-size:12px}.text-\[13px\]{font-size:13px}.leading-4{--tw-leading:calc(var(--spacing) * 4);line-height:calc(var(--spacing) * 4)}.leading-5{--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5)}.leading-6{--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6)}.leading-7{--tw-leading:calc(var(--spacing) * 7);line-height:calc(var(--spacing) * 7)}.leading-\[1\.5\]{--tw-leading:1.5;line-height:1.5}.leading-\[1\.6\]{--tw-leading:1.6;line-height:1.6}.leading-none{--tw-leading:1;line-height:1}.leading-relaxed{--tw-leading:var(--leading-relaxed);line-height:var(--leading-relaxed)}.leading-tight{--tw-leading:var(--leading-tight);line-height:var(--leading-tight)}.font-black{--tw-font-weight:var(--font-weight-black);font-weight:var(--font-weight-black)}.font-bold{--tw-font-weight:var(--font-weight-bold);font-weight:var(--font-weight-bold)}.font-extrabold{--tw-font-weight:var(--font-weight-extrabold);font-weight:var(--font-weight-extrabold)}.font-medium{--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium)}.font-normal{--tw-font-weight:var(--font-weight-normal);font-weight:var(--font-weight-normal)}.font-semibold{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.tracking-\[0\.2em\]{--tw-tracking:.2em;letter-spacing:.2em}.tracking-\[0\.12em\]{--tw-tracking:.12em;letter-spacing:.12em}.tracking-\[0\.14em\]{--tw-tracking:.14em;letter-spacing:.14em}.tracking-\[0\.16em\]{--tw-tracking:.16em;letter-spacing:.16em}.tracking-\[0\.18em\]{--tw-tracking:.18em;letter-spacing:.18em}.tracking-\[0\.22em\]{--tw-tracking:.22em;letter-spacing:.22em}.tracking-tight{--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight)}.tracking-wide{--tw-tracking:var(--tracking-wide);letter-spacing:var(--tracking-wide)}.tracking-wider{--tw-tracking:var(--tracking-wider);letter-spacing:var(--tracking-wider)}.break-words{overflow-wrap:break-word}.break-all{word-break:break-all}.whitespace-normal{white-space:normal}.whitespace-nowrap{white-space:nowrap}.whitespace-pre-line{white-space:pre-line}.whitespace-pre-wrap{white-space:pre-wrap}.\!text-pb-text{color:var(--pb-text-primary)!important}.\!text-pb-warning{color:var(--pb-warning)!important}.text-pb-border{color:var(--pb-border)}.text-pb-brand{color:var(--pb-brand)}.text-pb-error{color:var(--pb-error)}.text-pb-info{color:var(--pb-info)}.text-pb-interactive{color:var(--pb-interactive)}.text-pb-purple{color:var(--pb-purple)}.text-pb-success{color:var(--pb-success)}.text-pb-text{color:var(--pb-text-primary)}.text-pb-text-dim{color:var(--pb-text-dim)}.text-pb-text-inverse{color:var(--pb-text-inverse)}.text-pb-text-sec{color:var(--pb-text-secondary)}.text-pb-warning,.text-pb-warning\/70{color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.text-pb-warning\/70{color:color-mix(in oklab, var(--pb-warning) 70%, transparent)}}.text-red-300{color:var(--color-red-300)}.text-white{color:var(--color-white)}.capitalize{text-transform:capitalize}.lowercase{text-transform:lowercase}.uppercase{text-transform:uppercase}.italic{font-style:italic}.tabular-nums{--tw-numeric-spacing:tabular-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.no-underline{text-decoration-line:none}.underline{text-decoration-line:underline}.underline-offset-4{text-underline-offset:4px}.accent-pb-interactive{accent-color:var(--pb-interactive)}.opacity-0{opacity:0}.opacity-25{opacity:.25}.opacity-30{opacity:.3}.opacity-40{opacity:.4}.opacity-50{opacity:.5}.opacity-60{opacity:.6}.opacity-70{opacity:.7}.opacity-75{opacity:.75}.opacity-90{opacity:.9}.opacity-100{opacity:1}.shadow-2xl{--tw-shadow:0 25px 50px -12px var(--tw-shadow-color,#00000040);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-lg{--tw-shadow:0 10px 15px -3px var(--tw-shadow-color,#0000001a), 0 4px 6px -4px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-none{--tw-shadow:0 0 #0000;box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-sm{--tw-shadow:0 1px 3px 0 var(--tw-shadow-color,#0000001a), 0 1px 2px -1px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-xl{--tw-shadow:0 20px 25px -5px var(--tw-shadow-color,#0000001a), 0 8px 10px -6px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-xs{--tw-shadow:0 1px 2px 0 var(--tw-shadow-color,#0000000d);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.outline{outline-style:var(--tw-outline-style);outline-width:1px}.blur{--tw-blur:blur(8px);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}.filter{filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}.backdrop-blur-\[1px\]{--tw-backdrop-blur:blur(1px);-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.backdrop-blur-xs{--tw-backdrop-blur:blur(var(--blur-xs));-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.backdrop-filter{-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.transition{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to,opacity,box-shadow,transform,translate,scale,rotate,filter,-webkit-backdrop-filter,backdrop-filter,display,content-visibility,overlay,pointer-events;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-all{transition-property:all;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-colors{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-opacity{transition-property:opacity;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-transform{transition-property:transform,translate,scale,rotate;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-none{transition-property:none}.delay-75{transition-delay:75ms}.duration-75{--tw-duration:75ms;transition-duration:75ms}.duration-100{--tw-duration:.1s;transition-duration:.1s}.duration-150{--tw-duration:.15s;transition-duration:.15s}.duration-200{--tw-duration:.2s;transition-duration:.2s}.duration-300{--tw-duration:.3s;transition-duration:.3s}.duration-500{--tw-duration:.5s;transition-duration:.5s}.ease-in{--tw-ease:var(--ease-in);transition-timing-function:var(--ease-in)}.ease-linear{--tw-ease:linear;transition-timing-function:linear}.ease-out{--tw-ease:var(--ease-out);transition-timing-function:var(--ease-out)}.outline-none{--tw-outline-style:none;outline-style:none}.select-none{-webkit-user-select:none;user-select:none}@media (hover:hover){.group-hover\:border-pb-text-sec:is(:where(.group):hover *){border-color:var(--pb-text-secondary)}.group-hover\:text-pb-interactive:is(:where(.group):hover *){color:var(--pb-interactive)}.group-hover\:text-pb-text:is(:where(.group):hover *){color:var(--pb-text-primary)}.group-hover\:text-pb-text-sec:is(:where(.group):hover *){color:var(--pb-text-secondary)}.group-hover\:opacity-40:is(:where(.group):hover *){opacity:.4}}.placeholder\:text-pb-text-dim::placeholder{color:var(--pb-text-dim)}.placeholder\:text-pb-text-sec::placeholder{color:var(--pb-text-secondary)}.first\:pt-0:first-child{padding-top:0}.last\:mb-0:last-child{margin-bottom:0}.last\:border-b-0:last-child{border-bottom-style:var(--tw-border-style);border-bottom-width:0}.last\:pb-0:last-child{padding-bottom:0}@media (hover:hover){.hover\:\!border-pb-warning\/45:hover{border-color:var(--pb-warning)!important}@supports (color:color-mix(in lab, red, red)){.hover\:\!border-pb-warning\/45:hover{border-color:color-mix(in oklab, var(--pb-warning) 45%, transparent)!important}}.hover\:border-pb-border-hover:hover{border-color:var(--pb-border-hover)}.hover\:border-pb-border-strong:hover{border-color:var(--pb-border-strong)}.hover\:border-pb-error\/40:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-error\/40:hover{border-color:color-mix(in oklab, var(--pb-error) 40%, transparent)}}.hover\:border-pb-info\/40:hover{border-color:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-info\/40:hover{border-color:color-mix(in oklab, var(--pb-info) 40%, transparent)}}.hover\:border-pb-purple\/40:hover{border-color:var(--pb-purple)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-purple\/40:hover{border-color:color-mix(in oklab, var(--pb-purple) 40%, transparent)}}.hover\:border-pb-success\/40:hover{border-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-success\/40:hover{border-color:color-mix(in oklab, var(--pb-success) 40%, transparent)}}.hover\:border-pb-warning\/40:hover{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.hover\:border-pb-warning\/40:hover{border-color:color-mix(in oklab, var(--pb-warning) 40%, transparent)}}.hover\:\!bg-pb-warning-dim:hover{background-color:var(--pb-warning-dim)!important}.hover\:\!bg-transparent:hover{background-color:#0000!important}.hover\:bg-pb-card:hover{background-color:var(--pb-bg-card)}.hover\:bg-pb-card-hover:hover,.hover\:bg-pb-card-hover\/20:hover{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-card-hover\/20:hover{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 20%, transparent)}}.hover\:bg-pb-card-hover\/45:hover{background-color:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-card-hover\/45:hover{background-color:color-mix(in oklab, var(--pb-bg-card-hover) 45%, transparent)}}.hover\:bg-pb-error-dim:hover{background-color:var(--pb-error-dim)}.hover\:bg-pb-error\/25:hover{background-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-error\/25:hover{background-color:color-mix(in oklab, var(--pb-error) 25%, transparent)}}.hover\:bg-pb-info-dim:hover{background-color:var(--pb-info-dim)}.hover\:bg-pb-interactive-hover:hover{background-color:var(--pb-interactive-hover)}.hover\:bg-pb-interactive\/10:hover{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-interactive\/10:hover{background-color:color-mix(in oklab, var(--pb-interactive) 10%, transparent)}}.hover\:bg-pb-interactive\/25:hover{background-color:var(--pb-interactive)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-interactive\/25:hover{background-color:color-mix(in oklab, var(--pb-interactive) 25%, transparent)}}.hover\:bg-pb-purple-dim:hover{background-color:var(--pb-purple-dim)}.hover\:bg-pb-success-dim:hover{background-color:var(--pb-success-dim)}.hover\:bg-pb-success\/25:hover{background-color:var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.hover\:bg-pb-success\/25:hover{background-color:color-mix(in oklab, var(--pb-success) 25%, transparent)}}.hover\:bg-pb-warning-dim:hover{background-color:var(--pb-warning-dim)}.hover\:\!text-pb-text:hover{color:var(--pb-text-primary)!important}.hover\:\!text-pb-warning:hover{color:var(--pb-warning)!important}.hover\:text-pb-error:hover{color:var(--pb-error)}.hover\:text-pb-interactive:hover{color:var(--pb-interactive)}.hover\:text-pb-interactive-hover:hover{color:var(--pb-interactive-hover)}.hover\:text-pb-text:hover{color:var(--pb-text-primary)}.hover\:text-pb-text-sec:hover{color:var(--pb-text-secondary)}.hover\:underline:hover{text-decoration-line:underline}.hover\:opacity-95:hover{opacity:.95}.hover\:opacity-100:hover{opacity:1}.hover\:brightness-90:hover{--tw-brightness:brightness(90%);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}}.focus\:border-transparent:focus{border-color:#0000}.focus\:ring-1:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(1px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.focus\:ring-2:focus{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.focus\:ring-pb-border-hover:focus{--tw-ring-color:var(--pb-border-hover)}.focus\:ring-pb-error:focus{--tw-ring-color:var(--pb-error)}.focus\:ring-pb-interactive:focus{--tw-ring-color:var(--pb-interactive)}.focus\:ring-pb-warning:focus{--tw-ring-color:var(--pb-warning)}.focus\:ring-offset-0:focus{--tw-ring-offset-width:0px;--tw-ring-offset-shadow:var(--tw-ring-inset,) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color)}.focus\:ring-offset-2:focus{--tw-ring-offset-width:2px;--tw-ring-offset-shadow:var(--tw-ring-inset,) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color)}.focus\:ring-offset-pb-card:focus{--tw-ring-offset-color:var(--pb-bg-card)}.focus\:outline-hidden:focus{--tw-outline-style:none;outline-style:none}@media (forced-colors:active){.focus\:outline-hidden:focus{outline-offset:2px;outline:2px solid #0000}}.focus-visible\:ring-2:focus-visible{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(2px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.focus-visible\:ring-offset-2:focus-visible{--tw-ring-offset-width:2px;--tw-ring-offset-shadow:var(--tw-ring-inset,) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color)}.focus-visible\:outline-none:focus-visible{--tw-outline-style:none;outline-style:none}.disabled\:cursor-default:disabled{cursor:default}.disabled\:cursor-not-allowed:disabled{cursor:not-allowed}.disabled\:cursor-wait:disabled{cursor:wait}.disabled\:text-pb-text-dim:disabled{color:var(--pb-text-dim)}.disabled\:opacity-20:disabled{opacity:.2}.disabled\:opacity-30:disabled{opacity:.3}.disabled\:opacity-40:disabled{opacity:.4}.disabled\:opacity-50:disabled{opacity:.5}.disabled\:opacity-100:disabled{opacity:1}@media (min-width:40rem){.sm\:col-span-2{grid-column:span 2/span 2}.sm\:ml-auto{margin-left:auto}.sm\:flex{display:flex}.sm\:hidden{display:none}.sm\:inline{display:inline}.sm\:table-cell{display:table-cell}.sm\:max-h-\[90vh\]{max-height:90vh}.sm\:w-14{width:calc(var(--spacing) * 14)}.sm\:w-44{width:calc(var(--spacing) * 44)}.sm\:w-48{width:calc(var(--spacing) * 48)}.sm\:w-56{width:calc(var(--spacing) * 56)}.sm\:w-auto{width:auto}.sm\:max-w-xs{max-width:var(--container-xs)}.sm\:min-w-\[760px\]{min-width:760px}.sm\:shrink-0{flex-shrink:0}.sm\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.sm\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.sm\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.sm\:grid-cols-\[8rem_minmax\(0\,1fr\)\]{grid-template-columns:8rem minmax(0,1fr)}.sm\:grid-cols-\[9rem_minmax\(0\,1fr\)\]{grid-template-columns:9rem minmax(0,1fr)}.sm\:grid-cols-\[minmax\(0\,1fr\)_minmax\(0\,1\.4fr\)_auto\]{grid-template-columns:minmax(0,1fr) minmax(0,1.4fr) auto}.sm\:grid-cols-\[minmax\(0\,3fr\)_minmax\(0\,2fr\)\]{grid-template-columns:minmax(0,3fr) minmax(0,2fr)}.sm\:flex-row{flex-direction:row}.sm\:items-center{align-items:center}.sm\:items-end{align-items:flex-end}.sm\:items-start{align-items:flex-start}.sm\:justify-between{justify-content:space-between}.sm\:justify-end{justify-content:flex-end}.sm\:self-auto{align-self:auto}.sm\:rounded-xl{border-radius:var(--radius-xl)}.sm\:p-4{padding:calc(var(--spacing) * 4)}.sm\:p-5{padding:calc(var(--spacing) * 5)}.sm\:p-6{padding:calc(var(--spacing) * 6)}.sm\:p-7{padding:calc(var(--spacing) * 7)}.sm\:px-5{padding-inline:calc(var(--spacing) * 5)}.sm\:px-6{padding-inline:calc(var(--spacing) * 6)}.sm\:pt-5{padding-top:calc(var(--spacing) * 5)}.sm\:pt-6{padding-top:calc(var(--spacing) * 6)}.sm\:pt-7{padding-top:calc(var(--spacing) * 7)}.sm\:text-2xl{font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height))}}@media (min-width:48rem){.md\:block{display:block}.md\:flex{display:flex}.md\:table-cell{display:table-cell}.md\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.md\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.md\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.md\:grid-cols-\[1fr_auto_1fr_auto\]{grid-template-columns:1fr auto 1fr auto}.md\:grid-cols-\[minmax\(0\,1fr\)_auto_minmax\(0\,1fr\)\]{grid-template-columns:minmax(0,1fr) auto minmax(0,1fr)}.md\:flex-row{flex-direction:row}.md\:items-center{align-items:center}.md\:items-end{align-items:flex-end}.md\:items-start{align-items:flex-start}.md\:justify-between{justify-content:space-between}.md\:justify-center{justify-content:center}}@media (min-width:64rem){.lg\:col-span-2{grid-column:span 2/span 2}.lg\:flex{display:flex}.lg\:hidden{display:none}.lg\:table-cell{display:table-cell}.lg\:w-52{width:calc(var(--spacing) * 52)}.lg\:w-auto{width:auto}.lg\:max-w-\[11rem\]{max-width:11rem}.lg\:max-w-md{max-width:var(--container-md)}.lg\:min-w-\[920px\]{min-width:920px}.lg\:min-w-\[980px\]{min-width:980px}.lg\:min-w-\[1060px\]{min-width:1060px}.lg\:translate-x-0{--tw-translate-x:0px;translate:var(--tw-translate-x) var(--tw-translate-y)}.lg\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.lg\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.lg\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.lg\:grid-cols-6{grid-template-columns:repeat(6,minmax(0,1fr))}.lg\:grid-cols-\[minmax\(0\,1\.1fr\)_minmax\(280px\,0\.9fr\)\]{grid-template-columns:minmax(0,1.1fr) minmax(280px,.9fr)}.lg\:grid-cols-\[minmax\(0\,1\.15fr\)_minmax\(280px\,0\.85fr\)\]{grid-template-columns:minmax(0,1.15fr) minmax(280px,.85fr)}.lg\:grid-cols-\[minmax\(0\,1\.25fr\)_minmax\(280px\,0\.75fr\)\]{grid-template-columns:minmax(0,1.25fr) minmax(280px,.75fr)}.lg\:grid-cols-\[minmax\(0\,1fr\)_minmax\(16rem\,0\.65fr\)\]{grid-template-columns:minmax(0,1fr) minmax(16rem,.65fr)}.lg\:grid-cols-\[minmax\(0\,1fr\)_minmax\(18rem\,24rem\)\]{grid-template-columns:minmax(0,1fr) minmax(18rem,24rem)}.lg\:grid-cols-\[minmax\(0\,280px\)_minmax\(0\,1fr\)\]{grid-template-columns:minmax(0,280px) minmax(0,1fr)}.lg\:grid-cols-\[minmax\(12rem\,0\.7fr\)_minmax\(18rem\,1\.3fr\)\]{grid-template-columns:minmax(12rem,.7fr) minmax(18rem,1.3fr)}.lg\:grid-cols-\[repeat\(5\,minmax\(0\,1fr\)\)\]{grid-template-columns:repeat(5,minmax(0,1fr))}.lg\:flex-col{flex-direction:column}.lg\:flex-row{flex-direction:row}.lg\:items-center{align-items:center}.lg\:items-end{align-items:flex-end}.lg\:items-start{align-items:flex-start}.lg\:justify-between{justify-content:space-between}.lg\:text-right{text-align:right}}@media (min-width:80rem){.xl\:col-span-3{grid-column:span 3/span 3}.xl\:w-32{width:calc(var(--spacing) * 32)}.xl\:w-48{width:calc(var(--spacing) * 48)}.xl\:w-\[34rem\]{width:34rem}.xl\:max-w-sm{max-width:var(--container-sm)}.xl\:min-w-\[30rem\]{min-width:30rem}.xl\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.xl\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.xl\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}.xl\:grid-cols-5{grid-template-columns:repeat(5,minmax(0,1fr))}.xl\:grid-cols-7{grid-template-columns:repeat(7,minmax(0,1fr))}.xl\:grid-cols-\[minmax\(0\,1\.4fr\)_minmax\(320px\,0\.8fr\)\]{grid-template-columns:minmax(0,1.4fr) minmax(320px,.8fr)}.xl\:grid-cols-\[minmax\(0\,1\.45fr\)_minmax\(17rem\,1fr\)\]{grid-template-columns:minmax(0,1.45fr) minmax(17rem,1fr)}.xl\:grid-cols-\[minmax\(0\,1\.55fr\)_minmax\(320px\,0\.85fr\)\]{grid-template-columns:minmax(0,1.55fr) minmax(320px,.85fr)}.xl\:flex-row{flex-direction:row}.xl\:items-end{align-items:flex-end}.xl\:items-start{align-items:flex-start}.xl\:justify-between{justify-content:space-between}.xl\:justify-end{justify-content:flex-end}}@media (min-width:96rem){.\32 xl\:grid-cols-\[minmax\(0\,1\.45fr\)_minmax\(320px\,0\.75fr\)\]{grid-template-columns:minmax(0,1.45fr) minmax(320px,.75fr)}}}@font-face{font-family:Bricolage Grotesque;font-style:normal;font-weight:800;font-display:swap;src:url(/static/fonts/bricolage-grotesque-800.woff2)format("woff2")}@font-face{font-family:DM Sans;font-style:normal;font-weight:300 700;font-display:swap;src:url(/static/fonts/dm-sans-variable.woff2)format("woff2")}@font-face{font-family:Syne;font-style:normal;font-weight:400;font-display:swap;src:url(/static/fonts/syne-400.ttf)format("truetype")}@font-face{font-family:Syne;font-style:normal;font-weight:500;font-display:swap;src:url(/static/fonts/syne-500.ttf)format("truetype")}@font-face{font-family:Syne;font-style:normal;font-weight:600;font-display:swap;src:url(/static/fonts/syne-600.ttf)format("truetype")}@font-face{font-family:Syne;font-style:normal;font-weight:700;font-display:swap;src:url(/static/fonts/syne-700.ttf)format("truetype")}@font-face{font-family:Syne;font-style:normal;font-weight:800;font-display:swap;src:url(/static/fonts/syne-800.ttf)format("truetype")}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:400;font-display:swap;src:url(/static/fonts/jetbrains-mono-400.ttf)format("truetype")}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:500;font-display:swap;src:url(/static/fonts/jetbrains-mono-500.ttf)format("truetype")}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:600;font-display:swap;src:url(/static/fonts/jetbrains-mono-600.ttf)format("truetype")}@font-face{font-family:JetBrains Mono;font-style:normal;font-weight:700;font-display:swap;src:url(/static/fonts/jetbrains-mono-700.ttf)format("truetype")}.no-transitions,.no-transitions *,.no-transitions :before,.no-transitions :after,.boot-no-transitions,.boot-no-transitions *,.boot-no-transitions :before,.boot-no-transitions :after{transition-duration:0s!important}.safe-area-pb{padding-bottom:env(safe-area-inset-bottom,0px)}.skip-link{z-index:100;background:var(--pb-brand);color:var(--pb-text-inverse);border-radius:0 0 .5rem .5rem;padding:.5rem 1rem;font-size:.875rem;font-weight:600;text-decoration:none;transition:top .15s ease-out;position:absolute;top:-100%;left:1rem}.skip-link:focus{outline:2px solid var(--pb-interactive);outline-offset:2px;top:0}input[type=range].range-pb{appearance:none;background:var(--pb-border-hover);cursor:pointer;border-radius:9999px;outline:none;height:6px}input[type=range].range-pb::-webkit-slider-thumb{appearance:none;background:var(--pb-interactive);border:2px solid var(--pb-bg-card);width:18px;height:18px;box-shadow:var(--pb-shadow-1);cursor:pointer;border-radius:50%}input[type=range].range-pb::-moz-range-thumb{background:var(--pb-interactive);border:2px solid var(--pb-bg-card);width:18px;height:18px;box-shadow:var(--pb-shadow-1);cursor:pointer;border-radius:50%}.scrollbar-hidden{scrollbar-width:none;-ms-overflow-style:none}.scrollbar-hidden::-webkit-scrollbar{display:none}::-webkit-scrollbar{width:8px;height:8px}::-webkit-scrollbar-track{background:var(--pb-bg-surface)}::-webkit-scrollbar-thumb{background:var(--pb-border-strong);border-radius:4px}::-webkit-scrollbar-thumb:hover{background:var(--pb-text-dim)}*{scrollbar-width:thin;scrollbar-color:var(--pb-border-strong) var(--pb-bg-surface)}.htmx-indicator{opacity:0;transition:opacity .2s ease-in}.htmx-request .htmx-indicator,.htmx-request.htmx-indicator{opacity:1}#content[data-page-swap-phase=leaving],#content[data-page-swap-phase=entering]{pointer-events:none}#content[data-detail-history-hidden=true]{opacity:0;pointer-events:none}@media (prefers-reduced-motion:reduce){html:focus-within{scroll-behavior:auto}*,:before,:after{scroll-behavior:auto!important;transition-duration:.01ms!important;animation-duration:.01ms!important;animation-iteration-count:1!important}#content{transition:none}}@keyframes toast-enter{0%{opacity:0;transform:translate(100%)}to{opacity:1;transform:translate(0)}}@keyframes toast-exit{0%{opacity:1;transform:translate(0)}to{opacity:0;transform:translate(100%)}}.toast-enter{animation:.3s ease-out forwards toast-enter}.toast-exit{animation:.3s ease-in forwards toast-exit}.sidebar-transition{transition:width .2s ease-in-out,transform .2s ease-in-out}.app-sidebar-backdrop{background:var(--pb-bg-overlay)}@supports (color:color-mix(in lab, red, red)){.app-sidebar-backdrop{background:color-mix(in srgb, var(--pb-bg-overlay) 96%, transparent)}}.app-sidebar-shell{border-right:1px solid var(--pb-border);background:var(--pb-bg-surface);box-shadow:none;overflow:visible}.app-sidebar-brand{background:0 0;padding:1rem 1rem .75rem}.app-sidebar-brand-collapsed{background:0 0;justify-content:center;padding:1rem 0 .75rem;display:flex}.app-sidebar-brand-copy-collapsed{display:none!important}.app-sidebar-brand-card{align-items:center;gap:.75rem;padding:0;transition:color .18s,transform .18s;display:flex}.app-sidebar-brand-card-collapsed{justify-content:center;width:auto;min-width:0;min-height:0;padding:0}.app-sidebar-brand:hover .app-sidebar-brand-card{transform:translateY(-1px)}.app-sidebar-brand-collapsed:hover .app-sidebar-brand-card{transform:none}.app-sidebar-brand-mark{flex-shrink:0;justify-content:center;align-items:center;width:2.25rem;height:2.25rem;display:flex}.app-sidebar-brand-copy{white-space:nowrap;flex-direction:column;min-width:0;display:flex;overflow:hidden}.app-sidebar-brand-title{letter-spacing:.04em;text-transform:uppercase;color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:1.1rem;font-weight:800;line-height:1}.app-sidebar-brand-version{color:var(--pb-text-tertiary);margin-top:2px;font-family:JetBrains Mono,monospace;font-size:.58rem}.app-sidebar-nav{padding:.25rem .625rem 1rem}.app-sidebar-nav-collapsed{padding:.25rem 0 1rem;overflow:visible!important}.app-sidebar-link{color:var(--pb-text-secondary);border:0;border-radius:.5rem;align-items:center;gap:.625rem;margin:1px 0;padding:.4375rem .625rem;font-size:.82rem;font-weight:500;transition:background-color .18s,color .18s,transform .18s;display:flex;position:relative}.app-sidebar-link-collapsed{justify-content:center;gap:0;width:2.75rem;margin-inline:auto;padding:.5rem}.app-sidebar-link-copy-collapsed{display:none!important}.app-sidebar-link:hover{background:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.app-sidebar-link:hover{background:color-mix(in srgb, var(--pb-text-secondary) 4%, transparent)}}.app-sidebar-link:hover{color:var(--pb-text-primary)}.app-sidebar-link-collapsed:hover{transform:none}.app-sidebar-link-active{background:var(--pb-selected);color:var(--pb-interactive);font-weight:600}.app-sidebar-link-active:before{content:"";background:var(--pb-interactive);border-radius:0 2px 2px 0;width:3px;position:absolute;top:6px;bottom:6px;left:0}.app-sidebar-link-icon{width:1.25rem;height:1.25rem;color:inherit;flex-shrink:0;justify-content:center;align-items:center;transition:color .18s;display:flex}.app-sidebar-link-copy{flex:1;justify-content:space-between;align-items:center;gap:.75rem;min-width:0;display:flex}.app-sidebar-link-title{text-overflow:ellipsis;white-space:nowrap;min-width:0;font-size:.82rem;font-weight:inherit;overflow:hidden}.app-sidebar-badge-slot{flex-shrink:0;justify-content:flex-end;align-items:center;min-width:1.125rem;display:flex}.app-sidebar-badge-slot .count-badge{border-radius:6px;min-width:18px;min-height:auto;padding:1px 6px;font-family:JetBrains Mono,monospace;font-size:.6rem;font-weight:700;line-height:1.1}.app-sidebar-link-collapsed .app-sidebar-badge-slot{min-width:0;position:absolute;top:2px;right:2px}.app-sidebar-link-collapsed .app-sidebar-badge-slot .count-badge{border-radius:4px;min-width:14px;padding:0 4px;font-size:.52rem}.app-sidebar-section{margin:1rem 0 0}.app-sidebar-section-collapsed{width:3rem;margin:.75rem auto .375rem}.app-sidebar-section-label{letter-spacing:.14em;text-transform:uppercase;color:var(--pb-text-tertiary);align-items:center;gap:.375rem;margin-bottom:.25rem;padding:0 .5rem;font-family:Syne,sans-serif;font-size:.54rem;font-weight:700;display:flex}.app-sidebar-section-label-collapsed{display:none!important}.app-sidebar-section-label:after{content:"";background:var(--pb-border-subtle);flex:auto;height:1px}.app-sidebar-section-rule{border-color:var(--pb-border-subtle);width:1.5rem;margin:.5rem auto 0}.app-sidebar-toggle{z-index:70;border:1px solid var(--pb-border-hover);background:var(--pb-bg-card);width:1.75rem;height:1.75rem;color:var(--pb-text-secondary);pointer-events:auto;box-shadow:var(--pb-shadow-1);border-radius:999px;justify-content:center;align-items:center;transition:border-color .18s,background-color .18s,color .18s,box-shadow .18s;position:absolute;top:72px;right:-.75rem}.app-sidebar-toggle[data-tip]{position:absolute}.app-sidebar-toggle:hover{border-color:var(--pb-interactive);background:var(--pb-bg-card);color:var(--pb-interactive);box-shadow:var(--pb-shadow-2)}[data-tip]{--pb-tooltip-max-width:min(360px, calc(100vw - 32px), calc(100dvw - 32px));position:relative}.tooltips-overlay-enabled [data-tip]:hover:after,.tooltips-overlay-enabled [data-tip]:focus-visible:after{content:none;display:none}[data-tip]:hover:after,[data-tip]:focus-visible:after{content:attr(data-tip);width:max-content;max-width:var(--pb-tooltip-max-width);background:var(--pb-bg-card);border:1px solid var(--pb-border-hover);color:var(--pb-text-primary);white-space:normal;overflow-wrap:break-word;box-shadow:var(--pb-shadow-overlay);z-index:50;pointer-events:none;border-radius:8px;padding:6px 10px;font-size:11px;line-height:1.4;animation:.15s ease-out tip-in;position:absolute;bottom:calc(100% + 6px);left:0}.app-tooltip-host{z-index:140;pointer-events:none;position:fixed;inset:0}.app-tooltip-overlay{width:max-content;max-width:var(--pb-tooltip-max-width,min(360px, calc(100vw - 32px), calc(100dvw - 32px)));background:var(--pb-bg-card);border:1px solid var(--pb-border-hover);color:var(--pb-text-primary);white-space:normal;overflow-wrap:break-word;box-shadow:var(--pb-shadow-overlay);opacity:0;visibility:hidden;will-change:left, top, opacity;border-radius:8px;padding:6px 10px;font-size:11px;line-height:1.4;transition:opacity .12s ease-out,visibility .12s ease-out;position:absolute;top:0;left:0;transform:translate(0)}.app-tooltip-host[data-visible=true] .app-tooltip-overlay{opacity:1;visibility:visible}[data-search-field-clear][data-tip]{position:absolute}[data-tip-pos=left]:hover:after,[data-tip-pos=left]:focus-visible:after{animation:.15s ease-out tip-in-left;inset:50% calc(100% + 6px) auto auto;transform:translateY(-50%)}[data-tip-pos=right]:hover:after,[data-tip-pos=right]:focus-visible:after{animation:.15s ease-out tip-in-right;inset:50% auto auto calc(100% + 6px);transform:translateY(-50%)}[data-tip-pos=bottom]:hover:after,[data-tip-pos=bottom]:focus-visible:after{animation:.15s ease-out tip-in-bottom;top:calc(100% + 6px);bottom:auto;left:0}[data-tip-size=wide]:hover:after,[data-tip-size=wide]:focus-visible:after{max-width:var(--pb-tooltip-max-width,min(480px, calc(100vw - 32px), calc(100dvw - 32px)))}[data-tip-size=narrow]:hover:after,[data-tip-size=narrow]:focus-visible:after{max-width:var(--pb-tooltip-max-width,min(220px, calc(100vw - 32px), calc(100dvw - 32px)))}.app-header-icon-tip:hover:after,.app-header-icon-tip:focus-visible:after{text-align:center;min-width:7.25rem}.tooltip-wrap{position:relative}.tooltip-panel{z-index:50;visibility:hidden;opacity:0;pointer-events:none;background:var(--pb-bg-card);border:1px solid var(--pb-border-hover);width:max-content;max-width:360px;color:var(--pb-text-primary);white-space:normal;overflow-wrap:break-word;box-shadow:var(--pb-shadow-overlay);border-radius:.5rem;padding:.5rem .75rem;font-size:.75rem;line-height:1.4;transition:opacity .15s ease-out,transform .15s ease-out,visibility .15s ease-out;position:absolute}.tooltip-wrap:hover>.tooltip-panel,.tooltip-wrap:focus-within>.tooltip-panel{visibility:visible;opacity:1}.tooltip-panel-top{bottom:calc(100% + .5rem);left:0;transform:translateY(4px)}.tooltip-wrap:hover>.tooltip-panel-top,.tooltip-wrap:focus-within>.tooltip-panel-top{transform:translateY(0)}.tooltip-panel-left{top:50%;right:calc(100% + .5rem);transform:translate(4px,-50%)}.tooltip-wrap:hover>.tooltip-panel-left,.tooltip-wrap:focus-within>.tooltip-panel-left{transform:translateY(-50%)}.tooltip-panel-right{top:50%;left:calc(100% + .5rem);transform:translate(-4px,-50%)}.tooltip-wrap:hover>.tooltip-panel-right,.tooltip-wrap:focus-within>.tooltip-panel-right{transform:translateY(-50%)}.tooltip-panel-bottom{top:calc(100% + .5rem);left:0;transform:translateY(-4px)}.tooltip-wrap:hover>.tooltip-panel-bottom,.tooltip-wrap:focus-within>.tooltip-panel-bottom{transform:translateY(0)}.tooltip-panel-nowrap{white-space:nowrap}.tooltip-panel-wide{max-width:500px}@keyframes tip-in{0%{opacity:0;transform:translateY(4px)}to{opacity:1;transform:translateY(0)}}@keyframes tip-in-left{0%{opacity:0;transform:translate(4px,-50%)}to{opacity:1;transform:translateY(-50%)}}@keyframes tip-in-right{0%{opacity:0;transform:translate(-4px,-50%)}to{opacity:1;transform:translateY(-50%)}}@keyframes tip-in-bottom{0%{opacity:0;transform:translateY(-4px)}to{opacity:1;transform:translateY(0)}}.log-terminal{contain:inline-size;background:var(--pb-bg-surface);width:100%;min-width:0;max-width:100%;overflow-x:hidden}@supports (color:color-mix(in lab, red, red)){.log-terminal{background:color-mix(in srgb, var(--pb-bg-surface) 82%, var(--pb-bg-base))}}.log-terminal{background-image:repeating-linear-gradient(0deg, transparent, transparent 2px, var(--pb-border-subtle) 2px, var(--pb-border-subtle) 4px)}@supports (color:color-mix(in lab, red, red)){.log-terminal{background-image:repeating-linear-gradient(0deg, transparent, transparent 2px, color-mix(in srgb, var(--pb-border-subtle) 32%, transparent) 2px, color-mix(in srgb, var(--pb-border-subtle) 32%, transparent) 4px)}}.log-line{box-sizing:border-box;border-left:3px solid #0000;width:100%;min-width:0;max-width:100%;overflow:hidden}.log-line:hover{background:var(--pb-bg-card-hover)}@supports (color:color-mix(in lab, red, red)){.log-line:hover{background:color-mix(in srgb, var(--pb-bg-card-hover) 36%, transparent)}}.log-line[data-level=error],.log-line[data-level=critical]{border-left-color:var(--pb-error);background:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.log-line[data-level=error],.log-line[data-level=critical]{background:color-mix(in srgb, var(--pb-error) 12%, transparent)}}.log-line[data-level=warning]{border-left-color:var(--pb-warning)}.log-line[data-level=info]{border-left-color:var(--pb-info)}.log-line[data-level=debug]{border-left-color:var(--pb-text-dim)}.log-line.expanded{background:var(--pb-bg-card-hover)!important}@supports (color:color-mix(in lab, red, red)){.log-line.expanded{background:color-mix(in srgb, var(--pb-bg-card-hover) 54%, transparent)!important}}.log-detail{box-sizing:border-box;max-width:calc(100% - 140px);overflow:hidden}.log-path{overflow-wrap:anywhere}.log-badge{letter-spacing:.08em;text-transform:uppercase;text-align:center;border-radius:3px;min-width:38px;padding:1px 5px;font-size:9px;font-weight:700;display:inline-block}.badge-debug{color:var(--pb-text-dim);background:var(--pb-text-dim)}@supports (color:color-mix(in lab, red, red)){.badge-debug{background:color-mix(in srgb, var(--pb-text-dim) 16%, transparent)}}.badge-info{color:var(--pb-info);background:var(--pb-info-dim)}.badge-warning{color:var(--pb-warning);background:var(--pb-warning-dim)}.badge-error{color:var(--pb-error);background:var(--pb-error-dim)}.badge-critical{color:var(--pb-text-inverse);background:var(--pb-error)}.live-dot{background:var(--pb-success);border-radius:50%;width:6px;height:6px;animation:2s ease-in-out infinite live-pulse}@keyframes live-pulse{0%,to{opacity:1;box-shadow:0 0 0 0 color-mix(in srgb, var(--pb-success) 42%, transparent)}50%{opacity:.6;box-shadow:0 0 6px 2px color-mix(in srgb, var(--pb-success) 24%, transparent)}}.json-key{color:var(--pb-info)}.json-str{color:var(--pb-success)}.json-num{color:var(--pb-warning)}.json-bool{color:var(--pb-purple)}.json-null{color:var(--pb-text-dim)}.downloads-page-shell{width:100%}.downloads-view{width:100%;max-width:none;padding:0 0 var(--pb-page-footer-clearance);margin:0}.downloads-header{display:block}.downloads-header-main{flex-wrap:wrap;justify-content:space-between;align-items:flex-start;gap:1.25rem;width:100%;display:flex}.downloads-header-summary{flex-wrap:wrap;flex:auto;align-items:flex-start;gap:1.5rem;min-width:0;display:flex}.downloads-header-copy{min-width:0}.downloads-header-actions{flex:none;justify-content:flex-end;align-items:flex-start;margin-left:auto;display:flex}.downloads-title{letter-spacing:.04em;text-transform:uppercase;color:var(--pb-text-primary);font-family:Syne,sans-serif;font-size:1.6rem;font-weight:800;line-height:1}.downloads-title span{color:var(--pb-brand)}.downloads-subtitle{color:var(--pb-text-tertiary);letter-spacing:.01em;margin-top:.25rem;font-size:.78rem}.downloads-gauges{align-items:flex-end;gap:1rem;display:flex}.downloads-gauge{text-align:center}.downloads-gauge-ring{width:56px;height:56px;margin:0 auto;position:relative}.downloads-gauge-ring svg{transform:rotate(-90deg)}.downloads-gauge-bg{fill:none;stroke:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.downloads-gauge-bg{stroke:color-mix(in srgb, var(--pb-text-secondary) 10%, transparent)}}.downloads-gauge-bg{stroke-width:4.5px}.downloads-gauge-fill{fill:none;stroke-width:4.5px;stroke-linecap:round;transition:stroke-dashoffset .6s}.downloads-gauge-fill-success{stroke:var(--pb-status-success)}.downloads-gauge-fill-info{stroke:var(--pb-status-info)}.downloads-gauge-fill-warning{stroke:var(--pb-status-warning)}.downloads-gauge-fill-error{stroke:var(--pb-status-danger)}.downloads-gauge-fill-muted{stroke:var(--pb-text-dim)}.downloads-gauge-value{color:var(--pb-text-primary);justify-content:center;align-items:center;font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:700;display:flex;position:absolute;inset:0}.downloads-gauge-value-success{color:var(--pb-status-success)}.downloads-gauge-value-info{color:var(--pb-status-info)}.downloads-gauge-value-warning{color:var(--pb-status-warning)}.downloads-gauge-value-error{color:var(--pb-status-danger)}.downloads-gauge-value-muted{color:var(--pb-text-dim)}.downloads-gauge-label{letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-tertiary);margin-top:.25rem;font-size:.6rem;font-weight:600}.downloads-tab-rail{border:1px solid var(--pb-border);background:0 0;border-radius:10px;align-self:flex-start;display:inline-flex;overflow:hidden}.downloads-tab-btn{letter-spacing:.06em;text-transform:uppercase;color:var(--pb-text-dim);background:0 0;border:0;padding:.5rem 1.25rem;font-family:Syne,sans-serif;font-size:.78rem;font-weight:700;transition:background-color .14s,color .14s}.downloads-tab-btn+.downloads-tab-btn{border-left:1px solid var(--pb-border)}.downloads-tab-btn:hover{color:var(--pb-text);background:var(--pb-text-secondary)}@supports (color:color-mix(in lab, red, red)){.downloads-tab-btn:hover{background:color-mix(in srgb, var(--pb-text-secondary) 4%, transparent)}}.downloads-tab-btn.is-active{background:var(--pb-interactive);color:var(--pb-text-inverse)}.downloads-panel-stack{gap:1.25rem;margin-top:.875rem;display:grid}.import-history-page-shell{margin-top:0}.downloads-section{gap:.625rem;display:grid}.downloads-section-label{border-bottom:1px solid var(--pb-border-subtle);letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);padding-bottom:.375rem;font-family:Syne,sans-serif;font-size:.62rem;font-weight:700}.downloads-table-wrap{border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);border-radius:14px;position:relative;overflow:visible;box-shadow:0 1px 3px #1e1a170f,0 1px 2px #1e1a170a}.downloads-table-wrap.is-clipped{overflow:hidden}.downloads-table{border-collapse:collapse;width:100%}.downloads-table th{border-bottom:2px solid var(--pb-border);background:var(--pb-bg-surface);text-align:left;letter-spacing:.1em;text-transform:uppercase;color:var(--pb-text-dim);padding:.5625rem .875rem;font-family:Syne,sans-serif;font-size:.6rem;font-weight:700}.downloads-table td{border-bottom:1px solid var(--pb-border-subtle);color:var(--pb-text);vertical-align:middle;padding:.6875rem .875rem;font-size:.82rem;transition:background-color .1s}.downloads-table tbody:last-child tr:last-child td{border-bottom:0}.downloads-table tr:not(.table-detail-row):hover td{background:var(--pb-surface-selected)}.downloads-table th.is-center,.downloads-table td.is-center{text-align:center}.downloads-table th.is-right,.downloads-table td.is-right{text-align:right}.issue-search-results-table-wrap{max-height:62vh;overflow-y:auto}.issue-search-results-table thead th{z-index:1;position:sticky;top:0}.issue-search-sort-indicator{color:var(--pb-interactive)}.issue-search-release-link{color:var(--pb-text);transition:color .14s}.issue-search-release-link:hover{color:var(--pb-interactive)}.issue-search-release-title{text-overflow:ellipsis;white-space:nowrap;max-width:320px;color:inherit;font-weight:600;display:block;overflow:hidden}.issue-search-release-title-muted{color:var(--pb-text-dim)}.issue-search-release-match{color:var(--pb-text-dim);margin-top:.125rem;font-size:.64rem}.issue-search-action-row{justify-content:flex-end;align-items:center;gap:.25rem;display:inline-flex}.issue-search-rejected-row td{opacity:.45}.issue-search-rejected-label{color:var(--pb-text-dim);font-family:JetBrains Mono,monospace;font-size:.64rem}.issue-search-empty-state{text-align:center;padding:2.5rem 1.25rem}.issue-search-empty-icon{width:2.25rem;height:2.25rem;color:var(--pb-text-dim);margin-inline:auto}.issue-search-empty-title{color:var(--pb-text);margin-top:.625rem;font-size:.85rem;font-weight:600}.issue-search-empty-copy{color:var(--pb-text-sec);margin-top:.25rem;font-size:.78rem}.downloads-release-name{max-width:260px;color:var(--pb-text);font-weight:600;display:block}.downloads-release-meta{color:var(--pb-text-dim);margin-top:.2rem;font-size:.72rem}.downloads-issue-link{color:var(--pb-interactive);transition:color .14s}.downloads-issue-link:hover{color:var(--pb-interactive-hover)}.table-mono-dim,td.table-mono-dim,.downloads-mono-dim-cell,td.downloads-mono-dim-cell{font-family:JetBrains Mono,monospace;font-size:.72rem}.table-mono-dim{color:var(--pb-text-dim)}.downloads-table td.downloads-mono-cell,.downloads-table td.downloads-mono-dim-cell,.downloads-table .downloads-mono-cell,.downloads-table .downloads-mono-dim-cell{font-variant-numeric:tabular-nums;font-family:JetBrains Mono,monospace}.downloads-table td.downloads-mono-cell,.downloads-table .downloads-mono-cell{color:var(--pb-text-secondary);font-size:.75rem;font-weight:400;line-height:1.2}.downloads-table td.downloads-mono-dim-cell,.downloads-table .downloads-mono-dim-cell{color:var(--pb-text-dim);font-size:.72rem}.downloads-muted-text{color:var(--pb-text-dim)}.downloads-progress-cell{align-items:center;gap:.375rem;display:inline-flex}.downloads-progress-track{border:1px solid var(--pb-border-subtle);background:var(--pb-text-secondary);border-radius:2px;width:100px;height:6px;display:inline-block;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.downloads-progress-track{background:color-mix(in srgb, var(--pb-text-secondary) 6%, transparent)}}.downloads-progress-track{vertical-align:middle}.downloads-progress-fill{height:100%;display:block;position:relative}.downloads-progress-fill.is-blue{background:linear-gradient(90deg, var(--pb-info), var(--pb-info))}@supports (color:color-mix(in lab, red, red)){.downloads-progress-fill.is-blue{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-info) 45%, transparent), var(--pb-info))}}.downloads-progress-fill.is-amber{background:linear-gradient(90deg, var(--pb-warning), var(--pb-warning))}@supports (color:color-mix(in lab, red, red)){.downloads-progress-fill.is-amber{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-warning) 45%, transparent), var(--pb-warning))}}.downloads-progress-fill.is-green{background:linear-gradient(90deg, var(--pb-success), var(--pb-success))}@supports (color:color-mix(in lab, red, red)){.downloads-progress-fill.is-green{background:linear-gradient(90deg, color-mix(in srgb, var(--pb-success) 45%, transparent), var(--pb-success))}}.downloads-progress-fill:after{content:"";background:repeating-linear-gradient(90deg,#0000 0 2px,#ffffff1f 2px 4px);position:absolute;inset:0}@keyframes downloads-progress-indeterminate{0%{transform:translate(-130%)}to{transform:translate(280%)}}.downloads-progress-fill.is-indeterminate,.app-progress-fill.is-indeterminate{will-change:transform;animation:1.25s ease-in-out infinite downloads-progress-indeterminate}@media (prefers-reduced-motion:reduce){.downloads-progress-fill.is-indeterminate,.app-progress-fill.is-indeterminate{opacity:.7;animation:none;transform:translate(60%)}}.downloads-progress-pct{color:var(--pb-info);font-family:JetBrains Mono,monospace;font-size:.68rem;font-weight:600}.downloads-progress-pct-warning{color:var(--pb-warning)}.downloads-progress-pct-success{color:var(--pb-success)}.downloads-progress-pct-muted{color:var(--pb-text-dim)}.downloads-led{border-radius:999px;width:8px;height:8px;display:inline-block}.downloads-led-green{background:var(--pb-success);box-shadow:0 0 5px var(--pb-success)}@supports (color:color-mix(in lab, red, red)){.downloads-led-green{box-shadow:0 0 5px color-mix(in srgb, var(--pb-success) 35%, transparent)}}.downloads-led-blue{background:var(--pb-info);box-shadow:0 0 5px var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.downloads-led-blue{box-shadow:0 0 5px color-mix(in srgb, var(--pb-info) 35%, transparent)}}.downloads-led-amber{background:var(--pb-warning);box-shadow:0 0 5px var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.downloads-led-amber{box-shadow:0 0 5px color-mix(in srgb, var(--pb-warning) 35%, transparent)}}.downloads-led-off{background:var(--pb-text-dim);opacity:.35}.downloads-action-group{align-items:center;gap:.25rem;display:inline-flex}@media (hover:hover) and (pointer:fine){.downloads-action-group.is-hover-reveal{opacity:0;pointer-events:none;transition:opacity .14s,transform .14s;transform:translateY(2px)}.downloads-table tbody tr:hover .downloads-action-group.is-hover-reveal,.downloads-table tbody tr:focus-within .downloads-action-group.is-hover-reveal{opacity:1;pointer-events:auto;transform:translateY(0)}}.downloads-action-btn{border:1px solid var(--pb-border-subtle);width:28px;height:28px;color:var(--pb-text-dim);background:0 0;border-radius:6px;justify-content:center;align-items:center;transition:border-color .14s,color .14s,background-color .14s;display:inline-flex}.downloads-action-btn svg{width:12px;height:12px}.import-history-action-group{gap:.375rem}.import-history-action-btn{border-radius:8px;flex-shrink:0;width:30px;height:30px}.downloads-action-btn .import-history-action-icon{shape-rendering:geometricprecision;flex-shrink:0;width:12px;height:12px;overflow:visible}.downloads-action-btn .import-history-action-icon--detail{width:14px;height:14px}.downloads-action-btn:hover{border-color:var(--pb-interactive);background:var(--pb-info)}@supports (color:color-mix(in lab, red, red)){.downloads-action-btn:hover{background:color-mix(in srgb, var(--pb-info) 10%, transparent)}}.downloads-action-btn:hover{color:var(--pb-interactive)}.downloads-action-btn.is-danger:hover{border-color:var(--pb-error)}@supports (color:color-mix(in lab, red, red)){.downloads-action-btn.is-danger:hover{border-color:color-mix(in srgb, var(--pb-error) 40%, transparent)}}.downloads-action-btn.is-danger:hover{background:var(--pb-error-dim);color:var(--pb-error)}.downloads-action-btn.is-warn:hover{border-color:var(--pb-warning)}@supports (color:color-mix(in lab, red, red)){.downloads-action-btn.is-warn:hover{border-color:color-mix(in srgb, var(--pb-warning) 40%, transparent)}}.downloads-action-btn.is-warn:hover{background:var(--pb-warning-dim);color:var(--pb-warning)}.downloads-empty-state{text-align:center;padding:2.5rem 1.5rem}.downloads-empty-state.is-compact{padding:1.5rem 1.25rem}.downloads-empty-state.is-history{padding:2rem 1.5rem}.downloads-empty-icon{width:40px;height:40px;color:var(--pb-text-dim);margin:0 auto}.downloads-empty-title{letter-spacing:.04em;text-transform:uppercase;color:var(--pb-text);margin-top:.75rem;font-family:Syne,sans-serif;font-size:.88rem;font-weight:700}.downloads-empty-copy{color:var(--pb-text-dim);margin-top:.4rem;font-size:.78rem}.intervention-bulk-bar{border-top:1px solid var(--pb-border-subtle);border-bottom:1px solid var(--pb-border-subtle);background:var(--pb-info-bg);flex-wrap:wrap;justify-content:space-between;align-items:center;gap:.75rem;padding:.625rem .875rem;display:flex}@supports (color:color-mix(in lab, red, red)){.intervention-bulk-bar{background:color-mix(in srgb, var(--pb-info-bg) 40%, var(--pb-bg-surface))}}.intervention-bulk-left{flex-wrap:wrap;align-items:center;gap:.625rem;display:inline-flex}.intervention-bulk-count{color:var(--pb-text-dim);font-family:JetBrains Mono,monospace;font-size:.72rem}.intervention-bulk-actions{flex-wrap:wrap;align-items:center;gap:.5rem;display:inline-flex}.downloads-footer-strip{border:1px solid var(--pb-border-subtle);background:var(--pb-bg-card);color:var(--pb-text-dim);border-radius:10px;flex-wrap:wrap;align-items:center;gap:.375rem 1.5rem;margin-top:1.25rem;padding:.625rem 1rem;font-family:JetBrains Mono,monospace;font-size:.7rem;display:flex}.downloads-footer-strip strong{color:var(--pb-text);font-weight:600}.downloads-history-toolbar{z-index:2;border:0;border-bottom:1px solid var(--pb-border);border-top-left-radius:inherit;border-top-right-radius:inherit;background:var(--pb-bg-surface);box-shadow:none;border-bottom-right-radius:0;border-bottom-left-radius:0;position:relative}.downloads-toolbar{border-bottom:1px solid var(--pb-border);background:var(--pb-bg-surface);flex-wrap:wrap;justify-content:space-between;align-items:center;gap:.75rem;padding:.625rem .875rem;display:flex}.downloads-history-toolbar>.downloads-toolbar{border-top-left-radius:inherit;border-top-right-radius:inherit;border-bottom-right-radius:0;border-bottom-left-radius:0}.downloads-toolbar-left{flex-wrap:wrap;align-items:center;gap:.5rem;display:flex}.downloads-toolbar-search,.downloads-toolbar-select{border:1px solid var(--pb-border);background:var(--pb-bg-card);height:30px;color:var(--pb-text);border-radius:8px;outline:none;font-size:.78rem}.downloads-toolbar-search{width:200px;padding:0 .625rem}.downloads-toolbar-select{appearance:none;background-image:url("data:image/svg+xml,%3Csvg width='10' height='6' viewBox='0 0 10 6' fill='none' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M1 1L5 5L9 1' stroke='%2374675B' stroke-width='1.5' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E");background-position:right 8px center;background-repeat:no-repeat;padding:0 2rem 0 .625rem}[data-theme=dark] .downloads-toolbar-select{background-image:url("data:image/svg+xml,%3Csvg width='10' height='6' viewBox='0 0 10 6' fill='none' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M1 1L5 5L9 1' stroke='%239AAABA' stroke-width='1.5' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E")}.downloads-toolbar-search:focus,.downloads-toolbar-select:focus{border-color:var(--pb-interactive)}.downloads-clear-btn{background:var(--pb-error);color:var(--pb-text-inverse);white-space:nowrap;border:0;border-radius:8px;align-items:center;gap:.3125rem;padding:.375rem .75rem;font-size:.75rem;font-weight:600;transition:opacity .14s;display:inline-flex}.downloads-clear-btn svg{width:12px;height:12px}.downloads-clear-btn:hover{opacity:.92}.downloads-sort-btn{width:100%;color:inherit;font-family:inherit;font-size:inherit;font-weight:inherit;letter-spacing:inherit;line-height:inherit;text-align:inherit;text-transform:inherit;justify-content:flex-start;align-items:center;gap:.25rem;transition:color .14s;display:inline-flex}.import-review-cv-year-header{width:5.75rem;min-width:5.75rem}.import-review-cv-year-header .downloads-sort-btn{white-space:nowrap}.import-review-cv-year-header .downloads-sort-chevron{flex-shrink:0}.downloads-table th.is-center .downloads-sort-btn{text-align:center;justify-content:center}.downloads-table th.is-right .downloads-sort-btn{text-align:right;justify-content:flex-end}.downloads-sort-btn:hover,.downloads-sort-btn.is-active{color:var(--pb-text)}.downloads-sort-chevron{opacity:.35;width:12px;height:12px}.downloads-sort-btn.is-active .downloads-sort-chevron{opacity:1}.downloads-sort-chevron.is-desc{transform:rotate(180deg)}.downloads-error-row td{border-bottom:1px solid var(--pb-border-subtle);background:var(--pb-error-dim);padding:.5rem .875rem}@supports (color:color-mix(in lab, red, red)){.downloads-error-row td{background:color-mix(in srgb, var(--pb-error-dim) 50%, var(--pb-bg-card))}}.downloads-error-row td{color:var(--pb-error);font-size:.75rem}.downloads-error-content{align-items:flex-start;gap:.5rem;display:flex}.downloads-error-content svg{flex-shrink:0;width:14px;height:14px;margin-top:1px}.search-history-detail-row td{border-bottom:1px solid var(--pb-border-subtle);background:var(--pb-bg-surface);padding:.75rem .875rem}@supports (color:color-mix(in lab, red, red)){.search-history-detail-row td{background:color-mix(in srgb, var(--pb-bg-surface) 70%, var(--pb-bg-card))}}.search-history-detail-card{border:1px solid var(--pb-border)}@supports (color:color-mix(in lab, red, red)){.search-history-detail-card{border:1px solid color-mix(in srgb, var(--pb-border) 78%, transparent)}}.search-history-detail-card{background:var(--pb-bg-card-hover);border-radius:.75rem}@supports (color:color-mix(in lab, red, red)){.search-history-detail-card{background:color-mix(in srgb, var(--pb-bg-card-hover) 55%, transparent)}}.search-history-detail-card{padding:.625rem .75rem}.search-history-detail-label{letter-spacing:.12em;text-transform:uppercase;color:var(--pb-text-dim);font-size:10px;font-weight:600}.search-history-diagnostics-shell{border:1px solid var(--pb-warning);overflow:hidden}@supports (color:color-mix(in lab, red, red)){.search-history-diagnostics-shell{border:1px solid color-mix(in srgb, var(--pb-warning) 30%, transparent)}}.search-history-diagnostics-shell{background:var(--pb-warning-dim);border-radius:.75rem}@supports (color:color-mix(in lab, red, red)){.search-history-diagnostics-shell{background:color-mix(in srgb, var(--pb-warning-dim) 35%, transparent)}}.downloads-pagination{border-top:1px solid var(--pb-border-subtle);padding:.5rem .875rem}@media (max-width:1024px){.downloads-header{gap:1rem}.downloads-table-wrap{overflow-x:auto}.downloads-table-wrap.is-clipped{overflow:auto hidden}.downloads-table{min-width:760px}}@media (max-width:640px){.downloads-header-main{gap:.875rem}.downloads-gauges{gap:.625rem}.downloads-toolbar-search{width:100%}}.comic-reader{--pb-reader-canvas:#070b12;--pb-reader-surface:#0d141ef0;--pb-reader-surface-strong:#0d141efa;--pb-reader-text:#f7f1e8;--pb-reader-text-muted:#cbd5e1;--pb-reader-text-dim:#9aaaba;--pb-reader-interactive:#8fb9ee;--pb-reader-danger:#e38473;--pb-reader-warning:#f2c98d;background:var(--pb-reader-canvas);width:100vw;max-width:none;height:100dvh;max-height:none;color:var(--pb-reader-text);border:0;margin:0;padding:0;position:fixed;inset:0;overflow:hidden}.comic-reader::backdrop{background:var(--pb-reader-canvas)}.comic-reader__shell{background:var(--pb-reader-canvas);width:100%;min-width:0;height:100%;min-height:0;color:var(--pb-reader-text);grid-template-rows:auto minmax(0,1fr) auto;font-family:DM Sans,sans-serif;display:grid;overflow:hidden}.comic-reader__topbar,.comic-reader__controls{z-index:20;background:var(--pb-reader-surface);-webkit-backdrop-filter:blur(12px);backdrop-filter:blur(12px);border-color:#cbd5e129;transition:opacity .14s;position:relative}.comic-reader__topbar{min-height:58px;padding:calc(.45rem + env(safe-area-inset-top,0px)) max(.75rem, env(safe-area-inset-right,0px)) .45rem max(.75rem, env(safe-area-inset-left,0px));border-bottom-width:1px;grid-template-columns:auto minmax(0,1fr) auto auto;align-items:center;gap:.75rem;display:grid}.comic-reader__identity{min-width:0}.comic-reader__identity h2,.comic-reader__identity p{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.comic-reader__identity h2{color:var(--pb-reader-text);margin:0;font-family:Bricolage Grotesque,sans-serif;font-size:.95rem;font-weight:800;line-height:1.2}.comic-reader__identity p{color:var(--pb-reader-text-dim);margin:.1rem 0 0;font-size:.72rem;line-height:1.2}.comic-reader__top-actions,.comic-reader__issue-navigation,.comic-reader__navigation,.comic-reader__sizing,.comic-reader__state-actions,.comic-reader__help-heading{align-items:center;display:flex}.comic-reader__top-actions,.comic-reader__issue-navigation,.comic-reader__navigation,.comic-reader__sizing,.comic-reader__state-actions{gap:.4rem}.comic-reader__issue-navigation{min-width:0}.comic-reader__issue-navigation .comic-reader__button{gap:.35rem}.comic-reader__button,.comic-reader__zoom-label,.comic-reader__fit-select,.comic-reader__page-jump input{min-height:44px;color:var(--pb-reader-text);font:inherit;background:#f7f1e812;border:1px solid #cbd5e138;border-radius:.55rem}.comic-reader__button,.comic-reader__zoom-label{white-space:nowrap;cursor:pointer;justify-content:center;align-items:center;min-width:44px;padding:.55rem .7rem;font-size:.76rem;font-weight:700;line-height:1;display:inline-flex}.comic-reader__button:hover,.comic-reader__zoom-label:hover,.comic-reader__button[aria-pressed=true]{background:#8fb9ee29;border-color:#8fb9ee8c}.comic-reader__button:focus-visible,.comic-reader__zoom-label:focus-visible,.comic-reader__fit-select:focus-visible,.comic-reader__page-jump input:focus-visible,.comic-reader__viewport:focus-visible{outline:2px solid var(--pb-reader-interactive);outline-offset:-2px}.comic-reader__button:disabled{cursor:default;opacity:.38}.comic-reader__button svg{fill:none;stroke:currentColor;stroke-linecap:round;stroke-linejoin:round;stroke-width:1.8px;width:19px;height:19px}.comic-reader__button--prominent{background:#8fb9ee1f;border-color:#8fb9ee61}.comic-reader__viewport{z-index:1;overscroll-behavior:contain;background:var(--pb-reader-canvas);scrollbar-color:#9aaaba8c transparent;touch-action:pan-y pinch-zoom;min-width:0;min-height:0;position:relative;overflow:auto}.comic-reader__viewport.is-pannable{touch-action:auto}.comic-reader__page-stage{justify-content:center;align-items:center;width:100%;min-width:100%;height:100%;min-height:100%;padding:.75rem;display:flex}.comic-reader__page{z-index:1;object-fit:contain;-webkit-user-select:none;user-select:none;-webkit-user-drag:none;flex:none;max-width:none;max-height:none;display:block;position:relative;box-shadow:0 8px 30px #00000061}.comic-reader__page--page{width:auto;max-width:100%;height:auto;max-height:100%}.comic-reader__page--width{align-self:flex-start;width:100%;height:auto}.comic-reader__page--height{width:auto;height:100%}.comic-reader__page--actual{align-self:flex-start;width:auto;height:auto}.comic-reader__state,.comic-reader__page-busy{z-index:8;text-align:center;justify-content:center;align-items:center;display:flex;position:absolute;top:50%;left:50%;transform:translate(-50%,-50%)}.comic-reader__state{width:min(90vw,32rem);color:var(--pb-reader-text-muted);background:#0d141ef5;border:1px solid #cbd5e133;border-radius:.9rem;flex-direction:column;gap:.5rem;padding:1.5rem;box-shadow:0 20px 56px #00000075}.comic-reader__state strong{color:var(--pb-reader-text);font-size:1rem}.comic-reader__state--error{border-color:var(--pb-reader-danger)}@supports (color:color-mix(in lab, red, red)){.comic-reader__state--error{border-color:color-mix(in srgb, var(--pb-reader-danger) 45%, transparent)}}.comic-reader__state-actions{flex-wrap:wrap;justify-content:center;margin-top:.5rem}.comic-reader__completion{z-index:9;right:max(.75rem, env(safe-area-inset-right,0px));bottom:max(.75rem, env(safe-area-inset-bottom,0px));width:min(24rem,100% - 1.5rem);color:var(--pb-reader-text-muted);background:#0d141ef5;border:1px solid #8fb9ee6b;border-radius:.8rem;padding:.85rem;position:absolute;box-shadow:0 16px 42px #0000006b}.comic-reader__completion strong{color:var(--pb-reader-text);font-family:Bricolage Grotesque,sans-serif;font-size:1rem;display:block}.comic-reader__completion p{margin:.35rem 0 0;font-size:.78rem}.comic-reader__page-busy{color:var(--pb-reader-text-muted);background:#0d141ee6;border-radius:999px;padding:.45rem .7rem;font-size:.72rem;top:1rem;transform:translate(-50%)}.comic-reader__spinner{border:2px solid #8fb9ee40;border-top-color:var(--pb-reader-interactive);border-radius:999px;width:1.5rem;height:1.5rem;animation:.7s linear infinite comic-reader-spin}@keyframes comic-reader-spin{to{transform:rotate(360deg)}}.comic-reader__tap-zone{z-index:4;background:0 0;border:0;margin:0;padding:0;position:absolute;top:0;bottom:0}.comic-reader__tap-zone--left{width:30%;left:0}.comic-reader__tap-zone--center{width:40%;left:30%}.comic-reader__tap-zone--right{width:30%;right:0}.comic-reader__controls{min-width:0;padding:.45rem max(.75rem, env(safe-area-inset-right,0px)) calc(.45rem + env(safe-area-inset-bottom,0px)) max(.75rem, env(safe-area-inset-left,0px));border-top-width:1px;justify-content:space-between;align-items:center;gap:.75rem;display:flex}.comic-reader__navigation,.comic-reader__sizing{min-width:0;position:relative}.comic-reader__page-jump{color:var(--pb-reader-text-dim);white-space:nowrap;align-items:center;gap:.35rem;font-size:.75rem;display:inline-flex}.comic-reader__page-jump input{text-align:center;appearance:textfield;width:3.4rem;padding:.4rem}.comic-reader__page-jump input::-webkit-outer-spin-button{appearance:none;margin:0}.comic-reader__page-jump input::-webkit-inner-spin-button{appearance:none;margin:0}.comic-reader__fit-select{color-scheme:dark;max-width:8.5rem;padding:0 1.9rem 0 .65rem;display:none}.comic-reader__zoom-label{min-width:3.75rem;color:var(--pb-reader-text-muted);padding-inline:.45rem}.comic-reader__input-error{border:1px solid var(--pb-reader-danger);width:max-content;max-width:min(20rem,100vw - 1.5rem);padding:.35rem .5rem;position:absolute;bottom:calc(100% + .35rem);left:50%;transform:translate(-50%)}@supports (color:color-mix(in lab, red, red)){.comic-reader__input-error{border:1px solid color-mix(in srgb, var(--pb-reader-danger) 42%, transparent)}}.comic-reader__input-error{background:var(--pb-reader-surface-strong);color:var(--pb-reader-danger);border-radius:.45rem;font-size:.7rem}.comic-reader__help{z-index:30;background:var(--pb-reader-surface-strong);border:1px solid #8fb9ee66;border-radius:.9rem;width:min(92vw,34rem);max-height:min(78dvh,38rem);padding:1rem;position:absolute;top:50%;left:50%;overflow:auto;transform:translate(-50%,-50%);box-shadow:0 20px 56px #0009}.comic-reader__help-heading{justify-content:space-between;gap:1rem}.comic-reader__help h3{color:var(--pb-reader-text);margin:0;font-family:Bricolage Grotesque,sans-serif;font-size:1rem}.comic-reader__help dl{gap:.15rem;margin:.75rem 0 0;display:grid}.comic-reader__help dl>div{border-top:1px solid #cbd5e11f;grid-template-columns:minmax(7.5rem,.7fr) minmax(0,1.3fr);gap:.75rem;padding:.45rem 0;display:grid}.comic-reader__help dt{color:var(--pb-reader-interactive);font-family:JetBrains Mono,monospace;font-size:.72rem;font-weight:600}.comic-reader__help dd{color:var(--pb-reader-text-muted);margin:0;font-size:.75rem}.comic-reader__save-status{z-index:40;right:max(.75rem, env(safe-area-inset-right,0px));bottom:calc(4.1rem + env(safe-area-inset-bottom,0px));max-width:min(22rem,100vw - 1.5rem);color:var(--pb-reader-warning);background:#231b11f5;border:1px solid #d7a15b66;border-radius:.55rem;margin:0;padding:.45rem .65rem;font-size:.72rem;position:absolute}.comic-reader__shell.is-controls-hidden .comic-reader__topbar,.comic-reader__shell.is-controls-hidden .comic-reader__controls{visibility:hidden;opacity:0;pointer-events:none}@media (max-width:900px){.comic-reader__issue-navigation .comic-reader__button span{clip:rect(0, 0, 0, 0);white-space:nowrap;width:1px;height:1px;position:absolute;overflow:hidden}.comic-reader__fit-page,.comic-reader__fit-wide{display:none}.comic-reader__fit-select{display:block}}@media (max-width:600px){.comic-reader__topbar{grid-template-columns:auto minmax(0,1fr) auto;gap:.5rem;min-height:54px}.comic-reader__issue-navigation{grid-area:2/1/auto/-1;justify-content:center}.comic-reader__top-actions{grid-area:1/3}.comic-reader__desktop-action{display:none}.comic-reader__controls{flex-wrap:wrap;justify-content:center;gap:.35rem .6rem}.comic-reader__navigation,.comic-reader__sizing{justify-content:center}.comic-reader__page-stage{padding:.4rem}.comic-reader__help dl>div{grid-template-columns:1fr;gap:.15rem}}@media (max-height:480px){.comic-reader__topbar,.comic-reader__controls{padding-top:.25rem;padding-bottom:.25rem}.comic-reader__topbar{min-height:48px}}@media (prefers-reduced-motion:reduce){.comic-reader__topbar,.comic-reader__controls{transition:none;visibility:visible!important;opacity:1!important;pointer-events:auto!important}.comic-reader__spinner{animation-duration:1.4s}}.reading-workspace{flex-direction:column;gap:1rem;display:flex}.reading-workspace-header,.dashboard-reading-shelf-header{justify-content:space-between;align-items:flex-end;gap:1rem;display:flex}.reading-page-size{align-items:center;gap:.65rem;display:flex}.reading-tabs{border-bottom:1px solid var(--pb-border);gap:.4rem;display:flex;overflow-x:auto}.reading-tab{color:var(--pb-text-tertiary);letter-spacing:.08em;text-transform:uppercase;flex:none;padding:.65rem .85rem;font-family:Syne,sans-serif;font-size:.7rem;font-weight:700;position:relative}.reading-tab:after{content:"";background:0 0;border-radius:999px;height:2px;position:absolute;bottom:-1px;left:.65rem;right:.65rem}.reading-tab:hover,.reading-tab:focus-visible,.reading-tab.is-active{color:var(--pb-interactive)}.reading-tab.is-active:after{background:var(--pb-interactive)}.reading-card-grid,.dashboard-reading-card-grid{grid-template-columns:repeat(auto-fill,minmax(260px,1fr));gap:.85rem;display:grid}.dashboard-reading-card-grid{grid-template-columns:repeat(4,minmax(0,1fr))}.reading-card{border:1px solid var(--pb-border);background:linear-gradient(145deg, var(--pb-interactive), transparent 42%), var(--pb-bg-card);border-radius:12px;grid-template-columns:92px minmax(0,1fr);min-width:0;display:grid;overflow:hidden}@supports (color:color-mix(in lab, red, red)){.reading-card{background:linear-gradient(145deg, color-mix(in srgb, var(--pb-interactive) 5%, transparent), transparent 42%), var(--pb-bg-card)}}.reading-card{box-shadow:var(--pb-shadow-1)}.reading-card:not(.reading-card-dashboard){min-height:15.25rem}.reading-card-cover-link,.reading-card-cover-frame{min-height:138px;display:block}.reading-card-cover-frame{border-right:1px solid var(--pb-border-subtle);background:var(--pb-bg-shell);height:100%;position:relative;overflow:hidden}.reading-card-cover{object-fit:cover;width:100%;height:100%}.reading-card-cover-placeholder{width:100%;height:100%;color:var(--pb-text-dim);justify-content:center;align-items:center;display:flex}.reading-card-cover-placeholder svg{width:2.25rem;height:2.25rem}.reading-card-body{flex-direction:column;gap:.7rem;min-width:0;padding:.85rem;display:flex}.reading-card-series{color:var(--pb-text-primary);text-overflow:ellipsis;white-space:nowrap;font-family:Syne,sans-serif;font-size:.78rem;font-weight:700;display:block;overflow:hidden}.reading-card-series:hover,.reading-card-series:focus-visible{color:var(--pb-interactive)}.reading-card-issue{min-width:0;color:var(--pb-text-tertiary);gap:.4rem;margin-top:.2rem;font-size:.72rem;display:flex}.reading-card-issue>span:first-child,.reading-card-state{font-family:JetBrains Mono,monospace}.reading-card-issue-title{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.reading-card-state{color:var(--pb-text-secondary);font-size:.67rem;font-weight:600}.reading-card-progress{background:var(--pb-border-subtle);border-radius:999px;height:3px;margin-top:.4rem;overflow:hidden}.reading-card-progress span{border-radius:inherit;background:var(--pb-interactive);height:100%;display:block}.reading-card-state-region{min-height:1.75rem}.reading-card-actions{grid-template-rows:repeat(2,minmax(2.25rem,auto));grid-template-columns:repeat(2,minmax(0,1fr));align-items:stretch;gap:.4rem;margin-top:auto;display:grid}.reading-card-view-read .reading-card-actions{grid-template-columns:minmax(0,2fr) minmax(0,3fr)}.reading-card-view-read .reading-card-primary-action,.reading-card-view-read .reading-card-completion-action{white-space:nowrap;padding-inline:.5rem}.reading-card-primary-action,.reading-card-completion-action,.reading-card-queue-action{width:100%;min-width:0}.reading-card-queue-action{grid-column:1/-1}.reading-card-queue-action-placeholder{visibility:hidden;pointer-events:none;grid-column:1/-1;min-height:2.25rem}.reading-card-mutation-status{min-height:1rem;color:var(--pb-error);font-size:.68rem}.reading-empty-state{border:1px dashed var(--pb-border);text-align:center;background:var(--pb-bg-card);border-radius:12px;flex-direction:column;justify-content:center;align-items:center;min-height:260px;padding:2.5rem 1.5rem;display:flex}@supports (color:color-mix(in lab, red, red)){.reading-empty-state{background:color-mix(in srgb, var(--pb-bg-card) 78%, transparent)}}.reading-empty-icon{width:2.75rem;height:2.75rem;color:var(--pb-text-dim)}.reading-empty-state h2{color:var(--pb-text-primary);margin-top:.85rem;font-family:Syne,sans-serif;font-size:.95rem;font-weight:700}.reading-empty-state p{max-width:34rem;color:var(--pb-text-tertiary);margin-top:.35rem;font-size:.78rem}.dashboard-reading-shelf{border:1px solid var(--pb-border);background:var(--pb-bg-card);border-radius:12px;padding:.85rem}@supports (color:color-mix(in lab, red, red)){.dashboard-reading-shelf{background:color-mix(in srgb, var(--pb-bg-card) 88%, transparent)}}.dashboard-reading-shelf{box-shadow:var(--pb-shadow-1)}.dashboard-reading-shelf-header h2{color:var(--pb-text-primary);margin-top:.1rem;font-family:Syne,sans-serif;font-size:1rem;font-weight:700}.dashboard-reading-shelf-link{color:var(--pb-interactive);font-size:.72rem;font-weight:700}.dashboard-reading-shelf-link:hover,.dashboard-reading-shelf-link:focus-visible{text-decoration:underline}.reading-card-dashboard{grid-template-columns:76px minmax(0,1fr)}.reading-card-dashboard .reading-card-cover-link,.reading-card-dashboard .reading-card-cover-frame{min-height:124px}.reading-card-dashboard .reading-card-body{gap:.5rem;padding:.7rem}@media (max-width:1180px){.dashboard-reading-card-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (max-width:640px){.reading-workspace-header{flex-direction:column;align-items:flex-start}.reading-card-grid,.dashboard-reading-card-grid{grid-template-columns:1fr}.reading-card{grid-template-columns:82px minmax(0,1fr)}.reading-card-cover-link,.reading-card-cover-frame{min-height:128px}}@property --tw-translate-x{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-y{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-z{syntax:"*";inherits:false;initial-value:0}@property --tw-scale-x{syntax:"*";inherits:false;initial-value:1}@property --tw-scale-y{syntax:"*";inherits:false;initial-value:1}@property --tw-scale-z{syntax:"*";inherits:false;initial-value:1}@property --tw-space-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-divide-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-border-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-leading{syntax:"*";inherits:false}@property --tw-font-weight{syntax:"*";inherits:false}@property --tw-tracking{syntax:"*";inherits:false}@property --tw-ordinal{syntax:"*";inherits:false}@property --tw-slashed-zero{syntax:"*";inherits:false}@property --tw-numeric-figure{syntax:"*";inherits:false}@property --tw-numeric-spacing{syntax:"*";inherits:false}@property --tw-numeric-fraction{syntax:"*";inherits:false}@property --tw-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-shadow-color{syntax:"*";inherits:false}@property --tw-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-inset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-shadow-color{syntax:"*";inherits:false}@property --tw-inset-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-ring-color{syntax:"*";inherits:false}@property --tw-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-ring-color{syntax:"*";inherits:false}@property --tw-inset-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-ring-inset{syntax:"*";inherits:false}@property --tw-ring-offset-width{syntax:"";inherits:false;initial-value:0}@property --tw-ring-offset-color{syntax:"*";inherits:false;initial-value:#fff}@property --tw-ring-offset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-outline-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-blur{syntax:"*";inherits:false}@property --tw-brightness{syntax:"*";inherits:false}@property --tw-contrast{syntax:"*";inherits:false}@property --tw-grayscale{syntax:"*";inherits:false}@property --tw-hue-rotate{syntax:"*";inherits:false}@property --tw-invert{syntax:"*";inherits:false}@property --tw-opacity{syntax:"*";inherits:false}@property --tw-saturate{syntax:"*";inherits:false}@property --tw-sepia{syntax:"*";inherits:false}@property --tw-drop-shadow{syntax:"*";inherits:false}@property --tw-drop-shadow-color{syntax:"*";inherits:false}@property --tw-drop-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-drop-shadow-size{syntax:"*";inherits:false}@property --tw-backdrop-blur{syntax:"*";inherits:false}@property --tw-backdrop-brightness{syntax:"*";inherits:false}@property --tw-backdrop-contrast{syntax:"*";inherits:false}@property --tw-backdrop-grayscale{syntax:"*";inherits:false}@property --tw-backdrop-hue-rotate{syntax:"*";inherits:false}@property --tw-backdrop-invert{syntax:"*";inherits:false}@property --tw-backdrop-opacity{syntax:"*";inherits:false}@property --tw-backdrop-saturate{syntax:"*";inherits:false}@property --tw-backdrop-sepia{syntax:"*";inherits:false}@property --tw-duration{syntax:"*";inherits:false}@property --tw-ease{syntax:"*";inherits:false}@keyframes spin{to{transform:rotate(360deg)}}@keyframes pulse{50%{opacity:.5}} diff --git a/src/pullbox/ui/static/js/pullbox.js b/src/pullbox/ui/static/js/pullbox.js index 375c6536..d8f04d44 100644 --- a/src/pullbox/ui/static/js/pullbox.js +++ b/src/pullbox/ui/static/js/pullbox.js @@ -1038,6 +1038,63 @@ function importReviewStatusCount(shell, view) { return Number(counts && counts[view]) || 0; } +async function cancelImportReview(button) { + if (!button || button.disabled) { + return; + } + + var jobId = Number(button.getAttribute("data-import-review-cancel-job-id")); + if (!Number.isFinite(jobId) || jobId <= 0) { + showToast({ message: "Unable to identify this import.", level: "error" }); + return; + } + + var confirmed = await window.pbConfirm({ + title: "Cancel Import", + message: + "This removes the current import job and all of its matched data. Use this only when you are sure the run should not continue.", + confirmText: "Cancel Import", + destructive: true, + }); + if (!confirmed) { + return; + } + + var label = button.querySelector("[data-import-review-cancel-label]"); + button.disabled = true; + button.setAttribute("aria-busy", "true"); + if (label) { + label.textContent = "Cancelling..."; + } + + try { + var response = await fetch("/api/v1/import/" + jobId, { + method: "DELETE", + headers: { "X-CSRF-Token": readCsrfTokenFromBody() }, + }); + if (!response.ok) { + var error = await response.json().catch(function () { + return { detail: "Failed to cancel import." }; + }); + throw new Error(error.detail || "Failed to cancel import."); + } + purgeImportClientState(jobId); + window.location.replace("/import?tab=collection"); + } catch (err) { + button.disabled = false; + button.removeAttribute("aria-busy"); + if (label) { + label.textContent = "Cancel"; + } + showToast({ + message: (err && err.message) || "Failed to cancel import. Please try again.", + level: "error", + }); + } +} + +window.cancelImportReview = cancelImportReview; + function importReviewShellHasSeriesBucket(shell, seriesId, bucket) { var numericSeriesId = Number(seriesId); if (!shell || !Number.isFinite(numericSeriesId) || !bucket) { @@ -1055,6 +1112,126 @@ function importReviewShellHasSeriesBucket(shell, seriesId, bucket) { return buckets.indexOf(bucket) >= 0; } +function captureImportReviewViewport(shell, preferredElement) { + var scroller = document.getElementById("content"); + var state = { + scrollTop: scroller ? scroller.scrollTop : window.scrollY || 0, + anchors: [], + expandedRows: [], + }; + if (!shell || typeof shell.querySelectorAll !== "function") { + return state; + } + + var capturedRowKeys = Object.create(null); + var captureRow = function (row) { + if (!row || !shell.contains(row)) { + return; + } + var rowKey = row.getAttribute("data-import-review-row-key"); + if (!rowKey || capturedRowKeys[rowKey]) { + return; + } + capturedRowKeys[rowKey] = true; + state.anchors.push({ + key: rowKey, + top: row.getBoundingClientRect().top, + }); + if (row.querySelector("[data-import-review-expand-action][aria-expanded='true']")) { + state.expandedRows.push({ + key: rowKey, + pendingSubitems: + Number(row.getAttribute("data-import-review-pending-subitems")) || 0, + }); + } + }; + + var preferredRow = + preferredElement && typeof preferredElement.closest === "function" + ? preferredElement.closest("[data-import-review-series-row]") + : null; + captureRow(preferredRow); + + var viewportHeight = window.innerHeight || document.documentElement.clientHeight || 0; + var rows = shell.querySelectorAll("[data-import-review-series-row]"); + for (var i = 0; i < rows.length; i += 1) { + var rect = rows[i].getBoundingClientRect(); + if (rect.bottom < 0 || rect.top > viewportHeight) { + continue; + } + captureRow(rows[i]); + } + return state; +} + +function restoreImportReviewExpansionState(state, shell) { + if (!state) { + return; + } + var expandedRows = Array.isArray(state.expandedRows) ? state.expandedRows : []; + var renderedRows = shell ? shell.querySelectorAll("[data-import-review-series-row]") : []; + for (var expandedIndex = 0; expandedIndex < expandedRows.length; expandedIndex += 1) { + var expandedRow = expandedRows[expandedIndex]; + for (var renderedIndex = 0; renderedIndex < renderedRows.length; renderedIndex += 1) { + var renderedRow = renderedRows[renderedIndex]; + if (renderedRow.getAttribute("data-import-review-row-key") !== expandedRow.key) { + continue; + } + var previousPendingSubitems = Number(expandedRow.pendingSubitems) || 0; + var nextPendingSubitems = + Number(renderedRow.getAttribute("data-import-review-pending-subitems")) || 0; + if (previousPendingSubitems > 0 && nextPendingSubitems === 0) { + setImportReviewRowExpanded(renderedRow, false); + } else { + setImportReviewRowExpanded(renderedRow, true); + } + break; + } + } +} + +function restoreImportReviewViewport(state, shell) { + if (!state) { + return; + } + var renderedRows = shell ? shell.querySelectorAll("[data-import-review-series-row]") : []; + var scroller = document.getElementById("content"); + var anchors = Array.isArray(state.anchors) ? state.anchors : []; + for (var i = 0; i < anchors.length; i += 1) { + var anchor = anchors[i]; + for (var rowIndex = 0; rowIndex < renderedRows.length; rowIndex += 1) { + if (renderedRows[rowIndex].getAttribute("data-import-review-row-key") !== anchor.key) { + continue; + } + var delta = renderedRows[rowIndex].getBoundingClientRect().top - anchor.top; + if (scroller) { + scroller.scrollTop += delta; + } else { + window.scrollBy(0, delta); + } + return; + } + } + if (scroller) { + scroller.scrollTop = state.scrollTop; + } else { + window.scrollTo(0, state.scrollTop); + } +} + +var pendingImportReviewViewportState = null; + +document.body.addEventListener("htmx:beforeSwap", function (event) { + var target = event && event.detail ? event.detail.target : null; + if (target && target.id === "import-step-review-shell") { + var requestElement = + event.detail.requestConfig && event.detail.requestConfig.elt + ? event.detail.requestConfig.elt + : null; + pendingImportReviewViewportState = captureImportReviewViewport(target, requestElement); + } +}); + function loadImportReviewShell(url) { var shell = document.getElementById("import-step-review-shell"); if (!shell || !url) { @@ -1095,18 +1272,25 @@ function loadImportReviewShell(url) { throw new Error("Import review refresh returned an unexpected response."); } + if (typeof Idiomorph === "undefined" || typeof Idiomorph.morph !== "function") { + throw new Error("Import review refresh is unavailable."); + } + var viewportState = captureImportReviewViewport(currentShell); destroyAlpineTree(currentShell); - currentShell.replaceWith(nextShell); - + Idiomorph.morph(currentShell, nextShell, { morphStyle: "outerHTML" }); + var activeShell = document.getElementById("import-step-review-shell"); if (window.htmx && typeof window.htmx.process === "function") { - window.htmx.process(nextShell); + window.htmx.process(activeShell); } if (window.Alpine) { - Alpine.initTree(nextShell); + Alpine.initTree(activeShell); } + restoreImportReviewExpansionState(viewportState, activeShell); + _dispatchSyntheticHtmxAfterSettle(activeShell); _syncFooterDockFromResponse(html); - seedSearchFieldStates(nextShell); - return nextShell; + seedSearchFieldStates(activeShell); + restoreImportReviewViewport(viewportState, activeShell); + return activeShell; }) .finally(function () { var activeShell = document.getElementById("import-step-review-shell"); @@ -1773,7 +1957,40 @@ function fileBrowserMixin(config) { } function dispatchImportWizardAdvance(detail) { - window.dispatchEvent(new CustomEvent("wizard:advance", { detail: detail || {} })); + var payload = detail || {}; + var collectionPage = document.querySelector("[data-testid='import-collection-page']"); + if (collectionPage && collectionPage.isConnected) { + window.dispatchEvent(new CustomEvent("wizard:advance", { detail: payload })); + return; + } + + var jobId = payload.jobId; + var step = Number(payload.step); + if (jobId == null || !Number.isFinite(step) || step < 2) { + window.dispatchEvent(new CustomEvent("wizard:advance", { detail: payload })); + return; + } + + var path = + "/import?tab=collection&resume_job_id=" + + encodeURIComponent(jobId) + + "&resume_step=" + + encodeURIComponent(step); + var importContent = document.getElementById("import-content"); + if (!importContent || typeof htmx === "undefined") { + window.location.assign(path); + return; + } + + if (window.history && typeof window.history.pushState === "function") { + window.history.pushState({}, "", path); + } + performHtmxSwap("GET", path, { + target: "#import-content", + swap: "outerHTML", + }).catch(function () { + window.location.assign(path); + }); } function importReviewAdvanceStorageKey(jobId) { @@ -1871,109 +2088,33 @@ function clearImportReviewSelection(jobId) { void jobId; } -function importReviewExpansionStorageKey(jobId) { - return "pb-import-review-expanded:" + String(jobId || ""); -} - -function normalizeImportReviewExpandedRows(value) { - if (Array.isArray(value)) { - var rowsFromArray = {}; - for (var i = 0; i < value.length; i += 1) { - var rowId = String(value[i] || ""); - if (rowId) { - rowsFromArray[rowId] = true; - } - } - return rowsFromArray; - } - - if (!value || typeof value !== "object") { - return {}; - } - - var rows = {}; - var keys = Object.keys(value); - for (var j = 0; j < keys.length; j += 1) { - if (value[keys[j]]) { - rows[String(keys[j])] = true; - } - } - return rows; -} - -function readImportReviewExpandedRows(jobId) { - if (jobId == null) { - return {}; - } - - try { - var raw = window.sessionStorage.getItem(importReviewExpansionStorageKey(jobId)); - return raw ? normalizeImportReviewExpandedRows(JSON.parse(raw)) : {}; - } catch (_) { - return {}; - } -} - -function writeImportReviewExpandedRows(jobId, expandedRows) { - if (jobId == null) { +function setImportReviewRowExpanded(row, expanded) { + if (!row) { return; } - - try { - var rows = normalizeImportReviewExpandedRows(expandedRows); - var rowIds = Object.keys(rows); - if (rowIds.length === 0) { - window.sessionStorage.removeItem(importReviewExpansionStorageKey(jobId)); - return; - } - window.sessionStorage.setItem(importReviewExpansionStorageKey(jobId), JSON.stringify(rowIds)); - } catch (_) { - // Ignore storage availability failures. + var actions = row.querySelectorAll("[data-import-review-expand-action]"); + for (var actionIndex = 0; actionIndex < actions.length; actionIndex += 1) { + actions[actionIndex].setAttribute("aria-expanded", expanded ? "true" : "false"); } -} - -function isImportReviewRowExpanded(jobId, rowId) { - if (rowId == null) { - return false; + var icons = row.querySelectorAll("[data-import-review-expand-icon]"); + for (var iconIndex = 0; iconIndex < icons.length; iconIndex += 1) { + icons[iconIndex].classList.toggle("rotate-180", expanded); + } + var details = row.querySelectorAll("[data-import-review-detail-row]"); + for (var detailIndex = 0; detailIndex < details.length; detailIndex += 1) { + details[detailIndex].hidden = !expanded; } - var expandedRows = readImportReviewExpandedRows(jobId); - return expandedRows[String(rowId)] === true; } -function setImportReviewRowExpanded(jobId, rowId, expanded) { - if (rowId == null) { +function toggleImportReviewRow(button) { + var row = button ? button.closest("[data-import-review-series-row]") : null; + if (!row) { return; } - - var expandedRows = readImportReviewExpandedRows(jobId); - var key = String(rowId); - if (expanded) { - expandedRows[key] = true; - } else { - delete expandedRows[key]; - } - writeImportReviewExpandedRows(jobId, expandedRows); + setImportReviewRowExpanded(row, button.getAttribute("aria-expanded") !== "true"); } -function importReviewRowExpansionData(config) { - var cfg = config || {}; - return { - expanded: false, - jobId: cfg.jobId, - rowId: cfg.rowId, - - init: function () { - this.expanded = isImportReviewRowExpanded(this.jobId, this.rowId); - }, - - toggle: function () { - this.expanded = !this.expanded; - setImportReviewRowExpanded(this.jobId, this.rowId, this.expanded); - }, - }; -} - -window.importReviewRowExpansionData = importReviewRowExpansionData; +window.toggleImportReviewRow = toggleImportReviewRow; function readImportConflictCommitState(jobId) { function normalizeCommittedPages(pages) { @@ -2757,7 +2898,7 @@ function importCollectionFooterData(config) { Number(snapshot.series_found) || 0, ), recentJobs: Number(cfg.recentJobs) || 0, - unmatched: Number(cfg.unmatched) || 0, + followUp: Number(cfg.followUp) || 0, libraryRoots: Number(cfg.libraryRoots) || 0, footerPhaseLabel: function () { @@ -2817,7 +2958,7 @@ function importCollectionFooterData(config) { return [ { label: "active import", value: this.resumeJobId ? "ready" : "idle" }, { label: "recent jobs", value: String(this.recentJobs) }, - { label: "unmatched", value: String(this.unmatched) }, + { label: "follow-up", value: String(this.followUp) }, { label: "library roots", value: String(this.libraryRoots) }, ]; }, @@ -2917,6 +3058,55 @@ function importCollectionFooterData(config) { function importSourceData(config) { var cfg = config || {}; + var libraryRoots = Array.isArray(cfg.libraryRoots) ? cfg.libraryRoots : []; + var initialManagedRoots = libraryRoots.filter(function (root) { + return !!( + root && + root.enabled !== false && + root.allow_managed_writes !== false && + root.available !== false && + root.writable !== false + ); + }); + var defaultManagedRoot = initialManagedRoots.find(function (root) { + return !!root.is_default_managed_destination; + }); + var initialTargetRoot = + defaultManagedRoot || (initialManagedRoots.length === 1 ? initialManagedRoots[0] : null); + var initialTargetRootId = initialTargetRoot ? Number(initialTargetRoot.id) : null; + var emptyStoryArcPreview = function () { + return { + evidence_detected: false, + arcs_detected: 0, + entries_detected: 0, + resolution: { + resolved: 0, + pending: 0, + missing: 0, + ambiguous: 0, + conflicts: 0, + duplicates: 0, + }, + existing_arc_files_detected: false, + existing_arc_folders_detected: false, + pattern_summary: "", + settings: [], + examples: [], + provider_call_summary: "", + proposed_policy: { + mode: "logical", + destination_root_configured: false, + folder_template: "", + file_template: "", + reading_order_prefix: false, + synchronize: false, + }, + readlist_present: false, + readlist_count: 0, + partial: false, + warnings: [], + }; + }; return Object.assign(fileBrowserMixin(cfg), { sourceType: "", @@ -2927,59 +3117,1549 @@ function importSourceData(config) { minFilesPerSeries: 1, fileFormats: "cbz, cbr, cb7, cbt, pdf, epub", cvMatchThreshold: 70, + fileHandlingMode: "managed_copy", + layoutChoice: "auto", + layoutFallbackToAuto: true, + customSeriesPathTemplate: "{Publisher}/{Series} ({Year})", + customIssueFilenameTemplate: "{Series} {IssueTitle} Issue {Issue:03d}", + layoutPreview: null, + layoutPreviewLoading: false, + layoutPreviewError: "", + layoutPreviewTimer: null, + layoutPreviewController: null, + layoutPreviewRequestId: 0, + libraryRootsRefreshing: false, + mylarPathMappings: [], + mylarPathPreview: null, + mylarPathPreviewLoading: false, + mylarPathPreviewError: "", + mylarPathPreviewTimer: null, + mylarPathPreviewController: null, + mylarPathPreviewRequestId: 0, + mylarPathMappingId: 0, + mylarPathAutoDetect: true, + mylarPathConfirmed: false, + mylarUnresolvedConfirmed: false, + referenceRootRegistrationPath: "", + referenceRootRegistrationErrorPath: "", + referenceRootRegistrationError: "", + attentionResolvingKey: "", + attentionResolutionErrors: {}, + acknowledgedAttentionActions: {}, + attentionDetailsOpen: false, + attentionDetailsItem: null, + attentionDetailsReturnFocus: null, + storyArcPreview: emptyStoryArcPreview(), + storyArcPreviewLoading: false, + storyArcPreviewError: "", + storyArcPreviewTimer: null, + storyArcPreviewController: null, + storyArcPreviewRequestId: 0, + storyArcImportRequested: false, + storyArcMaterializationRequested: false, + libraryRoots: libraryRoots, + targetLibraryRootId: initialTargetRootId, + futureLayoutRequested: false, + futureRootPolicy: { + schema_version: 1, + series_path_template: "", + comic_file_template: "", + annual_file_template: "", + non_standard_file_template: "", + single_non_standard_file_template: "", + replace_illegal_characters: true, + colon_replacement: "dash", + }, + futurePolicyComparison: null, + futurePolicyLoading: false, + futurePolicyError: "", + futurePolicyRequestId: 0, + + selectSourceType: function (sourceType) { + var previousSourceType = this.sourceType; + if (previousSourceType !== sourceType) { + this.sourcePath = ""; + } + this.sourceType = sourceType; + if (sourceType !== "filesystem" && sourceType !== "mylar3") { + this.fileHandlingMode = "managed_copy"; + } + this.clearFuturePolicy(); + this.clearLayoutPreview(); + this.clearStoryArcPreview(); + this.clearMylarPathPreview(true); + if (sourceType === "filesystem") { + this.scheduleLayoutPreview(); + } else if (sourceType === "mylar3") { + this.scheduleMylarPathPreview(); + } + }, + + selectImportSource: function (selection) { + this.sourcePath = selection && selection.path ? String(selection.path) : ""; + this.clearMylarPathPreview(true); + this.scheduleLayoutPreview(); + this.scheduleMylarPathPreview(); + this.closeFileBrowser(); + }, - startScan: async function () { - if (!this.sourcePath.trim()) { - return; - } + importSourcePathChanged: function () { + this.clearMylarPathPreview(true); + this.scheduleLayoutPreview(); + this.scheduleMylarPathPreview(); + }, - this.scanning = true; + setFileHandlingMode: function (mode) { + var previousMode = this.fileHandlingMode; + this.fileHandlingMode = mode === "in_place" ? "in_place" : "managed_copy"; this.scanError = ""; + if (this.fileHandlingMode === "in_place" && previousMode !== "in_place") { + // A future managed destination is optional for in-place adoption and + // must be an explicit choice rather than an inherited default. + this.targetLibraryRootId = null; + } else if ( + this.fileHandlingMode === "managed_copy" && + !this.hasSelectedManagedDestination() + ) { + var defaultRoot = this.managedLibraryRoots().find(function (root) { + return !!root.is_default_managed_destination; + }); + var managedRoots = this.managedLibraryRoots(); + var automaticRoot = defaultRoot || (managedRoots.length === 1 ? managedRoots[0] : null); + this.targetLibraryRootId = automaticRoot ? Number(automaticRoot.id) : null; + } + if (this.fileHandlingMode === "in_place") { + this.scheduleLayoutPreview(); + } + if (this.sourceType === "mylar3") { + this.scheduleMylarPathPreview(); + } + }, + + managedLibraryRoots: function () { + return this.libraryRoots.filter(function (root) { + return !!( + root && + root.enabled !== false && + root.allow_managed_writes !== false && + root.available !== false && + root.writable !== false + ); + }); + }, + + managedLibraryRootOptions: function (emptyLabel) { + var options = this.managedLibraryRoots().map(function (root) { + return { + value: String(root.id), + label: + String(root.name || "Library") + + " — " + + String(root.path || "") + + (root.is_default_managed_destination ? " (default)" : ""), + }; + }); + if (emptyLabel) { + options.unshift({ value: "", label: String(emptyLabel) }); + } + return options; + }, + + selectedManagedLibraryRoot: function () { + var selectedRootId = Number(this.targetLibraryRootId); + return ( + this.managedLibraryRoots().find(function (root) { + return Number(root.id) === selectedRootId && selectedRootId > 0; + }) || null + ); + }, + + shouldShowFileDestinationControl: function () { + var managedRoots = this.managedLibraryRoots(); + if (this.fileHandlingMode === "managed_copy") { + return managedRoots.length > 1 || !this.hasSelectedManagedDestination(); + } + return managedRoots.length !== 1; + }, + + fileDestinationSummary: function () { + var managedRoots = this.managedLibraryRoots(); + var selectedRoot = this.selectedManagedLibraryRoot(); + if (this.fileHandlingMode === "managed_copy") { + if (selectedRoot) { + return ( + String(selectedRoot.name || "Library") + + " will receive the managed copies from this import." + ); + } + return managedRoots.length + ? "Choose the writable library that should receive this import." + : "Set up a writable library before importing copies."; + } + if (!managedRoots.length) { + return "Existing files can stay where they are, but future downloads need a writable library."; + } + return "Existing files stay where they are. You can override where Pullbox manages future files."; + }, + refreshImportLibraryRoots: async function () { + if (this.libraryRootsRefreshing) { + return; + } + this.libraryRootsRefreshing = true; + this.scanError = ""; try { - var response = await fetch("/api/v1/import", { + var response = await fetch("/api/v1/config/library-roots"); + var payload = await response.json().catch(function () { + return {}; + }); + if (!response.ok || !Array.isArray(payload)) { + throw new Error( + (payload.error && payload.error.message) || "Could not refresh library roots." + ); + } + this.libraryRoots = payload + .filter(function (root) { + return !!(root && root.enabled !== false); + }) + .sort(function (left, right) { + if (!!left.is_default_managed_destination !== !!right.is_default_managed_destination) { + return left.is_default_managed_destination ? -1 : 1; + } + return String(left.name || "").localeCompare(String(right.name || "")); + }); + if (!this.hasSelectedManagedDestination()) { + var managedRoots = this.managedLibraryRoots(); + var defaultRoot = managedRoots.find(function (root) { + return !!root.is_default_managed_destination; + }); + var automaticRoot = + defaultRoot || (managedRoots.length === 1 ? managedRoots[0] : null); + this.targetLibraryRootId = + this.fileHandlingMode === "managed_copy" && automaticRoot + ? Number(automaticRoot.id) + : null; + } + if (this.sourceType === "mylar3") { + this.scheduleMylarPathPreview(); + } + } catch (err) { + this.scanError = + err && err.message ? err.message : "Could not refresh library roots."; + } finally { + this.libraryRootsRefreshing = false; + } + }, + + registerReferenceRoot: async function (group) { + var rootPath = String((group && group.root_path) || "").trim(); + if (!rootPath || this.referenceRootRegistrationPath) { + return false; + } + var leaf = rootPath.replace(/[\\/]+$/, "").split(/[\\/]/).pop() || "library"; + var rootPayload = { + name: ("Existing files - " + leaf + " - " + rootPath).slice(0, 255), + path: rootPath, + allow_referenced_registrations: true, + allow_managed_writes: false, + is_default_managed_destination: false, + }; + var request = async function (path) { + var response = await fetch(path, { method: "POST", headers: { "Content-Type": "application/json", "X-CSRF-Token": this.csrfToken(), }, - body: JSON.stringify({ - source_path: this.sourcePath.trim(), - source_type: this.sourceType, - cv_match_threshold: this.cvMatchThreshold / 100, - min_files_per_series: this.minFilesPerSeries, - file_formats: this.fileFormats.trim() || null, - }), + body: JSON.stringify(rootPayload), + }); + var payload = await response.json().catch(function () { + return {}; }); - if (!response.ok) { - var error = await response - .json() - .catch(function () { - return { detail: "Failed to create import job" }; - }); - throw new Error(error.detail || "Server error (" + response.status + ")"); + var detail = payload.detail; + if (Array.isArray(detail)) { + detail = detail + .map(function (item) { + return item && item.msg ? item.msg : ""; + }) + .filter(Boolean) + .join(" "); + } + throw new Error( + (payload.error && payload.error.message) || + detail || + "Pullbox could not register this library root.", + ); } + return payload; + }.bind(this); - var job = await response.json(); - dispatchImportWizardAdvance({ - step: 2, - jobId: job.id, - jobStatus: job.status, - }); + this.referenceRootRegistrationPath = rootPath; + this.referenceRootRegistrationErrorPath = ""; + this.referenceRootRegistrationError = ""; + try { + var preview = await request("/api/v1/config/library-roots/preview"); + if (!preview.can_create) { + throw new Error( + (preview.blocking_reasons || []).join(" ") || + "This path cannot be registered for existing files.", + ); + } + await request("/api/v1/config/library-roots"); + await this.refreshImportLibraryRoots(); + if (this.sourceType === "mylar3") { + await this.previewMylarPaths(); + } else { + await this.previewLayout(); + } + if (typeof showToast === "function") { + showToast({ message: "Existing-file library root registered.", level: "success" }); + } + return true; } catch (err) { - this.scanError = err && err.message ? err.message : "Failed to start scan."; + this.referenceRootRegistrationErrorPath = rootPath; + this.referenceRootRegistrationError = + err && err.message ? err.message : "Pullbox could not register this library root."; + return false; } finally { - this.scanning = false; + this.referenceRootRegistrationPath = ""; } }, - }); -} -function importJobLogViewerData(config) { - var cfg = config || {}; - var _REQUEST_TIMEOUT_MS = - Number(cfg.requestTimeoutMs || 0) > 0 ? Number(cfg.requestTimeoutMs) : 12000; + referenceLibraryRoots: function () { + return this.libraryRoots.filter(function (root) { + return !!( + root && + root.enabled !== false && + root.allow_referenced_registrations !== false && + root.available !== false + ); + }); + }, + + requiresManagedDestination: function () { + return this.fileHandlingMode === "managed_copy"; + }, + + hasSelectedManagedDestination: function () { + var selectedRootId = Number(this.targetLibraryRootId); + return this.managedLibraryRoots().some(function (root) { + return Number(root.id) === selectedRootId && selectedRootId > 0; + }); + }, + + setLayoutChoice: function (choice) { + this.layoutChoice = choice; + this.scheduleLayoutPreview(); + }, + + clearFuturePolicy: function () { + this.futurePolicyRequestId += 1; + this.futureLayoutRequested = false; + this.futurePolicyComparison = null; + this.futurePolicyLoading = false; + this.futurePolicyError = ""; + }, + + canRequestFutureLayout: function () { + return !!( + this.sourceType === "filesystem" && + this.targetLibraryRootId && + this.layoutPreview && + this.layoutPreview.can_apply_future_policy && + Array.isArray(this.layoutPreview.clusters) && + this.layoutPreview.clusters.length === 1 && + this.layoutPreview.clusters[0].proposed_series_path_template + ); + }, + + toggleFutureLayout: function () { + if (!this.futureLayoutRequested) { + this.futurePolicyRequestId += 1; + this.futurePolicyComparison = null; + this.futurePolicyLoading = false; + this.futurePolicyError = ""; + return; + } + if (!this.canRequestFutureLayout()) { + this.clearFuturePolicy(); + return; + } + this.prepareFuturePolicy(); + }, + + futureLayoutRootChanged: function () { + this.futurePolicyRequestId += 1; + this.futurePolicyComparison = null; + this.futurePolicyError = ""; + if (this.futureLayoutRequested) { + this.prepareFuturePolicy(); + } + }, + + futurePolicyRequest: async function (path, options) { + var response = await fetch( + path, + Object.assign({}, options || {}, { + headers: Object.assign( + { + "Content-Type": "application/json", + "X-CSRF-Token": this.csrfToken(), + }, + (options && options.headers) || {}, + ), + }), + ); + var payload = await response.json().catch(function () { + return {}; + }); + if (!response.ok) { + var detail = payload.detail; + if (Array.isArray(detail)) { + detail = detail + .map(function (item) { + return item && item.msg ? item.msg : ""; + }) + .filter(Boolean) + .join(" "); + } + throw new Error( + (payload.error && payload.error.message) || + detail || + "Pullbox could not prepare this future library policy.", + ); + } + return payload; + }, + + prepareFuturePolicy: async function () { + if (!this.canRequestFutureLayout()) { + this.clearFuturePolicy(); + return; + } + var requestId = ++this.futurePolicyRequestId; + var rootId = Number(this.targetLibraryRootId); + this.futurePolicyLoading = true; + this.futurePolicyError = ""; + this.futurePolicyComparison = null; + try { + var current = await this.futurePolicyRequest( + "/api/v1/config/library-roots/" + rootId + "/naming-policy", + ); + if (requestId !== this.futurePolicyRequestId || rootId !== this.targetLibraryRootId) { + return; + } + var currentPolicy = current.effective_policy; + var cluster = this.layoutPreview.clusters[0]; + this.futureRootPolicy = { + schema_version: 1, + series_path_template: cluster.proposed_series_path_template, + comic_file_template: + cluster.proposed_issue_filename_template || currentPolicy.comic_file_template, + annual_file_template: currentPolicy.annual_file_template, + non_standard_file_template: currentPolicy.non_standard_file_template, + single_non_standard_file_template: currentPolicy.single_non_standard_file_template, + replace_illegal_characters: currentPolicy.replace_illegal_characters, + colon_replacement: currentPolicy.colon_replacement, + }; + await this.previewFuturePolicy(requestId); + } catch (err) { + if (requestId === this.futurePolicyRequestId) { + this.futurePolicyError = + err && err.message + ? err.message + : "Pullbox could not prepare this future library policy."; + } + } finally { + if (requestId === this.futurePolicyRequestId) { + this.futurePolicyLoading = false; + } + } + }, + + futurePolicyExamples: function () { + if ( + !this.layoutPreview || + !Array.isArray(this.layoutPreview.clusters) || + this.layoutPreview.clusters.length !== 1 || + !Array.isArray(this.layoutPreview.clusters[0].examples) + ) { + return []; + } + return this.layoutPreview.clusters[0].examples + .filter(function (example) { + return ( + example && + typeof example.series === "string" && + example.series.trim() && + example.issue_number !== null && + example.issue_number !== "" && + Number.isFinite(Number(example.issue_number)) + ); + }) + .slice(0, 5) + .map(function (example) { + return { + publisher: example.publisher || null, + series: example.series, + year: + example.year !== null && + example.year !== "" && + Number.isFinite(Number(example.year)) && + Number(example.year) > 0 && + Number(example.year) <= 9999 + ? Number(example.year) + : null, + issue_number: Number(example.issue_number), + issue_title: example.issue_title || null, + }; + }); + }, + + previewFuturePolicy: async function (existingRequestId) { + if (!this.futureLayoutRequested || !this.targetLibraryRootId) { + return; + } + var requestId = existingRequestId || ++this.futurePolicyRequestId; + this.futurePolicyLoading = true; + this.futurePolicyError = ""; + try { + var comparison = await this.futurePolicyRequest( + "/api/v1/config/library-roots/" + + Number(this.targetLibraryRootId) + + "/naming-policy/preview", + { + method: "POST", + body: JSON.stringify({ + policy: this.futureRootPolicy, + examples: this.futurePolicyExamples(), + }), + }, + ); + if (requestId === this.futurePolicyRequestId) { + this.futurePolicyComparison = comparison; + } + } catch (err) { + if (requestId === this.futurePolicyRequestId) { + this.futurePolicyComparison = null; + this.futurePolicyError = + err && err.message ? err.message : "Pullbox could not preview this future policy."; + } + } finally { + if (requestId === this.futurePolicyRequestId) { + this.futurePolicyLoading = false; + } + } + }, + + sourceLayoutPayload: function () { + if (this.layoutChoice === "series_folders") { + return { + schema_version: 1, + mode: "preset", + preset: "series_folders", + fallback_to_auto: this.layoutFallbackToAuto, + }; + } + if (this.layoutChoice === "publisher_series") { + return { + schema_version: 1, + mode: "preset", + preset: "publisher_series", + fallback_to_auto: this.layoutFallbackToAuto, + }; + } + if (this.layoutChoice === "custom") { + return { + schema_version: 1, + mode: "custom", + series_path_template: this.customSeriesPathTemplate.trim(), + issue_filename_template: this.customIssueFilenameTemplate.trim() || null, + fallback_to_auto: this.layoutFallbackToAuto, + }; + } + return { + schema_version: 1, + mode: "auto", + fallback_to_auto: true, + }; + }, + + canAnalyzeLayout: function () { + if (this.sourceType !== "filesystem" || !this.sourcePath.trim()) { + return false; + } + return this.layoutChoice !== "custom" || !!this.customSeriesPathTemplate.trim(); + }, + + canAnalyzeStoryArcs: function () { + return !!(this.sourceType && this.sourcePath.trim()); + }, + + canAnalyzeMylarPaths: function () { + if (this.sourceType !== "mylar3" || !this.sourcePath.trim()) { + return false; + } + return this.mylarPathMappings.every(function (mapping) { + return !!( + mapping && + String(mapping.stored_prefix || "").trim() && + String(mapping.pullbox_prefix || "").trim() + ); + }); + }, + + advancedAttentionItems: function () { + if (!this.sourceType || !this.sourcePath.trim()) { + return []; + } + var items = []; + if (this.requiresManagedDestination() && !this.hasSelectedManagedDestination()) { + var managedRoots = this.managedLibraryRoots(); + var preferredRoot = managedRoots.find(function (root) { + return !!root.is_default_managed_destination; + }); + if (!preferredRoot && managedRoots.length === 1) { + preferredRoot = managedRoots[0]; + } + items.push({ + key: "managed-destination", + code: "managed_destination_required", + blocks_import: true, + reason: "Choose a managed destination", + suggested_action: + "Open Where new files go and select an available, writable Pullbox library.", + root_path: "", + action: preferredRoot + ? { kind: "select_managed_destination", library_root_id: Number(preferredRoot.id) } + : null, + details: { + title: "Choose where Pullbox will manage files", + series_count: 0, + location_count: 0, + known_paths: managedRoots.map(function (root) { + return String(root.path || ""); + }), + steps: [ + "Open Where new files go.", + "Choose an available, writable Pullbox library.", + "Recheck the import issues before starting the scan.", + ], + }, + }); + } + if ( + this.sourceType === "filesystem" && + this.fileHandlingMode === "in_place" && + this.layoutPreview && + !this.layoutPreview.can_keep_in_place + ) { + var layoutWarnings = Array.isArray(this.layoutPreview.warnings) + ? this.layoutPreview.warnings + : []; + var sourceOutsideLibraryRoot = layoutWarnings.includes( + "source_outside_library_root", + ); + var otherLayoutWarnings = layoutWarnings.filter(function (warning) { + return warning !== "source_outside_library_root"; + }); + if (sourceOutsideLibraryRoot) { + items.push({ + key: "in-place-reference-root", + code: "source_outside_library_root", + blocks_import: true, + reason: "Register this folder for existing files", + suggested_action: + "Pullbox can add this exact folder as a reference-only library root. Files stay where they are and will not be renamed or modified.", + root_path: this.sourcePath.trim(), + action: { + kind: "register_reference_root", + root_path: this.sourcePath.trim(), + }, + details: { + title: "Register a reference-only library root", + series_count: 0, + location_count: Number(this.layoutPreview.files_considered || 0), + known_paths: [this.sourcePath.trim()], + steps: [ + "Register this exact folder as an enabled library root for existing-file references.", + "Keep managed writes disabled so Pullbox cannot rename or modify files in this folder.", + "Recheck the folder before starting the scan.", + ], + }, + }); + } + if ( + !sourceOutsideLibraryRoot || + otherLayoutWarnings.length || + Number(this.layoutPreview.files_outside_root || 0) > 0 || + !!this.layoutPreview.partial + ) { + items.push({ + key: "in-place-layout", + code: "in_place_layout_unavailable", + blocks_import: true, + reason: "Some files cannot be safely referenced in place", + suggested_action: + "Review the folder details and correct the listed source layout or access problem.", + root_path: this.sourcePath.trim(), + action: null, + details: { + title: "Review files that cannot be referenced", + series_count: 0, + location_count: Number(this.layoutPreview.files_considered || 0), + known_paths: [this.sourcePath.trim()], + steps: [ + "Review the folder-layout details for paths Pullbox could not safely reference.", + "Correct the source layout or access problem shown in the preview.", + "Recheck the import issues before starting the scan.", + ], + }, + }); + } + } + if (this.sourceType !== "mylar3") { + return items; + } + if (this.mylarPathPreviewError) { + items.push({ + key: "mylar-path-error", + code: "mylar_path_analysis_failed", + blocks_import: true, + reason: "Pullbox could not analyze the Mylar library paths", + suggested_action: this.mylarPathPreviewError, + root_path: this.sourcePath.trim(), + action: null, + details: { + title: "Mylar path analysis did not finish", + series_count: 0, + location_count: 0, + known_paths: [this.sourcePath.trim()], + steps: [ + "Confirm that the selected Mylar database still exists and is readable inside Pullbox.", + "Correct the mount or file permissions if the database is unavailable.", + "Run the path check again after access is restored.", + ], + }, + }); + return items; + } + if (!this.mylarPathPreview) { + return items; + } + var serverItems = Array.isArray(this.mylarPathPreview.attention_items) + ? this.mylarPathPreview.attention_items + : []; + serverItems.forEach( + function (item) { + var action = item && item.action ? item.action : null; + var acknowledged = action + ? this.acknowledgedAttentionActions[String(item.key || "")] + : null; + if ( + action && + action.kind === "acknowledge_unavailable" && + acknowledged === action.fingerprint + ) { + return; + } + items.push(item); + }.bind(this), + ); + return items; + }, + + advancedAttentionCount: function () { + return this.advancedAttentionItems().length; + }, + + advancedAttentionError: function (item) { + return this.attentionResolutionErrors[String((item && item.key) || "")] || ""; + }, + + restoreSkippedAttentionActions: function (preview) { + this.acknowledgedAttentionActions = {}; + var sourcePath = this.sourcePath.trim(); + var attentionFingerprint = String((preview && preview.attention_fingerprint) || ""); + if (!sourcePath || !attentionFingerprint) { + return; + } + try { + var raw = window.sessionStorage.getItem("pb-import-mylar-skips:v1"); + if (!raw) { + return; + } + var stored = JSON.parse(raw); + if ( + !stored || + stored.source_path !== sourcePath || + stored.attention_fingerprint !== attentionFingerprint + ) { + window.sessionStorage.removeItem("pb-import-mylar-skips:v1"); + return; + } + var available = {}; + (Array.isArray(preview.attention_items) ? preview.attention_items : []).forEach( + function (item) { + if ( + item && + item.action && + item.action.kind === "acknowledge_unavailable" && + item.action.fingerprint + ) { + available[String(item.key || "")] = String(item.action.fingerprint); + } + }, + ); + var restored = {}; + Object.keys(stored.actions || {}).forEach(function (key) { + var fingerprint = String(stored.actions[key] || ""); + if (fingerprint && available[key] === fingerprint) { + restored[key] = fingerprint; + } + }); + this.acknowledgedAttentionActions = restored; + if (!Object.keys(restored).length) { + window.sessionStorage.removeItem("pb-import-mylar-skips:v1"); + } + } catch (_err) { + this.acknowledgedAttentionActions = {}; + try { + window.sessionStorage.removeItem("pb-import-mylar-skips:v1"); + } catch (_storageErr) { + // Storage may be unavailable in privacy-restricted browser contexts. + } + } + }, + + persistSkippedAttentionActions: function () { + var sourcePath = this.sourcePath.trim(); + var attentionFingerprint = String( + (this.mylarPathPreview && this.mylarPathPreview.attention_fingerprint) || "", + ); + try { + if ( + !sourcePath || + !attentionFingerprint || + !Object.keys(this.acknowledgedAttentionActions).length + ) { + window.sessionStorage.removeItem("pb-import-mylar-skips:v1"); + return; + } + window.sessionStorage.setItem( + "pb-import-mylar-skips:v1", + JSON.stringify({ + source_path: sourcePath, + attention_fingerprint: attentionFingerprint, + actions: this.acknowledgedAttentionActions, + }), + ); + } catch (_err) { + // Skips still work for this page when browser storage is unavailable. + } + }, + + resolveAdvancedAttention: async function (item) { + var key = String((item && item.key) || ""); + var action = item && item.action ? item.action : null; + if (!key || !action || this.attentionResolvingKey) { + return; + } + this.attentionResolvingKey = key; + this.attentionResolutionErrors = Object.assign({}, this.attentionResolutionErrors, { + [key]: "", + }); + try { + switch (action.kind) { + case "select_managed_destination": + this.targetLibraryRootId = Number(action.library_root_id || 0) || null; + if (this.sourceType === "mylar3") { + await this.previewMylarPaths(); + } + break; + case "register_reference_root": + if (!(await this.registerReferenceRoot(action))) { + throw new Error( + this.referenceRootRegistrationError || + "Pullbox could not register this path for existing files.", + ); + } + break; + case "remove_ineffective_mapping": + var mappingIndex = this.mylarPathMappings.findIndex(function (mapping) { + return !!( + mapping && + mapping.stored_prefix === action.stored_prefix && + mapping.pullbox_prefix === action.pullbox_prefix + ); + }); + if (mappingIndex < 0) { + throw new Error("This mapping changed. Review the current path details."); + } + this.mylarPathAutoDetect = false; + this.mylarPathMappings.splice(mappingIndex, 1); + await this.previewMylarPaths(); + break; + default: + throw new Error("This issue does not have a supported automatic resolution."); + } + if (this.sourceType === "mylar3" && this.mylarPathPreviewError) { + throw new Error(this.mylarPathPreviewError); + } + if (this.sourceType === "filesystem" && this.layoutPreviewError) { + throw new Error(this.layoutPreviewError); + } + if ( + this.advancedAttentionItems().some(function (candidate) { + return String((candidate && candidate.key) || "") === key; + }) + ) { + throw new Error( + "Pullbox made the change, but this issue is still present after rechecking.", + ); + } + } catch (err) { + this.attentionResolutionErrors = Object.assign({}, this.attentionResolutionErrors, { + [key]: err && err.message ? err.message : "Pullbox could not resolve this issue.", + }); + } finally { + this.attentionResolvingKey = ""; + } + }, + + skipAdvancedAttention: async function (item) { + var key = String((item && item.key) || ""); + var action = item && item.action ? item.action : null; + if ( + !key || + !action || + action.kind !== "acknowledge_unavailable" || + this.attentionResolvingKey + ) { + return; + } + this.attentionResolvingKey = key; + this.attentionResolutionErrors = Object.assign({}, this.attentionResolutionErrors, { + [key]: "", + }); + try { + await this.previewMylarPaths(); + if (this.mylarPathPreviewError) { + throw new Error(this.mylarPathPreviewError); + } + var attentionItems = + this.mylarPathPreview && Array.isArray(this.mylarPathPreview.attention_items) + ? this.mylarPathPreview.attention_items + : []; + var confirmed = attentionItems.find(function (candidate) { + return !!( + candidate && + candidate.key === key && + candidate.action && + candidate.action.kind === "acknowledge_unavailable" && + candidate.action.fingerprint === action.fingerprint + ); + }); + if (!confirmed) { + throw new Error("The path evidence changed. Review the updated issue before continuing."); + } + this.acknowledgedAttentionActions = Object.assign( + {}, + this.acknowledgedAttentionActions, + { [key]: action.fingerprint }, + ); + this.persistSkippedAttentionActions(); + } catch (err) { + this.attentionResolutionErrors = Object.assign({}, this.attentionResolutionErrors, { + [key]: err && err.message ? err.message : "Pullbox could not skip this issue.", + }); + } finally { + this.attentionResolvingKey = ""; + } + }, + + openAdvancedAttentionDetails: function (item, returnFocus) { + this.attentionDetailsItem = item || null; + this.attentionDetailsReturnFocus = returnFocus || null; + this.attentionDetailsOpen = !!item; + var self = this; + this.$nextTick(function () { + if (self.$refs.attentionDetailsDialog) { + self.$refs.attentionDetailsDialog.focus(); + } + }); + }, + + closeAdvancedAttentionDetails: function () { + var returnFocus = this.attentionDetailsReturnFocus; + this.attentionDetailsOpen = false; + this.attentionDetailsItem = null; + this.attentionDetailsReturnFocus = null; + if (returnFocus && typeof returnFocus.focus === "function") { + this.$nextTick(function () { + returnFocus.focus(); + }); + } + }, + + toggleStoryArcImport: function () { + if (!this.storyArcImportRequested) { + this.storyArcMaterializationRequested = false; + } + }, + + canStartScan: function () { + if (!this.sourcePath.trim() || this.scanning) { + return false; + } + if (this.requiresManagedDestination() && !this.hasSelectedManagedDestination()) { + return false; + } + if ( + this.sourceType === "mylar3" && + (!this.mylarPathPreview || + !(this.mylarPathPreview.can_confirm || this.mylarPathPreview.can_continue_with_unresolved)) + ) { + return false; + } + if (this.fileHandlingMode === "in_place") { + if (this.sourceType !== "mylar3" && ( + this.sourceType !== "filesystem" || + !this.layoutPreview || + !this.layoutPreview.can_keep_in_place + )) { + return false; + } + } + if ( + this.futureLayoutRequested && + (!this.canRequestFutureLayout() || + this.futurePolicyLoading || + this.futurePolicyError || + !this.futurePolicyComparison) + ) { + return false; + } + return this.layoutChoice !== "custom" || !!this.customSeriesPathTemplate.trim(); + }, + + cancelMylarPathPreviewRefresh: function (preserveInitialLoading) { + if (this.mylarPathPreviewTimer) { + clearTimeout(this.mylarPathPreviewTimer); + this.mylarPathPreviewTimer = null; + } + if (this.mylarPathPreviewController) { + this.mylarPathPreviewController.abort(); + this.mylarPathPreviewController = null; + } + this.mylarPathPreviewRequestId += 1; + this.mylarPathPreviewLoading = !!preserveInitialLoading && !this.mylarPathPreview; + }, + + clearMylarPathPreview: function (resetMappings) { + this.cancelMylarPathPreviewRefresh(false); + this.mylarPathPreview = null; + this.mylarPathPreviewError = ""; + this.mylarPathConfirmed = false; + this.mylarUnresolvedConfirmed = false; + this.acknowledgedAttentionActions = {}; + this.attentionResolutionErrors = {}; + if (resetMappings) { + this.mylarPathMappings = []; + this.mylarPathAutoDetect = true; + } + }, + + scheduleMylarPathPreview: function () { + if (this.sourceType !== "mylar3") { + return; + } + var preserveInitialLoading = this.mylarPathPreviewLoading && !this.mylarPathPreview; + this.cancelMylarPathPreviewRefresh(preserveInitialLoading); + if (!this.canAnalyzeMylarPaths()) { + this.mylarPathPreviewLoading = false; + return; + } + var self = this; + this.mylarPathPreviewTimer = setTimeout(function () { + self.mylarPathPreviewTimer = null; + self.previewMylarPaths(); + }, 650); + }, + + mylarPathMappingChanged: function () { + this.mylarPathAutoDetect = false; + this.scheduleMylarPathPreview(); + }, + + addMylarPathMapping: function () { + this.mylarPathAutoDetect = false; + this.cancelMylarPathPreviewRefresh(false); + this.mylarPathMappings.push({ + id: ++this.mylarPathMappingId, + stored_prefix: "", + pullbox_prefix: "", + }); + }, + + removeMylarPathMapping: function (index) { + this.mylarPathAutoDetect = false; + this.mylarPathMappings.splice(index, 1); + this.scheduleMylarPathPreview(); + }, + + resetAutomaticMylarPaths: function () { + this.previewMylarPaths({ resetAutomatic: true }); + }, + + reconcileMylarPathMappings: function (mappings) { + var available = this.mylarPathMappings.slice(); + return (mappings || []).map( + function (mapping) { + var existingIndex = available.findIndex(function (candidate) { + return !!( + candidate && + candidate.stored_prefix === mapping.stored_prefix && + candidate.pullbox_prefix === mapping.pullbox_prefix + ); + }); + var existing = existingIndex >= 0 ? available.splice(existingIndex, 1)[0] : null; + return { + id: existing ? existing.id : ++this.mylarPathMappingId, + stored_prefix: mapping.stored_prefix, + pullbox_prefix: mapping.pullbox_prefix, + }; + }.bind(this), + ); + }, + + previewMylarPaths: async function (options) { + var opts = options || {}; + var resetAutomatic = opts.resetAutomatic === true; + if ( + this.sourceType !== "mylar3" || + !this.sourcePath.trim() || + (!resetAutomatic && !this.canAnalyzeMylarPaths()) + ) { + return; + } + this.cancelMylarPathPreviewRefresh(false); + var requestId = this.mylarPathPreviewRequestId; + var controller = new AbortController(); + var requestAutoDetect = resetAutomatic ? true : this.mylarPathAutoDetect; + var requestMappings = resetAutomatic + ? [] + : this.mylarPathMappings.map(function (mapping) { + return { + stored_prefix: String(mapping.stored_prefix || "").trim(), + pullbox_prefix: String(mapping.pullbox_prefix || "").trim(), + }; + }); + this.mylarPathPreviewController = controller; + this.mylarPathPreviewLoading = true; + try { + var response = await fetch("/api/v1/import/mylar-path-preview", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": this.csrfToken(), + }, + signal: controller.signal, + body: JSON.stringify({ + source_path: this.sourcePath.trim(), + source_type: "mylar3", + file_handling_mode: this.fileHandlingMode, + auto_detect: requestAutoDetect, + mappings: requestMappings, + }), + }); + var payload = await response.json().catch(function () { + return {}; + }); + if (!response.ok) { + var detail = payload.detail; + if (Array.isArray(detail)) { + detail = detail + .map(function (item) { + return item && item.msg ? item.msg : ""; + }) + .filter(Boolean) + .join(" "); + } + throw new Error( + detail || + (payload.error && payload.error.message) || + "Pullbox could not analyze the Mylar paths.", + ); + } + if (requestId !== this.mylarPathPreviewRequestId) { + return; + } + this.mylarPathPreviewError = ""; + this.mylarPathAutoDetect = requestAutoDetect; + this.restoreSkippedAttentionActions(payload); + this.mylarPathPreview = payload; + this.mylarPathMappings = this.reconcileMylarPathMappings(payload.mappings); + this.mylarPathConfirmed = !!( + payload.can_confirm || payload.can_continue_with_unresolved + ); + this.mylarUnresolvedConfirmed = !!payload.can_continue_with_unresolved; + } catch (err) { + if (err && err.name === "AbortError") { + return; + } + if (requestId === this.mylarPathPreviewRequestId) { + this.mylarPathPreviewError = + err && err.message ? err.message : "Pullbox could not analyze the Mylar paths."; + } + } finally { + if (requestId === this.mylarPathPreviewRequestId) { + this.mylarPathPreviewLoading = false; + this.mylarPathPreviewController = null; + } + } + }, + + mylarPathMappingEvidence: function (index) { + if (!this.mylarPathPreview || !Array.isArray(this.mylarPathPreview.mappings)) { + return null; + } + return this.mylarPathPreview.mappings[index] || null; + }, + + clearLayoutPreview: function () { + if (this.layoutPreviewTimer) { + clearTimeout(this.layoutPreviewTimer); + this.layoutPreviewTimer = null; + } + if (this.layoutPreviewController) { + this.layoutPreviewController.abort(); + this.layoutPreviewController = null; + } + this.layoutPreviewRequestId += 1; + this.layoutPreview = null; + this.layoutPreviewLoading = false; + this.layoutPreviewError = ""; + this.clearFuturePolicy(); + }, + + clearStoryArcPreview: function () { + if (this.storyArcPreviewTimer) { + clearTimeout(this.storyArcPreviewTimer); + this.storyArcPreviewTimer = null; + } + if (this.storyArcPreviewController) { + this.storyArcPreviewController.abort(); + this.storyArcPreviewController = null; + } + this.storyArcPreviewRequestId += 1; + this.storyArcPreview = emptyStoryArcPreview(); + this.storyArcPreviewLoading = false; + this.storyArcPreviewError = ""; + this.storyArcMaterializationRequested = false; + }, + + scheduleStoryArcPreview: function () { + if (this.storyArcPreviewTimer) { + clearTimeout(this.storyArcPreviewTimer); + this.storyArcPreviewTimer = null; + } + if (this.storyArcPreviewController) { + this.storyArcPreviewController.abort(); + this.storyArcPreviewController = null; + } + this.storyArcPreviewRequestId += 1; + this.storyArcPreview = emptyStoryArcPreview(); + this.storyArcPreviewLoading = false; + this.storyArcPreviewError = ""; + this.storyArcMaterializationRequested = false; + if (!this.canAnalyzeStoryArcs()) { + return; + } + var self = this; + this.storyArcPreviewTimer = setTimeout(function () { + self.storyArcPreviewTimer = null; + self.previewStoryArcs(); + }, 650); + }, + + previewStoryArcs: async function () { + if (!this.canAnalyzeStoryArcs()) { + return; + } + if (this.storyArcPreviewTimer) { + clearTimeout(this.storyArcPreviewTimer); + this.storyArcPreviewTimer = null; + } + if (this.storyArcPreviewController) { + this.storyArcPreviewController.abort(); + } + var requestId = ++this.storyArcPreviewRequestId; + var controller = new AbortController(); + this.storyArcPreviewController = controller; + this.storyArcPreviewLoading = true; + this.storyArcPreviewError = ""; + this.storyArcPreview = emptyStoryArcPreview(); + this.storyArcMaterializationRequested = false; + try { + var response = await fetch("/api/v1/import/story-arc-preview", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": this.csrfToken(), + }, + signal: controller.signal, + body: JSON.stringify({ + source_path: this.sourcePath.trim(), + source_type: this.sourceType, + }), + }); + var payload = await response.json().catch(function () { + return {}; + }); + if (!response.ok) { + var detail = payload.detail; + if (Array.isArray(detail)) { + detail = detail + .map(function (item) { + return item && item.msg ? item.msg : ""; + }) + .filter(Boolean) + .join(" "); + } + throw new Error( + detail || + (payload.error && payload.error.message) || + "Pullbox could not inspect Story Arc evidence.", + ); + } + if (requestId === this.storyArcPreviewRequestId) { + this.storyArcPreview = payload; + } + } catch (err) { + if (err && err.name === "AbortError") { + return; + } + if (requestId === this.storyArcPreviewRequestId) { + this.storyArcPreviewError = + err && err.message ? err.message : "Pullbox could not inspect Story Arc evidence."; + } + } finally { + if (requestId === this.storyArcPreviewRequestId) { + this.storyArcPreviewLoading = false; + this.storyArcPreviewController = null; + } + } + }, + + storyArcSettingValue: function (setting) { + if (!setting || setting.value === null || setting.value === undefined) { + return "Not configured"; + } + if (setting.value === true) { + return "Enabled"; + } + if (setting.value === false) { + return "Disabled"; + } + return String(setting.value); + }, + + scheduleLayoutPreview: function () { + if (this.layoutPreviewTimer) { + clearTimeout(this.layoutPreviewTimer); + this.layoutPreviewTimer = null; + } + if (this.layoutPreviewController) { + this.layoutPreviewController.abort(); + this.layoutPreviewController = null; + } + this.layoutPreviewRequestId += 1; + this.layoutPreview = null; + this.layoutPreviewLoading = false; + this.layoutPreviewError = ""; + this.clearFuturePolicy(); + if (!this.canAnalyzeLayout()) { + return; + } + var self = this; + this.layoutPreviewTimer = setTimeout(function () { + self.layoutPreviewTimer = null; + self.previewLayout(); + }, 500); + }, + + previewLayout: async function () { + if (!this.canAnalyzeLayout()) { + return; + } + if (this.layoutPreviewTimer) { + clearTimeout(this.layoutPreviewTimer); + this.layoutPreviewTimer = null; + } + if (this.layoutPreviewController) { + this.layoutPreviewController.abort(); + } + this.clearFuturePolicy(); + + var requestId = ++this.layoutPreviewRequestId; + var controller = new AbortController(); + this.layoutPreviewController = controller; + this.layoutPreviewLoading = true; + this.layoutPreviewError = ""; + + try { + var response = await fetch("/api/v1/import/layout-preview", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": this.csrfToken(), + }, + signal: controller.signal, + body: JSON.stringify({ + source_path: this.sourcePath.trim(), + source_type: "filesystem", + layout: this.sourceLayoutPayload(), + }), + }); + var payload = await response.json().catch(function () { + return {}; + }); + if (!response.ok) { + var detail = payload.detail; + if (Array.isArray(detail)) { + detail = detail + .map(function (item) { + return item && item.msg ? item.msg : ""; + }) + .filter(Boolean) + .join(" "); + } + var message = + detail || + (payload.error && payload.error.message) || + "Pullbox could not analyze this folder layout."; + throw new Error(message); + } + if (requestId !== this.layoutPreviewRequestId) { + return; + } + this.layoutPreview = payload; + } catch (err) { + if (err && err.name === "AbortError") { + return; + } + if (requestId === this.layoutPreviewRequestId) { + this.layoutPreviewError = + err && err.message ? err.message : "Pullbox could not analyze this folder layout."; + } + } finally { + if (requestId === this.layoutPreviewRequestId) { + this.layoutPreviewLoading = false; + this.layoutPreviewController = null; + } + } + }, + + layoutClassificationLabel: function (value) { + return String(value || "needs_review") + .replace(/_/g, " ") + .replace(/\b\w/g, function (letter) { + return letter.toUpperCase(); + }); + }, + + layoutPreviewSummary: function () { + if (!this.layoutPreview) { + return ""; + } + return ( + String(this.layoutPreview.files_fitting || 0) + + " of " + + String(this.layoutPreview.files_considered || 0) + + " sampled files fit this interpretation" + ); + }, + + startScan: async function () { + if (!this.canStartScan()) { + return; + } + + this.scanning = true; + this.scanError = ""; + + try { + var response = await fetch("/api/v1/import", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": this.csrfToken(), + }, + body: JSON.stringify({ + source_path: this.sourcePath.trim(), + source_type: this.sourceType, + cv_match_threshold: this.cvMatchThreshold / 100, + min_files_per_series: this.minFilesPerSeries, + file_formats: this.fileFormats.trim() || null, + file_handling_mode: this.fileHandlingMode, + source_layout: this.sourceLayoutPayload(), + target_library_root_id: this.targetLibraryRootId + ? Number(this.targetLibraryRootId) + : null, + future_layout_requested: this.futureLayoutRequested, + future_root_policy: this.futureLayoutRequested ? this.futureRootPolicy : null, + story_arc_import_requested: false, + story_arc_materialization_requested: false, + mylar3_path_map: this.sourceType === "mylar3" + ? Object.assign({}, this.mylarPathPreview.path_map || {}) + : {}, + mylar3_path_map_confirmed: this.sourceType === "mylar3", + mylar3_allow_unresolved_paths: this.sourceType === "mylar3" && + !!this.mylarPathPreview.can_continue_with_unresolved, + mylar3_unresolved_fingerprint: this.sourceType === "mylar3" && + this.mylarPathPreview.can_continue_with_unresolved + ? this.mylarPathPreview.unresolved_fingerprint : null, + }), + }); + + if (!response.ok) { + var error = await response + .json() + .catch(function () { + return { detail: "Failed to create import job" }; + }); + throw new Error(error.detail || "Server error (" + response.status + ")"); + } + + var job = await response.json(); + dispatchImportWizardAdvance({ + step: 2, + jobId: job.id, + jobStatus: job.status, + }); + } catch (err) { + this.scanError = err && err.message ? err.message : "Failed to start scan."; + } finally { + this.scanning = false; + } + }, + }); +} + +function importJobLogViewerData(config) { + var cfg = config || {}; + var _REQUEST_TIMEOUT_MS = + Number(cfg.requestTimeoutMs || 0) > 0 ? Number(cfg.requestTimeoutMs) : 12000; + var requestedMaxEntries = Number(cfg.maxEntries || 500); + var _MAX_RETAINED_ENTRIES = Math.max( + 250, + Math.min( + 500, + Number.isFinite(requestedMaxEntries) ? Math.floor(requestedMaxEntries) : 500, + ), + ); return { jobId: Number(cfg.jobId || 0), @@ -3174,9 +4854,19 @@ function importJobLogViewerData(config) { return "0 entries"; } if (!this.levelFilter && !this.searchQuery) { + if (this.totalCount > this.entries.length) { + return this.entries.length + " recent of " + this.totalCount + " entries"; + } return this.totalCount + " entries"; } - return this.filteredCount + " entries (filtered from " + this.totalCount + ")"; + return ( + this.filteredCount + + " recent matches (" + + this.entries.length + + " recent of " + + this.totalCount + + " entries)" + ); }, get downloadHref() { @@ -3265,6 +4955,17 @@ function importJobLogViewerData(config) { }; }, + _trimRetainedEntries: function () { + var overflow = this.entries.length - _MAX_RETAINED_ENTRIES; + if (overflow <= 0) { + return; + } + this.entries.splice(0, overflow); + if (this.currentPage > this.totalPages) { + this.currentPage = this.totalPages; + } + }, + _appendStreamEntry: function (event) { if (!event || !event.data) { return; @@ -3283,6 +4984,7 @@ function importJobLogViewerData(config) { var shouldFollowTail = this._shouldFollowLiveTail(); this.entries.push(this._normalizeStreamEntry(payload)); this.totalCount += 1; + this._trimRetainedEntries(); if (shouldFollowTail) { this.currentPage = this.totalPages; } else if (this.currentPage > this.totalPages) { @@ -3370,11 +5072,12 @@ function importJobLogViewerData(config) { } if (incrementalData) { var newItems = Array.isArray(incrementalData.items) ? incrementalData.items : []; - this.totalCount = Number(incrementalData.total || this.totalCount || this.entries.length) || this.entries.length; if (newItems.length) { for (var n = 0; n < newItems.length; n++) { this.entries.push(this._normalizeEntry(newItems[n])); } + this.totalCount += newItems.length; + this._trimRetainedEntries(); if (this.currentPage > this.totalPages) { this.currentPage = this.totalPages; } @@ -3383,35 +5086,25 @@ function importJobLogViewerData(config) { } } - var page = 1; - var pageSize = 500; - var allEntries = []; - var total = 0; - - while (true) { - var data = await this._fetchJson( - "/api/v1/import/" + this.jobId + "/logs?page=" + page + "&page_size=" + pageSize + "&order=asc", - request, - ); - if (request.token !== this._requestToken) { - return; - } - if (!data) { - break; - } - var items = Array.isArray(data.items) ? data.items : []; - if (page === 1) { - total = Number(data.total || items.length) || 0; - } - for (var i = 0; i < items.length; i++) { - allEntries.push(this._normalizeEntry(items[i])); - } - if (!items.length || allEntries.length >= total) { - break; - } - page += 1; + var data = await this._fetchJson( + "/api/v1/import/" + + this.jobId + + "/logs?page=1&page_size=" + + _MAX_RETAINED_ENTRIES + + "&order=desc", + request, + ); + if (request.token !== this._requestToken) { + return; + } + if (!data) { + return; } + var items = Array.isArray(data.items) ? data.items : []; + var allEntries = items.slice().reverse().map(this._normalizeEntry.bind(this)); + var total = Number(data.total || items.length) || 0; + var shouldFollowTail = this._shouldFollowLiveTail(); this.entries = allEntries; this.totalCount = total || allEntries.length; @@ -3494,6 +5187,9 @@ function importProgressData(jobId, nextStep, sourceType) { pausing: false, optimisticPauseRequested: false, resuming: false, + retryingStoryArcPlacements: false, + storyArcPlacementRetryError: "", + storyArcPlacementRetrySuccess: "", cancelPrompting: false, cancelling: false, cancelReturnStarted: false, @@ -3712,6 +5408,16 @@ function importProgressData(jobId, nextStep, sourceType) { ); }, + showRetryStoryArcPlacementsAction: function () { + if (!this.isImportMode() || this.completed || this.failed) { + return false; + } + return ( + this.retryingStoryArcPlacements || + !!(this.controlState && this.controlState.can_retry_story_arc_placements) + ); + }, + canResumeAction: function () { if (!this.showResumeAction()) { return false; @@ -3770,6 +5476,7 @@ function importProgressData(jobId, nextStep, sourceType) { matching: "Matching series against ComicVine...", file_matching: "Matching files to issues...", importing: "Importing series into Pullbox...", + story_arc_placements: "Creating Story Arc placements...", rollback: "Rolling back import actions...", review: "Complete", done: "Run stopped", @@ -3916,6 +5623,11 @@ function importProgressData(jobId, nextStep, sourceType) { return ""; }, + currentItemIsIndeterminate: function () { + return this.currentItemKind === "scan" && + this.currentItemStage === "inventory" && this.currentItemProgressValue == null; + }, + currentItemProgress: function () { if (this.currentFileName) { return this.currentFileProgress; @@ -4282,6 +5994,7 @@ function importProgressData(jobId, nextStep, sourceType) { return ( this.showPauseAction() || this.showResumeAction() || + this.showRetryStoryArcPlacementsAction() || this.showCancelAction() || this.failed || this.completed @@ -4343,8 +6056,8 @@ function importProgressData(jobId, nextStep, sourceType) { return; } - var computed = this._computeEtaSeconds(this.startedAt, this.progress); - this.etaSeconds = computed; + // Weighted workflow milestones are not a measure of remaining runtime. + this.etaSeconds = null; this.etaCapturedAt = Date.now(); }, @@ -4403,7 +6116,7 @@ function importProgressData(jobId, nextStep, sourceType) { this.etaSeconds - Math.max(0, Math.floor((this.nowMs - this.etaCapturedAt) / 1000)), ) - : this._computeEtaSeconds(this.startedAt, this.progress); + : null; if (etaSeconds == null) { return "Estimating..."; @@ -5333,6 +7046,75 @@ function importProgressData(jobId, nextStep, sourceType) { } }, + retryStoryArcPlacements: async function () { + if ( + !this.jobId || + this.retryingStoryArcPlacements || + !this.showRetryStoryArcPlacementsAction() + ) { + return; + } + + this.retryingStoryArcPlacements = true; + this.storyArcPlacementRetryError = ""; + this.storyArcPlacementRetrySuccess = ""; + try { + var response = await fetch( + "/api/v1/import/" + this.jobId + "/story-arc-placements/retry", + { + method: "POST", + headers: { "X-CSRF-Token": readCsrfTokenFromBody() }, + }, + ); + var payload = await response.json().catch(function () { + return {}; + }); + if (!response.ok) { + var detail = + payload && typeof payload.detail === "string" + ? payload.detail + : "Failed to retry Story Arc placements."; + throw new Error(detail); + } + + var retryingCount = Math.max(0, Number(payload.retrying_count) || 0); + var placementLabel = retryingCount === 1 ? "placement" : "placements"; + this.controlState = Object.assign({}, this.controlState, { + can_pause: false, + can_resume: false, + can_retry_story_arc_placements: false, + can_cancel: true, + requested_action: "none", + }); + this.jobStatus = "importing"; + this.phase = "story_arc_placements"; + this.phaseLabel = this.phaseLabelForKey(this.phase); + this.progress = 99; + this.failed = false; + this.completed = false; + this.message = "Retrying " + retryingCount + " Story Arc " + placementLabel + "..."; + this.storyArcPlacementRetrySuccess = + "Retry requested for " + retryingCount + " Story Arc " + placementLabel + "."; + this.emitFooterState(); + this.startClock(); + this.startPolling(); + if (!this.evtSource) { + this.connectSSE(); + } + } catch (err) { + var retryMessage = + err && err.message + ? err.message + : "Failed to retry Story Arc placements. Please try again."; + this.storyArcPlacementRetryError = retryMessage; + if (typeof showToast === "function") { + showToast({ message: retryMessage, level: "error" }); + } + } finally { + this.retryingStoryArcPlacements = false; + } + }, + cancelRun: async function () { if ( !this.jobId || @@ -5570,9 +7352,12 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { jobId: cfg.jobId, currentView: cfg.currentView || "series", conflictsCommitted: false, - showCancelModal: false, - cancelling: false, reviewToken: typeof cfg.reviewToken === "string" ? cfg.reviewToken : "", + preferredRootId: + cfg.preferredRootId === null || typeof cfg.preferredRootId === "undefined" + ? null + : Number(cfg.preferredRootId), + splitSeriesRequiresPreferredRoot: Boolean(cfg.splitSeriesRequiresPreferredRoot), init: function () { this.rehydrateAfterShellSwap(); @@ -5634,6 +7419,14 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { return Number(this.selectedItemCount) || 0; }, + hasRequiredPreferredRoot: function () { + if (!this.splitSeriesRequiresPreferredRoot) { + return true; + } + var rootId = Number(this.preferredRootId); + return Number.isFinite(rootId) && rootId > 0; + }, + importSelectionLabel: function () { var total = this.totalSelectionCount(); return total + " items selected for import"; @@ -5673,7 +7466,8 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { var importButton = root.querySelector("[data-import-review-import-button]"); if (importButton) { - importButton.disabled = total === 0 || this.confirming; + importButton.disabled = + total === 0 || this.confirming || !this.hasRequiredPreferredRoot(); } }, @@ -5793,26 +7587,193 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { "Content-Type": "application/json", "X-CSRF-Token": readCsrfTokenFromBody(), }, - body: JSON.stringify({ include_in_import: checked }), + body: JSON.stringify({ include_in_import: checked }), + }, + ); + + if (!response.ok) { + var error = await response + .json() + .catch(function () { + return {}; + }); + throw new Error(error.detail || "Failed to update series selection."); + } + await this.refreshReviewSummary(); + await this.refreshSeriesReview(); + } catch (err) { + await this.refreshSeriesReviewQuietly(); + if (typeof showToast === "function") { + showToast({ + message: + err && err.message ? err.message : "Failed to update series selection.", + level: "error", + }); + } + } + }, + + updateStoryArcDecision: async function (id, action, targetElement) { + var numericId = Number(id); + if (!Number.isFinite(numericId) || ["select", "skip"].indexOf(action) === -1) { + return; + } + + var proposedStoryArcId = null; + if (action === "select" && targetElement && targetElement.value) { + proposedStoryArcId = Number(targetElement.value); + if (!Number.isFinite(proposedStoryArcId)) { + proposedStoryArcId = null; + } + } + + try { + var response = await fetch( + "/api/v1/import/" + this.jobId + "/story-arcs/" + numericId + "/decision", + { + method: "PUT", + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": readCsrfTokenFromBody(), + }, + body: JSON.stringify({ + action: action, + proposed_story_arc_id: proposedStoryArcId, + }), + }, + ); + + if (!response.ok) { + var error = await response + .json() + .catch(function () { + return {}; + }); + throw new Error(error.detail || "Failed to update story arc decision."); + } + + await this.refreshReviewSummary(); + await this.refreshSeriesReview(); + } catch (err) { + if (typeof showToast === "function") { + showToast({ + message: + err && err.message ? err.message : "Failed to update story arc decision.", + level: "error", + }); + } + } + }, + + confirmStoryArcPolicy: async function (id, formElement) { + var numericId = Number(id); + if (!Number.isFinite(numericId) || !formElement) { + return; + } + + var field = function (name) { + return formElement.querySelector("[name='" + name + "']"); + }; + var checked = function (name) { + var element = field(name); + return Boolean(element && element.checked); + }; + var materialize = checked("materialize_filesystem"); + var modeElement = field("mode"); + var mode = materialize && modeElement ? modeElement.value : "logical"; + var rootElement = field("target_library_root_id"); + var rootId = materialize && rootElement ? Number(rootElement.value) : null; + if (!Number.isFinite(rootId)) { + rootId = null; + } + if (materialize && rootId === null) { + var rootTrigger = formElement.querySelector( + "[data-testid^='import-story-arc-policy-root-'] [data-dropdown-select-trigger]", + ); + if (rootTrigger) { + rootTrigger.focus(); + } + if (typeof showToast === "function") { + showToast({ + message: "Choose an approved library root before confirming this policy.", + level: "warning", + }); + } + return; + } + var destinationElement = field("destination_root"); + var symlinkElement = field("symlink_style"); + var digestElement = field("expected_policy_digest"); + var folderElement = field("folder_template"); + var fileElement = field("file_template"); + var monitored = checked("monitored"); + + var payload = { + confirm_policy: checked("confirm_policy"), + expected_policy_digest: digestElement ? digestElement.value : "", + materialize_filesystem: materialize, + monitored: monitored, + search_missing: monitored, + include_upcoming: monitored, + placement_policy: { + mode: mode, + target_library_root_id: materialize ? rootId : null, + destination_root: + materialize && destinationElement ? destinationElement.value : null, + folder_template: folderElement ? folderElement.value : "{StoryArc}", + file_template: + fileElement + ? fileElement.value + : "{ReadingOrder:03d} - {Series} {IssueNumber}", + symlink_style: + materialize && mode === "symlink" && symlinkElement + ? symlinkElement.value + : null, + synchronize: materialize && checked("synchronize"), + }, + }; + + try { + var response = await fetch( + "/api/v1/import/" + + this.jobId + + "/story-arcs/" + + numericId + + "/policy-confirmation", + { + method: "PUT", + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": readCsrfTokenFromBody(), + }, + body: JSON.stringify(payload), }, ); - if (!response.ok) { var error = await response .json() .catch(function () { return {}; }); - throw new Error(error.detail || "Failed to update series selection."); + var detail = error.detail; + if (Array.isArray(detail)) { + detail = detail + .map(function (item) { + return item && item.msg ? item.msg : "Invalid policy field"; + }) + .join("; "); + } + throw new Error(detail || "Failed to confirm story arc policy."); } - await this.refreshReviewSummary(); await this.refreshSeriesReview(); + if (typeof showToast === "function") { + showToast({ message: "Story arc policy confirmed.", level: "success" }); + } } catch (err) { - await this.refreshSeriesReviewQuietly(); if (typeof showToast === "function") { showToast({ message: - err && err.message ? err.message : "Failed to update series selection.", + err && err.message ? err.message : "Failed to confirm story arc policy.", level: "error", }); } @@ -5962,16 +7923,15 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { } }, - selectAllImportable: async function () { - try { - var seriesResponse = await fetch("/api/v1/import/" + this.jobId + "/series/selection-bulk", { + setAllImportableSelection: async function (includeInImport) { + var seriesResponse = await fetch("/api/v1/import/" + this.jobId + "/series/selection-bulk", { method: "POST", headers: { "Content-Type": "application/json", "X-CSRF-Token": readCsrfTokenFromBody(), }, body: JSON.stringify({ - include_in_import: true, + include_in_import: includeInImport, imported_series_ids: [], }), }); @@ -5982,7 +7942,12 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { .catch(function () { return {}; }); - throw new Error(seriesError.detail || "Failed to select matched series."); + throw new Error( + seriesError.detail || + (includeInImport + ? "Failed to select matched series." + : "Failed to clear series selection."), + ); } var fileResponse = await fetch("/api/v1/import/" + this.jobId + "/files/selection-bulk", { @@ -5992,7 +7957,7 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { "X-CSRF-Token": readCsrfTokenFromBody(), }, body: JSON.stringify({ - include_in_import: true, + include_in_import: includeInImport, }), }); @@ -6002,9 +7967,18 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { .catch(function () { return {}; }); - throw new Error(fileError.detail || "Failed to select in-library files."); + throw new Error( + fileError.detail || + (includeInImport + ? "Failed to select in-library files." + : "Failed to clear in-library file selection."), + ); } + }, + selectAllImportable: async function () { + try { + await this.setAllImportableSelection(true); await this.refreshReviewSummary(); await this.refreshSeriesReview(); } catch (err) { @@ -6021,47 +7995,7 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { deselectAllImportable: async function () { try { - var seriesResponse = await fetch("/api/v1/import/" + this.jobId + "/series/selection-bulk", { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-CSRF-Token": readCsrfTokenFromBody(), - }, - body: JSON.stringify({ - include_in_import: false, - imported_series_ids: [], - }), - }); - - if (!seriesResponse.ok) { - var seriesError = await seriesResponse - .json() - .catch(function () { - return {}; - }); - throw new Error(seriesError.detail || "Failed to clear series selection."); - } - - var fileResponse = await fetch("/api/v1/import/" + this.jobId + "/files/selection-bulk", { - method: "POST", - headers: { - "Content-Type": "application/json", - "X-CSRF-Token": readCsrfTokenFromBody(), - }, - body: JSON.stringify({ - include_in_import: false, - }), - }); - - if (!fileResponse.ok) { - var fileError = await fileResponse - .json() - .catch(function () { - return {}; - }); - throw new Error(fileError.detail || "Failed to clear in-library file selection."); - } - + await this.setAllImportableSelection(false); await this.refreshReviewSummary(); await this.refreshSeriesReview(); } catch (err) { @@ -6246,19 +8180,6 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { } }, - cancelImport: async function () { - this.cancelling = true; - try { - await fetch("/api/v1/import/" + this.jobId, { - method: "DELETE", - headers: { "X-CSRF-Token": readCsrfTokenFromBody() }, - }); - purgeImportClientState(this.jobId); - } finally { - window.location.replace("/import"); - } - }, - syncSelectionUi: function () { this.syncSelectionSummaryUi(); }, @@ -6267,6 +8188,12 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { if (this.totalSelectionCount() === 0) { return; } + if (!this.hasRequiredPreferredRoot()) { + this.confirmError = + "Choose a preferred managed destination for future acquisitions before importing this split series."; + this.syncSelectionSummaryUi(); + return; + } this.confirming = true; this.confirmError = ""; @@ -6280,6 +8207,9 @@ function importReviewData(configOrDefaultRootId, maybeJobId) { }, body: JSON.stringify({ series_ids: [], + story_arc_ids: [], + story_arc_decisions: [], + target_library_root_id: this.preferredRootId, }), }); @@ -6313,8 +8243,40 @@ function importResultsData(config) { showFailedFiles: cfg.failedFilesCount > 0, retrying: false, rollingBack: false, + archiving: false, safetyRetryingFileId: null, + cleanupRunningAction: "", + cleanupError: "", + cleanLibraryTargetRootId: cfg.defaultCleanLibraryRootId || "", + cleanLibraryRootPolicies: Array.isArray(cfg.cleanLibraryRootPolicies) + ? cfg.cleanLibraryRootPolicies + : [], + cleanLibraryJobId: + cfg.activeCleanLibraryJob && cfg.activeCleanLibraryJob.id + ? Number(cfg.activeCleanLibraryJob.id) + : null, + cleanLibraryStatus: + cfg.activeCleanLibraryJob && cfg.activeCleanLibraryJob.status + ? String(cfg.activeCleanLibraryJob.status).toLowerCase() + : "", + cleanLibraryProgress: + cfg.activeCleanLibraryJob && cfg.activeCleanLibraryJob.progress_snapshot + ? Math.max( + 0, + Math.min(100, Number(cfg.activeCleanLibraryJob.progress_snapshot.progress) || 0) + ) + : 0, + cleanLibraryProgressMessage: + cfg.activeCleanLibraryJob && cfg.activeCleanLibraryJob.progress_snapshot + ? String(cfg.activeCleanLibraryJob.progress_snapshot.message || "") + : "", + cleanLibraryProgressPollTimer: null, + cleanLibraryRunning: false, + cleanLibraryError: "", + cleanLibraryReturnFocus: null, retryError: "", + refreshUrl: cfg.refreshUrl || "/import/" + (cfg.jobId || "") + "/results-partial", + refreshTarget: cfg.refreshTarget || "[data-testid='import-collection-results']", toggleFailedSeries: function () { this.showFailedSeries = !this.showFailedSeries; @@ -6324,6 +8286,202 @@ function importResultsData(config) { this.showFailedFiles = !this.showFailedFiles; }, + initializeCleanLibraryModal: function () { + this.cleanLibraryReturnFocus = document.activeElement; + var self = this; + this.$nextTick(function () { + window.requestAnimationFrame(function () { + var target = self.$refs.cleanLibraryInitialFocus || self.$refs.cleanLibraryDialog; + if (target && typeof target.focus === "function") { + target.focus({ preventScroll: true }); + } + }); + }); + if (this.cleanLibraryJobId) { + this.refreshCleanLibraryProgress(); + } + }, + + closeCleanLibraryModal: function () { + this.clearCleanLibraryProgressPoll(); + var returnFocus = this.cleanLibraryReturnFocus; + this.cleanLibraryReturnFocus = null; + this.open = false; + window.setTimeout(function () { + closeImportCvSearchModal(); + if ( + returnFocus && + document.contains(returnFocus) && + typeof returnFocus.focus === "function" + ) { + returnFocus.focus({ preventScroll: true }); + } + }, 0); + }, + + selectedCleanLibraryPolicy: function () { + var targetRootId = Number(this.cleanLibraryTargetRootId); + return ( + this.cleanLibraryRootPolicies.find(function (root) { + return Number(root && root.id) === targetRootId; + }) || {} + ); + }, + + cleanLibraryPolicyLabel: function (key) { + var policy = this.selectedCleanLibraryPolicy(); + var enabled = policy[key] === true; + var labels = { + rename_on_import: enabled + ? "Use the selected root's naming templates" + : "Keep current file and folder names", + normalize_to_cbz: enabled + ? "Normalize supported archives to CBZ" + : "Keep each supported archive format", + update_comicinfo: enabled + ? "Write matched metadata to ComicInfo.xml" + : "Keep embedded ComicInfo.xml unchanged", + skip_existing: enabled + ? "Skip issues already in the destination" + : "Allow another managed copy when valid", + }; + return labels[key] || "Use current import settings"; + }, + + clearCleanLibraryProgressPoll: function () { + if (this.cleanLibraryProgressPollTimer) { + window.clearTimeout(this.cleanLibraryProgressPollTimer); + this.cleanLibraryProgressPollTimer = null; + } + }, + + cleanLibraryIsTerminal: function () { + return ["completed", "failed", "cancelled", "rolled_back"].indexOf( + this.cleanLibraryStatus + ) !== -1; + }, + + cleanLibraryStatusLabel: function () { + var labels = { + importing: "Building", + pausing: "Pausing", + paused: "Paused", + stalled: "Needs attention", + cancelling: "Cancelling", + rolling_back: "Rolling back", + completed: "Complete", + failed: "Failed", + cancelled: "Cancelled", + rolled_back: "Rolled back", + }; + return labels[this.cleanLibraryStatus] || "Queued"; + }, + + cleanLibraryJobUrl: function () { + var step = this.cleanLibraryStatus === "completed" ? 5 : 4; + return ( + "/import?tab=collection&resume_job_id=" + + encodeURIComponent(this.cleanLibraryJobId || "") + + "&resume_step=" + + step + ); + }, + + scheduleCleanLibraryProgressPoll: function () { + var self = this; + self.clearCleanLibraryProgressPoll(); + if (!self.cleanLibraryJobId || self.cleanLibraryIsTerminal() || !self.open) { + return; + } + self.cleanLibraryProgressPollTimer = window.setTimeout(function () { + self.cleanLibraryProgressPollTimer = null; + self.refreshCleanLibraryProgress(); + }, 1000); + }, + + refreshCleanLibraryProgress: async function () { + if (!this.cleanLibraryJobId) { + return; + } + try { + var response = await fetch( + "/import/" + this.cleanLibraryJobId + "/progress-state", + { headers: { Accept: "application/json" } } + ); + if (!response.ok) { + throw new Error("Could not refresh clean-library progress."); + } + var progress = await response.json(); + this.cleanLibraryStatus = String(progress.status || this.cleanLibraryStatus).toLowerCase(); + var nextProgress = Number(progress.progress); + if (Number.isFinite(nextProgress)) { + this.cleanLibraryProgress = Math.max( + this.cleanLibraryProgress, + Math.max(0, Math.min(100, Math.round(nextProgress))) + ); + } + if (this.cleanLibraryStatus === "completed") { + this.cleanLibraryProgress = 100; + } + this.cleanLibraryProgressMessage = String( + progress.message || this.cleanLibraryProgressMessage || "Building the clean library..." + ); + if (this.cleanLibraryStatus === "failed" && progress.error_message) { + this.cleanLibraryError = String(progress.error_message); + } + } catch (err) { + this.cleanLibraryError = + err && err.message ? err.message : "Could not refresh clean-library progress."; + } finally { + this.scheduleCleanLibraryProgressPoll(); + } + }, + + trapCleanLibraryModalFocus: function (event) { + if (!event || !this.open || !this.$refs.cleanLibraryDialog) { + return; + } + var dialog = this.$refs.cleanLibraryDialog; + var panel = event.target && event.target.closest + ? event.target.closest("[data-dropdown-select-panel]") + : null; + var focusable = Array.prototype.filter.call( + dialog.querySelectorAll( + 'a[href], button:not([disabled]), input:not([disabled]):not([type="hidden"]), ' + + 'select:not([disabled]), textarea:not([disabled]), [tabindex]:not([tabindex="-1"])' + ), + function (element) { + return element.getClientRects().length > 0; + } + ); + if (!focusable.length) { + event.preventDefault(); + dialog.focus({ preventScroll: true }); + return; + } + if (panel) { + event.preventDefault(); + var labelledBy = panel.getAttribute("aria-labelledby"); + var trigger = labelledBy ? dialog.querySelector("#" + CSS.escape(labelledBy)) : null; + var triggerIndex = trigger ? focusable.indexOf(trigger) : -1; + var destination = event.shiftKey + ? trigger || focusable[0] + : focusable[Math.min(focusable.length - 1, triggerIndex + 1)]; + destination.focus({ preventScroll: true }); + return; + } + var active = document.activeElement; + var first = focusable[0]; + var last = focusable[focusable.length - 1]; + if (event.shiftKey && (active === first || !dialog.contains(active))) { + event.preventDefault(); + last.focus({ preventScroll: true }); + } else if (!event.shiftKey && (active === last || !dialog.contains(active))) { + event.preventDefault(); + first.focus({ preventScroll: true }); + } + }, + retryFailed: async function () { if (!this.jobId || this.retrying) { return; @@ -6347,6 +8505,64 @@ function importResultsData(config) { throw new Error(error.detail || "Failed to retry"); } + var payload = await response.json(); + if (Math.max(0, Number(payload.retrying_count) || 0) === 0) { + var blockedMessage = + "No failed files passed source revalidation. Review the updated failure details before retrying."; + this.retryError = blockedMessage; + if (typeof showToast === "function") { + showToast({ message: blockedMessage, level: "warning" }); + } + return; + } + + dispatchImportWizardAdvance({ + step: 4, + jobId: this.jobId, + jobStatus: "importing", + }); + } catch (err) { + var message = + err && err.message ? err.message : "Retry failed. Please try again."; + this.retryError = message; + if (typeof showToast === "function") { + showToast({ message: message, level: "error" }); + } + } finally { + this.retrying = false; + } + }, + + allowSafetyOnceAndRetry: async function (fileId) { + if (!this.jobId || this.safetyRetryingFileId) { + return; + } + + this.safetyRetryingFileId = fileId; + this.retryError = ""; + + try { + var response = await fetch( + "/api/v1/import/" + + this.jobId + + "/files/" + + fileId + + "/safety/allow-once-and-retry", + { + method: "POST", + headers: { "X-CSRF-Token": readCsrfTokenFromBody() }, + } + ); + + if (!response.ok) { + var error = await response + .json() + .catch(function () { + return {}; + }); + throw new Error(error.detail || "Failed to approve safety retry"); + } + dispatchImportWizardAdvance({ step: 4, jobId: this.jobId, @@ -6354,60 +8570,216 @@ function importResultsData(config) { }); } catch (err) { var message = - err && err.message ? err.message : "Retry failed. Please try again."; - this.retryError = message; + err && err.message ? err.message : "Safety retry failed. Please try again."; + this.retryError = message; + if (typeof showToast === "function") { + showToast({ message: message, level: "error" }); + } + } finally { + this.safetyRetryingFileId = null; + } + }, + + refreshResults: function () { + var url = this.refreshUrl; + var target = this.refreshTarget; + if (window.htmx && document.querySelector(target)) { + window.htmx.ajax("GET", url, { target: target, swap: "outerHTML" }); + return; + } + window.location.reload(); + }, + + applyCleanup: async function (action, label) { + if (!this.jobId || this.cleanupRunningAction) { + return; + } + this.cleanupRunningAction = action; + this.cleanupError = ""; + try { + var previewResponse = await fetch( + "/api/v1/import/" + this.jobId + "/cleanup/" + action + "/preview" + ); + var preview = await previewResponse.json().catch(function () { + return {}; + }); + if (!previewResponse.ok) { + throw new Error(preview.detail || "Could not preview this cleanup action."); + } + var unit = + preview.item_unit === "group" + ? "conflict group" + : preview.item_unit === "follow-up item" + ? "follow-up item" + : "file"; + var count = Math.max(0, Number(preview.affected_count) || 0); + var examples = Array.isArray(preview.examples) ? preview.examples.slice(0, 3) : []; + var message = + label + + " will update " + + count.toLocaleString() + + " " + + unit + + (count === 1 ? "" : "s") + + ". Source files will remain unchanged."; + if (action === "recheck_deferred_files") { + message = + "Check " + count.toLocaleString() + " deferred file paths and stale series records in the background. " + + "Pullbox will reconcile repeated records and import only proven matches using " + + "this import's original file settings. Uncertain matches stay in Follow-up. " + + "Source files will remain unchanged."; + } + if (examples.length) { + message += " Examples: " + examples.join(", ") + "."; + } + var confirmed = await pbConfirm({ + title: label + "?", + message: message, + confirmText: action === "recheck_deferred_files" ? "Recheck files" : "Apply cleanup", + destructive: false, + }); + if (!confirmed) { + return; + } + + var response = await fetch( + "/api/v1/import/" + this.jobId + "/cleanup/" + action, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": readCsrfTokenFromBody(), + }, + body: JSON.stringify({ + preview_token: preview.preview_token, + confirmation: "APPLY CLEANUP", + }), + } + ); + var result = await response.json().catch(function () { + return {}; + }); + if (!response.ok) { + throw new Error(result.detail || "The cleanup action could not be applied."); + } + if (result.requires_import_retry) { + dispatchImportWizardAdvance({ + step: 4, + jobId: this.jobId, + jobStatus: "importing", + }); + return; + } + if (typeof showToast === "function") { + showToast({ message: label + " completed.", level: "success" }); + } + this.refreshResults(); + } catch (err) { + var message = + err && err.message ? err.message : "The cleanup action could not be applied."; + this.cleanupError = message; if (typeof showToast === "function") { showToast({ message: message, level: "error" }); } } finally { - this.retrying = false; + this.cleanupRunningAction = ""; } }, - allowSafetyOnceAndRetry: async function (fileId) { - if (!this.jobId || this.safetyRetryingFileId) { + buildCleanLibrary: async function () { + if (!this.jobId || !this.cleanLibraryTargetRootId || this.cleanLibraryRunning) { return; } - - this.safetyRetryingFileId = fileId; - this.retryError = ""; - + this.cleanLibraryRunning = true; + this.cleanLibraryError = ""; try { - var response = await fetch( + var targetRootId = Number(this.cleanLibraryTargetRootId); + var previewResponse = await fetch( "/api/v1/import/" + this.jobId + - "/files/" + - fileId + - "/safety/allow-once-and-retry", + "/clean-library/preview?target_root_id=" + + encodeURIComponent(targetRootId) + ); + var preview = await previewResponse.json().catch(function () { + return {}; + }); + if (!previewResponse.ok) { + throw new Error(preview.detail || "Could not preview the clean library."); + } + var response = await fetch( + "/api/v1/import/" + this.jobId + "/clean-library", { method: "POST", - headers: { "X-CSRF-Token": readCsrfTokenFromBody() }, + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": readCsrfTokenFromBody(), + }, + body: JSON.stringify({ + target_root_id: targetRootId, + preview_token: preview.preview_token, + confirmation: "BUILD CLEAN LIBRARY", + }), } ); - + var result = await response.json().catch(function () { + return {}; + }); if (!response.ok) { - var error = await response - .json() - .catch(function () { - return {}; - }); - throw new Error(error.detail || "Failed to approve safety retry"); + throw new Error(result.detail || "Could not start the clean-library build."); + } + this.cleanLibraryJobId = Number(result.job_id); + this.cleanLibraryStatus = "importing"; + this.cleanLibraryProgress = 0; + this.cleanLibraryProgressMessage = "Preparing the clean-library work plan..."; + await this.refreshCleanLibraryProgress(); + } catch (err) { + var message = + err && err.message ? err.message : "Could not start the clean-library build."; + this.cleanLibraryError = message; + if (typeof showToast === "function") { + showToast({ message: message, level: "error" }); } + } finally { + this.cleanLibraryRunning = false; + } + }, - dispatchImportWizardAdvance({ - step: 4, - jobId: this.jobId, - jobStatus: "importing", + archiveImport: async function () { + if (!this.jobId || this.archiving) { + return; + } + var confirmed = await pbConfirm({ + title: "Archive import results?", + message: + "This hides the finished import from current history without deleting its records, logs, or recovery evidence.", + confirmText: "Archive results", + destructive: false, + }); + if (!confirmed) { + return; + } + this.archiving = true; + try { + var response = await fetch("/api/v1/import/" + this.jobId + "/archive", { + method: "POST", + headers: { "X-CSRF-Token": readCsrfTokenFromBody() }, }); + if (!response.ok) { + var error = await response.json().catch(function () { + return {}; + }); + throw new Error(error.detail || "Failed to archive import results."); + } + window.location.assign("/import?tab=history"); } catch (err) { var message = - err && err.message ? err.message : "Safety retry failed. Please try again."; - this.retryError = message; + err && err.message ? err.message : "Failed to archive import results."; + this.cleanupError = message; if (typeof showToast === "function") { showToast({ message: message, level: "error" }); } } finally { - this.safetyRetryingFileId = null; + this.archiving = false; } }, @@ -8781,6 +11153,8 @@ function libraryBrowserPage(config) { linked_file_count: 0, tracked_file_count: 0, tracked_series_count: 0, + managed_file_count: 0, + referenced_file_count: 0, has_linked_issue: false, issue_status_after_delete: null, issue_status_reason: null, @@ -8849,6 +11223,9 @@ function libraryBrowserPage(config) { }, deleteSubmitLabel: function () { + if (this.deleteReferencedFileCount() > 0 && this.deleteManagedFileCount() === 0) { + return "Remove from Pullbox"; + } if (this.deleteMode() === "series") return "Delete Series"; if (this.deleteMode() === "folder") return "Delete Folder"; return "Delete File"; @@ -8871,6 +11248,9 @@ function libraryBrowserPage(config) { (this.modalEntry && this.modalEntry.name) || "this series"; var base = "This folder is associated with the series " + title + " in Pullbox."; + if (this.deleteReferencedFileCount() > 0) { + return base + " Referenced files will stay on disk and be detached from Pullbox. Any folder containing them will also stay in place."; + } if (this.deleteUsesTrash()) { return base + " Deleting it will move the folder into the configured trash folder and remove the series and all issue records from the database."; } @@ -8878,12 +11258,18 @@ function libraryBrowserPage(config) { }, deleteSeriesDispositionLabel: function () { + if (this.deleteReferencedFileCount() > 0) { + return "Detach referenced files; remove managed files only"; + } return this.deleteUsesTrash() ? "Move folder to trash and delete series" : "Permanent folder delete and series removal"; }, deleteFolderMessage: function () { + if (this.deleteReferencedFileCount() > 0) { + return "Referenced files and any folder containing them will stay in place. Pullbox will detach their records and remove only managed files in this folder."; + } if (this.deleteUsesTrash()) { return "This will move the selected folder and everything inside it into the configured trash folder."; } @@ -8891,6 +11277,9 @@ function libraryBrowserPage(config) { }, deleteFileMessage: function () { + if (this.deleteReferencedFileCount() > 0) { + return "This referenced file will stay exactly where it is. Pullbox will remove only its tracked record and update the linked issue state."; + } var disposition = this.deleteUsesTrash() ? "move the selected file into the configured trash folder" : "permanently delete the selected file"; @@ -8917,6 +11306,9 @@ function libraryBrowserPage(config) { }, deleteDispositionLabel: function () { + if (this.deleteReferencedFileCount() > 0 && this.deleteManagedFileCount() === 0) { + return "Detach from Pullbox"; + } return this.deleteUsesTrash() ? "Move to trash" : "Permanent delete"; }, @@ -8987,6 +11379,24 @@ function libraryBrowserPage(config) { ); }, + deleteManagedFileCount: function () { + return ( + (this.modalEntry && + this.modalEntry.deleteContext && + this.modalEntry.deleteContext.managed_file_count) || + 0 + ); + }, + + deleteReferencedFileCount: function () { + return ( + (this.modalEntry && + this.modalEntry.deleteContext && + this.modalEntry.deleteContext.referenced_file_count) || + 0 + ); + }, + deleteTrackedSeriesCount: function () { return ( (this.modalEntry && @@ -10481,13 +12891,15 @@ function utilitiesPermissionsPage(config) { }, syncFooterDock: function () { + var detail = { + mode: this.runModeLabel(), + scope: this.scopeLabel(), + targets: this.targetsLabel(), + }; + window.pullboxUtilitiesPermissionsFooterState = detail; window.dispatchEvent( new CustomEvent("utilities:permissions-footer", { - detail: { - mode: this.runModeLabel(), - scope: this.scopeLabel(), - targets: this.targetsLabel(), - }, + detail: detail, }) ); }, @@ -10663,6 +13075,8 @@ function appShell() { activityAttentionCount: 0, activitySource: null, activityPollTimer: null, + activityRefreshTimer: null, + activityStreamOpen: false, activityRefreshing: false, get collapsed() { @@ -10710,6 +13124,7 @@ function appShell() { } this.disconnectActivityStream(); this.clearActivityTimer(); + this.clearActivityRefreshTimer(); }, clearActivityTimer: function () { @@ -10728,6 +13143,22 @@ function appShell() { }, delayMs || 3000); }, + clearActivityRefreshTimer: function () { + if (this.activityRefreshTimer) { + window.clearTimeout(this.activityRefreshTimer); + this.activityRefreshTimer = null; + } + }, + + scheduleActivityRefresh: function (delayMs) { + var self = this; + self.clearActivityRefreshTimer(); + self.activityRefreshTimer = window.setTimeout(function () { + self.activityRefreshTimer = null; + self.refreshActivity(); + }, delayMs || 500); + }, + bootstrapActivity: function () { this.connectActivityStream(); this.refreshActivity(); @@ -10769,7 +13200,9 @@ function appShell() { }) .finally(function () { self.activityRefreshing = false; - self.scheduleActivityPoll(3000); + if (!self.activityStreamOpen) { + self.scheduleActivityPoll(3000); + } if (!self.activitySource) { self.connectActivityStream(); } @@ -10785,9 +13218,16 @@ function appShell() { self.activitySource = source; var refreshFromEvent = function () { if (self.activitySource === source) { - self.refreshActivity(); + self.scheduleActivityRefresh(500); } }; + source.onopen = function () { + if (self.activitySource !== source) { + return; + } + self.activityStreamOpen = true; + self.clearActivityTimer(); + }; source.addEventListener("ready", refreshFromEvent); source.addEventListener("progress", refreshFromEvent); source.onmessage = refreshFromEvent; @@ -10796,6 +13236,7 @@ function appShell() { return; } self.disconnectActivityStream(); + self.scheduleActivityPoll(3000); }; }, @@ -10809,6 +13250,7 @@ function appShell() { // Closing a stale activity stream is best-effort. } this.activitySource = null; + this.activityStreamOpen = false; }, acknowledgeActivity: function (operationId) { @@ -11779,33 +14221,36 @@ function downloadsPage(config) { function dropdownSelectData(config) { var cfg = config || {}; - var rawOptions = Array.isArray(cfg.options) ? cfg.options : []; - var normalizedOptions = rawOptions.map(function (option) { - if (Array.isArray(option)) { + function normalizeOptions(options) { + return (Array.isArray(options) ? options : []).map(function (option) { + if (Array.isArray(option)) { + return { + value: option[0] == null ? "" : String(option[0]), + label: + option[1] == null + ? option[0] == null + ? "" + : String(option[0]) + : String(option[1]), + disabled: false, + }; + } + + var item = option || {}; return { - value: option[0] == null ? "" : String(option[0]), + value: item.value == null ? "" : String(item.value), label: - option[1] == null - ? option[0] == null + item.label == null + ? item.value == null ? "" - : String(option[0]) - : String(option[1]), - disabled: false, + : String(item.value) + : String(item.label), + disabled: Boolean(item.disabled), }; - } + }); + } - var item = option || {}; - return { - value: item.value == null ? "" : String(item.value), - label: - item.label == null - ? item.value == null - ? "" - : String(item.value) - : String(item.label), - disabled: Boolean(item.disabled), - }; - }); + var normalizedOptions = normalizeOptions(cfg.options); function findIndexByValue(value) { var normalizedValue = value == null ? "" : String(value); @@ -11912,6 +14357,44 @@ function dropdownSelectData(config) { this.applyFitWidth(); }, + syncExternalOptions: function (nextOptions) { + var normalizedNext = normalizeOptions(nextOptions); + var unchanged = + normalizedNext.length === normalizedOptions.length && + normalizedNext.every(function (option, index) { + var current = normalizedOptions[index]; + return !!( + current && + current.value === option.value && + current.label === option.label && + current.disabled === option.disabled + ); + }); + if (unchanged) { + return; + } + + normalizedOptions = normalizedNext; + this.options = normalizedNext; + this.syncFromValue(); + this.syncInput(); + this.applyFitWidth(); + if (this.open) { + this.schedulePanelPositionUpdate(); + } + }, + + syncExternalDisabled: function (nextDisabled) { + var disabled = Boolean(nextDisabled); + if (disabled === this.disabled) { + return; + } + this.disabled = disabled; + if (disabled && this.open) { + this.close(false); + } + }, + runChangeExpression: function () { if (!this.changeExpression || !this.$el) { return; @@ -12228,7 +14711,10 @@ function dropdownSelectData(config) { requestAnimationFrame(function () { self.syncPanelControlVars(); self.updatePanelPosition(); - self.focusOption(self.activeIndex); + // Apply data-ready before focusing: hidden panels reject focus. + self.$nextTick(function () { + if (self.open) self.focusOption(self.activeIndex); + }); }); }); }, @@ -12413,6 +14899,9 @@ function dropdownSelectData(config) { this.currentLabel = option.label; this.syncInput(); this.close(false); + if (this.$refs && this.$refs.trigger) { + this.$refs.trigger.focus(); + } this.runChangeExpression(); @@ -12432,12 +14921,6 @@ function dropdownSelectData(config) { ); } - var self = this; - this.$nextTick(function () { - if (self.$refs && self.$refs.trigger) { - self.$refs.trigger.focus(); - } - }); }, selectActive: function () { @@ -13728,6 +16211,54 @@ function importHistoryPage(config) { } }, + restoreJob: async function (jobId) { + try { + var response = await fetch("/api/v1/import/" + jobId + "/restore", { + method: "POST", + headers: { "X-CSRF-Token": this.csrfToken() }, + }); + if (!response.ok) { + var error = await response.json().catch(function () { + return {}; + }); + throw new Error(error.detail || "Failed to restore import history."); + } + this.removeJobRow(jobId); + this.dispatchToast("Import restored to current history.", "success"); + } catch (error) { + this.dispatchToast(error.message || "Failed to restore import history.", "error"); + } + }, + + archiveJob: async function (jobId) { + var confirmed = await pbConfirm({ + title: "Archive import results?", + message: + "This hides the finished import without deleting its records, logs, or recovery evidence.", + confirmText: "Archive results", + destructive: false, + }); + if (!confirmed) { + return; + } + try { + var response = await fetch("/api/v1/import/" + jobId + "/archive", { + method: "POST", + headers: { "X-CSRF-Token": this.csrfToken() }, + }); + if (!response.ok) { + var error = await response.json().catch(function () { + return {}; + }); + throw new Error(error.detail || "Failed to archive import history."); + } + this.removeJobRow(jobId); + this.dispatchToast("Import moved to archived history.", "success"); + } catch (error) { + this.dispatchToast(error.message || "Failed to archive import history.", "error"); + } + }, + rollbackJob: async function (jobId) { var confirmed = await pbConfirm({ title: "Rollback import", @@ -13876,8 +16407,18 @@ function importHistoryPage(config) { headers: { "X-CSRF-Token": self.csrfToken() }, }) .then(function (response) { + if (response.status === 202) { + return response.json().then(function (data) { + return { + rollbackPending: true, + message: + data.message || + "Rollback is still finishing. This import remains in history.", + }; + }); + } if (response.ok || response.status === 204) { - return null; + return { rollbackPending: false }; } return response .json() @@ -13888,11 +16429,16 @@ function importHistoryPage(config) { throw new Error(data.detail || "Unable to delete this import job right now."); }); }) - .then(function () { + .then(function (result) { var deletedJobId = self.deleteJobId; - self.dispatchToast("Import job deleted.", "success"); self.deleteJobId = null; self.deleting = false; + if (result && result.rollbackPending) { + self.dispatchToast(result.message, "info"); + self.refreshResults(buildHistoryPath()); + return; + } + self.dispatchToast("Import job deleted.", "success"); self.removeJobRow(deletedJobId); self.syncClearHistoryButtonVisibility(); self.refreshResults(buildHistoryPath()); @@ -14439,7 +16985,7 @@ function orphanedSeriesPage(config) { searchCv: function (importedSeriesId, query) { if (typeof htmx === "undefined") { - window.location.assign("/import?tab=unmatched"); + window.location.assign("/import?tab=follow-up"); return; } @@ -14452,7 +16998,7 @@ function orphanedSeriesPage(config) { openRecovery: function (importedSeriesId) { if (typeof htmx === "undefined") { - window.location.assign("/import?tab=unmatched"); + window.location.assign("/import?tab=follow-up"); return; } @@ -14552,7 +17098,7 @@ function orphanedSeriesPage(config) { self.dispatchToast( "Identified " + ((data && data.cv_title) || "series") + - ". Finish the file recovery to remove it from Unmatched.", + ". Finish the file recovery to remove it from Follow-up.", "success" ); self.refreshResults(); @@ -15092,7 +17638,7 @@ function orphanedRecoveryModal(config) { reload: function () { if (typeof htmx === "undefined") { - window.location.assign("/import?tab=unmatched"); + window.location.assign("/import?tab=follow-up"); return; } htmx.ajax("GET", "/import/orphaned/" + cfg.importedSeriesId + "/recovery", { @@ -16700,10 +19246,14 @@ function seriesDetailPage(config) { statusSaving: false, refreshing: false, searching: false, + metadataSyncing: false, + metadataSyncTimer: null, issueSearchState: {}, init: function () { this.monitored = !!cfg.monitored; + this.metadataSyncing = !!cfg.metadataSyncing; + this.startMetadataSyncPolling(); var self = this; var runNormalize = function () { @@ -16719,6 +19269,51 @@ function seriesDetailPage(config) { } }, + destroy: function () { + if (this.metadataSyncTimer !== null) { + clearTimeout(this.metadataSyncTimer); + this.metadataSyncTimer = null; + } + }, + + startMetadataSyncPolling: function () { + var self = this; + if (!self.metadataSyncing || self.metadataSyncTimer !== null) { + return; + } + self.metadataSyncTimer = window.setTimeout(function () { + self.metadataSyncTimer = null; + self.pollMetadataSyncState(); + }, 3000); + }, + + pollMetadataSyncState: function () { + var self = this; + if (!self.metadataSyncing || !cfg.updateUrl) { + return; + } + fetch(cfg.updateUrl, { + method: "GET", + headers: { Accept: "application/json" }, + cache: "no-store", + }) + .then(function (response) { + if (!response.ok) throw new Error("Failed to load metadata sync state"); + return response.json(); + }) + .then(function (data) { + if (data.issue_catalog_state !== "hydrating") { + self.metadataSyncing = false; + self.refreshIssuesPanel(); + return; + } + self.startMetadataSyncPolling(); + }) + .catch(function () { + self.startMetadataSyncPolling(); + }); + }, + csrfToken: function () { return cfg.csrfToken || readCsrfTokenFromBody(); }, @@ -16885,13 +19480,19 @@ function seriesDetailPage(config) { emitToast("Queued " + issue.label + " for review", "info"); } else if (data.status === "no_results") { emitToast("No results found for " + issue.label, "warning"); + } else if (data.status === "source_unavailable") { + var queueMessage = data.message || "Matches found, but downloads could not be queued."; + if (Array.isArray(data.notices) && data.notices.length) { + queueMessage += " " + data.notices.join(" "); + } + emitToast(queueMessage, "warning"); } else { var infoMessage = data.error && data.error.message ? data.error.message : typeof data.error === "string" ? data.error - : "Search completed"; + : data.message || "Search completed"; emitToast(infoMessage, "info"); } }) @@ -18389,6 +20990,24 @@ function prepareAlpineSwap(detail, target) { return true; } +function initializePreparedAlpineSwap(detail) { + if (!detail || !detail.xhr || !_htmxRequestsNeedingAlpineInit.has(detail.xhr)) { + return false; + } + + _htmxRequestsNeedingAlpineInit.delete(detail.xhr); + var target = resolveHtmxLiveTarget(detail.target); + if (!target || target.isConnected === false) { + return false; + } + + if (window.Alpine) { + Alpine.initTree(target); + } + seedSearchFieldStates(target); + return true; +} + function _purgeDetailHistoryRestoreEntry(pathname, search) { var normalizedPath = normalizePath(pathname || window.location.pathname); if (!_isDetailHistoryRestorePath(normalizedPath)) { @@ -18529,21 +21148,9 @@ document.body.addEventListener("htmx:beforeSwap", function (e) { }); document.addEventListener("htmx:afterRequest", function (e) { - var detail = e.detail || {}; - if (!detail.xhr || !_htmxRequestsNeedingAlpineInit.has(detail.xhr)) { - return; - } - - _htmxRequestsNeedingAlpineInit.delete(detail.xhr); - var target = resolveHtmxLiveTarget(detail.target); - if (!target || target.isConnected === false) { - return; - } - - if (window.Alpine) { - Alpine.initTree(target); - } - seedSearchFieldStates(target); + // afterSwap normally initializes the replacement. Keep this as a fallback + // for HTMX request paths that complete without dispatching afterSwap. + initializePreparedAlpineSwap(e.detail || {}); }); // After a shell content swap, update the header title from the full-page response. @@ -18586,6 +21193,17 @@ function _syncFooterDockFromResponse(responseText) { } document.addEventListener("htmx:afterSwap", function (e) { + initializePreparedAlpineSwap(e.detail || {}); + var swappedTarget = resolveHtmxLiveTarget(e.detail.target); + if ( + swappedTarget && + swappedTarget.id === "import-step-review-shell" && + pendingImportReviewViewportState + ) { + // Restore disclosure state in the swap task so a collapsed frame never paints. + restoreImportReviewExpansionState(pendingImportReviewViewportState, swappedTarget); + } + if (e.detail.target && e.detail.target.id === "content" && e.detail.xhr) { _startContentSwapEnter(); _syncFooterDockFromResponse(e.detail.xhr.responseText); @@ -19592,11 +22210,6 @@ document.addEventListener("htmx:afterSettle", function (e) { window.htmx.process(settledTarget); } - // Re-initialize Alpine components in the primary HTMX swap target. - if (window.Alpine && settledTarget) { - Alpine.initTree(settledTarget); - } - if ( settledTarget && (settledTarget.id === "import-step-review" || @@ -19628,6 +22241,15 @@ document.addEventListener("htmx:afterSettle", function (e) { } } + if ( + settledTarget && + settledTarget.id === "import-step-review-shell" && + pendingImportReviewViewportState + ) { + restoreImportReviewViewport(pendingImportReviewViewportState, settledTarget); + pendingImportReviewViewportState = null; + } + if (settledTarget) { seedSearchFieldStates(settledTarget); } diff --git a/src/pullbox/ui/static/js/story-arc-detail.js b/src/pullbox/ui/static/js/story-arc-detail.js new file mode 100644 index 00000000..3f5e8eda --- /dev/null +++ b/src/pullbox/ui/static/js/story-arc-detail.js @@ -0,0 +1,33 @@ +function storyArcDetail() { + return { + coverModalOpen: false, coverModalUrl: '', reordering: false, reorderError: '', + beginReorder(event) { + if (!event.detail.elt?.matches('[data-story-arc-move]')) return; + if (this.reordering) { event.preventDefault(); return; } + this.reordering = true; + this.reorderError = ''; + }, + reorderRequestFinished(event) { + if (!event.detail.elt?.matches('[data-story-arc-move]')) return; + if (event.detail.successful) return; + this.reordering = false; + this.reorderError = 'The new order could not be confirmed. Refresh the page to check the saved order before trying again.'; + }, + finishReorder(detail) { + this.reordering = false; + this.$nextTick(() => { + const row = this.$root.querySelector(`[data-membership-id="${Number(detail.membershipId)}"]`); + const direction = detail.direction === 'up' ? 'up' : 'down'; + const button = row?.querySelector(`[data-order-direction="${direction}"]`); + const target = button?.disabled ? row.querySelector('[data-order-direction]:not(:disabled)') : button; + target?.focus({ preventScroll: true }); + if (detail.pageChanged) row?.scrollIntoView({ block: 'nearest' }); + }); + }, + submitMove(button, direction) { + if (this.reordering || button.disabled) return; + button.form.elements.direction.value = direction; + button.form.requestSubmit(); + }, + }; +} diff --git a/src/pullbox/ui/static/js/story-arc-preview.js b/src/pullbox/ui/static/js/story-arc-preview.js new file mode 100644 index 00000000..9ebab321 --- /dev/null +++ b/src/pullbox/ui/static/js/story-arc-preview.js @@ -0,0 +1,189 @@ +/* The complete bounded preview stays local: paging never refetches Comic Vine + * or drops order/skip choices for members on another page. */ +function storyArcPreview() { + return { + members: [], page: 1, perPage: '25', ready: false, + libraryRootId: '', monitored: false, rootOptions: [], + title: '', description: '', coverUrl: '', coverFailed: false, + fingerprint: '', fileDefaultsFingerprint: '', fileSummary: '', fileExample: '', + error: '', submitError: '', notice: '', busy: '', controller: null, endpoint: '', reorderAnnouncement: '', + + init() { + this.endpoint = this.$el.dataset.previewUrl; + this.applySnapshot(JSON.parse(this.$el.querySelector('[data-preview-data]').textContent), false); + this.$watch('page', () => this.publish()); + this.$watch('perPage', () => { this.page = 1; this.publish(); }); + this.$watch('skippedCount', () => this.publish()); + this.$watch('libraryRootId', () => { this.submitError = ''; }); + this.$nextTick(() => this.publish()); + }, + destroy() { if (this.controller) this.controller.abort(); }, + get totalPages() { return Math.max(1, Math.ceil(this.members.length / Number(this.perPage))); }, + get visibleMembers() { + const start = (this.page - 1) * Number(this.perPage); + return this.members.slice(start, start + Number(this.perPage)); + }, + get skippedCount() { return this.members.filter(member => member.skipped).length; }, + publish() { + this.$dispatch('story-arc-preview-status', { + page: this.page, totalPages: this.totalPages, total: this.members.length, + skipped: this.skippedCount, ready: this.ready, + }); + }, + setPage(page) { + this.page = Math.max(1, Math.min(this.totalPages, Number(page))); + }, + renumber(members) { + this.members = members.map((member, index) => ({ ...member, order: index + 1 })); + }, + moveMember(providerId, direction) { + if (this.busy || ![-1, 1].includes(direction)) return; + const index = this.members.findIndex(member => member.provider_id === providerId); + const target = index + direction; + if (index < 0 || target < 0 || target >= this.members.length) return; + const previousFocus = document.activeElement; + // Replace the array in one pass so keyed rows never see duplicate IDs. + const reordered = [...this.members]; + [reordered[index], reordered[target]] = [reordered[target], reordered[index]]; + this.renumber(reordered); + const previousPage = this.page; + this.setPage(Math.floor(target / Number(this.perPage)) + 1); + const moved = this.members[target]; + this.reorderAnnouncement = `${moved.series_name} #${moved.issue_number} moved to position ${target + 1}.`; + this.$nextTick(() => { + // A newer keyboard/pointer focus choice takes precedence over restoring + // focus lost while Alpine moves or replaces the keyed row. + if (document.activeElement !== previousFocus && document.activeElement !== document.body) return; + const row = this.$root.querySelector(`[data-provider-issue-id="${providerId}"]`); + const button = row?.querySelector(`[data-order-direction="${direction < 0 ? 'up' : 'down'}"]`); + const focusTarget = button?.disabled ? row.querySelector('[data-order-direction]:not(:disabled)') : button; + focusTarget?.focus({ preventScroll: true }); + if (previousPage !== this.page) row?.scrollIntoView({ block: 'nearest' }); + }); + }, + applySnapshot(data, preserve = true) { + this.error = data.error; + this.ready = data.ready; + // A failed/incomplete response is not a replacement for a complete draft. + if (!data.ready && preserve && this.members.length) { this.publish(); return; } + const previous = new Map(this.members.map(member => [member.provider_id, member])); + let added = 0; + const incoming = new Set(data.members.map(member => member.provider_id)); + const removed = this.members.filter(member => !incoming.has(member.provider_id)).length; + const merged = data.members.map(member => { + const old = preserve ? previous.get(member.provider_id) : null; + if (preserve && !old) added += 1; + return { ...member, skipped: old ? old.skipped : false }; + }); + // Retain the user's workspace row positions for surviving members, too. + if (preserve) { + const positions = new Map(this.members.map((member, index) => [member.provider_id, index])); + merged.sort((a, b) => (positions.get(a.provider_id) ?? Infinity) - (positions.get(b.provider_id) ?? Infinity)); + } + this.renumber(merged); + this.title = data.title; + this.description = data.description; + if (this.coverUrl !== data.coverUrl) this.coverFailed = false; + this.coverUrl = data.coverUrl; + this.fingerprint = data.fingerprint; + const defaultsChanged = preserve && this.fileDefaultsFingerprint !== data.fileDefaultsFingerprint; + this.fileDefaultsFingerprint = data.fileDefaultsFingerprint; + this.fileSummary = data.fileSummary; + this.fileExample = data.fileExample; + this.rootOptions = data.roots; + const rootRemoved = this.libraryRootId && !data.roots.some(root => root[0] === this.libraryRootId); + if (rootRemoved) this.libraryRootId = ''; + this.setPage(this.page); + this.notice = preserve ? (added || removed + ? `Preview updated: ${added} added, ${removed} no longer listed. Choices for remaining issues were kept. Review the updated list before adding.` + : 'Preview updated. Your reading order, skips, and settings were kept.') : ''; + if (defaultsChanged) this.notice += ' Story Arc file defaults changed; review the settings summary below.'; + if (rootRemoved) this.notice += ' The selected library root is no longer available. Choose another root.'; + this.publish(); + }, + async retry() { await this.requestPreview(); }, + explainSubmitError(message) { + this.submitError = message; + this.$nextTick(() => this.$root.querySelector('[data-testid="story-arc-preview-submit-error"]').focus()); + }, + async submit(form) { + if (this.busy) return; + if (!this.ready) { + this.explainSubmitError('The preview is incomplete. Select Retry preview to load all issues before adding this Story Arc.'); + return; + } + if (!this.libraryRootId) { + this.explainSubmitError(this.rootOptions.length < 2 + ? 'No managed library root is available. Configure one in Settings > Media Management > Library roots, then select Retry preview.' + : 'Choose a library root for new series before adding this Story Arc.'); + return; + } + const body = new FormData(form); + // Always submit every page. Positions come from list order, never user input. + for (const field of ['issue_provider_ids', 'reading_orders', 'skipped_issue_provider_ids']) body.delete(field); + for (const [index, member] of this.members.entries()) { + body.append('issue_provider_ids', member.provider_id); + body.append('reading_orders', index + 1); + if (member.skipped) body.append('skipped_issue_provider_ids', member.provider_id); + } + await this.requestPreview(body); + }, + async requestPreview(body = null) { + if (this.busy) return; + this.busy = body ? 'adding' : 'retrying'; + this.error = ''; + this.submitError = ''; + this.notice = ''; + const controller = new AbortController(); + this.controller = controller; + const timeout = setTimeout(() => controller.abort(), 120000); + try { + const response = await fetch(this.endpoint, { + method: body ? 'POST' : 'GET', body, credentials: 'same-origin', cache: 'no-store', + headers: body ? { 'X-CSRF-Token': readCsrfTokenFromBody() } : {}, signal: controller.signal, + }); + if (!this.$el.isConnected) return; + const target = new URL(response.url); + if (response.ok && target.origin === location.origin && /^\/story-arcs\/\d+$/.test(target.pathname)) { + location.assign(target.href); + return; + } + if (target.pathname === '/login' || response.status === 401 || response.status === 403) { + throw new Error('Your session expired. Sign in again before adding this Story Arc.'); + } + if (!response.ok) throw new Error('The preview could not be saved or refreshed. Your edits are still here. Retry preview.'); + const document = new DOMParser().parseFromString(await response.text(), 'text/html'); + const seed = document.querySelector('[data-preview-data]'); + if (!seed) throw new Error('The preview response was unavailable. Your edits are still here. Retry preview.'); + this.applySnapshot(JSON.parse(seed.textContent)); + } catch (error) { + if (!this.$el.isConnected) return; + this.ready = false; + this.error = error.name === 'AbortError' + ? 'Comic Vine took too long to respond. Your edits are still here. Retry preview.' + : error instanceof TypeError || error instanceof SyntaxError + ? 'The connection failed. Your edits are still here. Retry preview.' : error.message; + } finally { + clearTimeout(timeout); + this.controller = null; + this.busy = ''; + if (this.$el.isConnected) this.publish(); + } + }, + }; +} + +function storyArcPreviewFooter() { + return { + page: 1, totalPages: 1, total: 0, skipped: 0, ready: false, + init() { this.$nextTick(() => this.$dispatch('story-arc-preview-request-status')); }, + get pageTokens() { + const total = this.totalPages, page = this.page; + if (total <= 5) return Array.from({ length: total }, (_, index) => index + 1); + if (page <= 2 || page >= total - 1) return [1, 2, null, total - 1, total]; + return page <= Math.floor((total + 1) / 2) + ? [page - 1, page, null, total - 1, total] : [1, 2, null, page, page + 1]; + }, + goToPage(page) { this.$dispatch('story-arc-preview-page', { page }); }, + }; +} diff --git a/src/pullbox/ui/story_arc_catalog_forms.py b/src/pullbox/ui/story_arc_catalog_forms.py new file mode 100644 index 00000000..a95bd8f1 --- /dev/null +++ b/src/pullbox/ui/story_arc_catalog_forms.py @@ -0,0 +1,36 @@ +"""Validated browser fields for reviewing and adopting a provider Story Arc.""" + +from pydantic import BaseModel, Field + +from pullbox.services.story_arc_service import StoryArcValidationError + + +class StoryArcCatalogAddForm(BaseModel): + """Validate the submitted order, canonical root and current file defaults.""" + + fingerprint: str = Field(max_length=128) + file_defaults_fingerprint: str = Field(default="", max_length=128) + issue_provider_ids: list[str] = Field(default_factory=list, max_length=2000) + reading_orders: list[int] = Field(default_factory=list, max_length=2000) + skipped_issue_provider_ids: list[str] = Field(default_factory=list, max_length=2000) + library_root_id: int = Field(ge=1) + monitored: bool = False + search_missing: bool = False + include_upcoming: bool = False + + def reviewed_order(self) -> list[str]: + """Every member must have one distinct positive position before adoption.""" + if ( + not self.issue_provider_ids + or len(self.issue_provider_ids) != len(self.reading_orders) + or len(set(self.issue_provider_ids)) != len(self.issue_provider_ids) + or len(set(self.reading_orders)) != len(self.reading_orders) + or any(order < 1 for order in self.reading_orders) + ): + raise StoryArcValidationError("Give every member a different positive reading order.") + return [ + provider_id + for _, provider_id in sorted( + zip(self.reading_orders, self.issue_provider_ids, strict=True) + ) + ] diff --git a/src/pullbox/ui/story_arc_catalog_routes.py b/src/pullbox/ui/story_arc_catalog_routes.py new file mode 100644 index 00000000..a3399c53 --- /dev/null +++ b/src/pullbox/ui/story_arc_catalog_routes.py @@ -0,0 +1,483 @@ +"""Preview-first Comic Vine arc discovery and explicit membership refresh.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import TYPE_CHECKING, Annotated +from urllib.parse import urlencode + +import structlog +from fastapi import APIRouter, BackgroundTasks, Form, HTTPException, Path, Query, Request +from fastapi.responses import RedirectResponse +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from starlette.responses import Response + +from pullbox.api.deps import ( + AuthenticatedUser, + DbSession, + get_request_session_factory, +) +from pullbox.core.issue_numbers import format_issue_number +from pullbox.core.library_policy import load_search_on_add_default +from pullbox.models.story_arc import StoryArc, StoryArcLifecycle +from pullbox.providers.metadata.comicvine import ComicVineError +from pullbox.services.cover_url_service import build_story_arc_cover_url +from pullbox.services.story_arc_file_defaults import load_story_arc_file_defaults +from pullbox.services.story_arc_placement_integration import StoryArcPlacementIntegrationError +from pullbox.services.story_arc_service import StoryArcServiceError, StoryArcValidationError +from pullbox.ui.comicvine_provider import ( + ComicVineNotConfiguredError, + open_comicvine_ui_provider, +) +from pullbox.ui.story_arc_catalog_forms import StoryArcCatalogAddForm # noqa: TC001 +from pullbox.ui.story_arc_presenters import load_story_arc_placement_roots + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + + from fastapi.templating import Jinja2Templates + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + from pullbox.services.story_arc_catalog import ( + StoryArcCatalogPreview, + StoryArcCatalogService, + ) + +logger = structlog.get_logger(__name__) +router = APIRouter() +_get_templates: Callable[[], Jinja2Templates] | None = None +_build_context: Callable[..., dict[str, object]] | None = None +_ProviderId = Annotated[str, Path(pattern=r"^[1-9][0-9]{0,18}$")] +_ERRORS = { + "provider": "Comic Vine couldn't load this arc. Check the provider settings and try again.", + "validation": ( + "The arc wasn't added. Review the reading order and storage choices, then confirm again." + ), + "stale": ( + "The provider membership changed after preview. Review the latest list before confirming." + ), + "conflict": "This arc changed in another tab. Review the latest changes before confirming.", + "file-defaults": ( + "Story Arc file defaults changed or are unavailable. " + "Review Settings → Media Management, then preview this arc again." + ), + "catalog_limit_exceeded": ( + "This arc exceeds the supported review limit. Nothing was added or changed." + ), + "incomplete_hydration": ( + "Some member details couldn't be loaded. Nothing was added or changed; retry preview." + ), + "canonical_root_required": ( + "Choose a library root for new series before saving provider changes." + ), + "canonical_root_unavailable": ( + "Restore or enable the saved library root in Settings, then retry provider changes. " + "Existing series paths and arc storage haven't changed." + ), +} + + +@dataclass(frozen=True) +class _TemplateUser: + username: str + + +def configure_story_arc_catalog_routes( + *, get_templates: Callable[[], Jinja2Templates], build_context: Callable[..., dict[str, object]] +) -> None: + global _get_templates, _build_context + _get_templates, _build_context = get_templates, build_context + + +def _render(request: Request, username: str, template: str, **values: object) -> Response: + if _get_templates is None or _build_context is None: + raise RuntimeError("Story Arc catalog routes are not configured") + context = _build_context(request, _TemplateUser(username), **values) + return _get_templates().TemplateResponse(request, template, context) + + +def _redirect(request: Request, url: str) -> Response: + if request.headers.get("HX-Request"): + return Response(status_code=204, headers={"HX-Redirect": url}) + # All callers build /story-arcs routes from integer IDs or the numeric + # _ProviderId validator; no request value supplies an origin or URL prefix. + # codeql[py/url-redirection] + return RedirectResponse(url, status_code=303) + + +@asynccontextmanager +async def _catalog_service( + session: DbSession, + *, + session_factory: async_sessionmaker[AsyncSession] | None = None, +) -> AsyncIterator[StoryArcCatalogService]: + from pullbox.services.story_arc_catalog import StoryArcCatalogService + + try: + async with open_comicvine_ui_provider( + session, + session_factory=session_factory, + ) as provider: + yield StoryArcCatalogService(provider) + except ComicVineNotConfiguredError as exc: + raise StoryArcValidationError("Comic Vine is not configured") from exc + + +def _failure_code(exc: Exception) -> str: + code = getattr(exc, "code", "") + if code in ("catalog_limit_exceeded", "incomplete_hydration"): + return str(code) + return "provider" + + +async def _existing(session: DbSession, provider_id: str) -> int | None: + if int(provider_id) > 2**63 - 1: + raise HTTPException(status_code=404, detail="Story Arc provider identity not found") + existing_id = await session.scalar( + select(StoryArc.id).where(StoryArc.comicvine_id == int(provider_id)) + ) + return int(existing_id) if existing_id is not None else None + + +async def load_story_arc_catalog_search_context( + session: DbSession, + *, + q: str, + page: int, + base_url: str, + session_factory: async_sessionmaker[AsyncSession] | None = None, +) -> dict[str, object]: + """Load one bounded Comic Vine arc result page for full or HTMX shells.""" + query = q.strip() + results: list[dict[str, object]] = [] + total = 0 + error = "" + if len(query) >= 2: + try: + async with _catalog_service(session, session_factory=session_factory) as service: + found, total = await service.search(query, limit=20, offset=(page - 1) * 20) + existing = await service.find_existing( + session, [item.provider_id for item in found] + ) + results = [ + {"metadata": item, "existing_id": existing.get(item.provider_id)} + for item in found + ] + except (ComicVineError, StoryArcServiceError): + await session.rollback() + error = "Comic Vine search failed. Check the provider settings and try again." + logger.warning("story_arc_catalog_search_failed") + + total_pages = max(1, (total + 19) // 20) + return { + "query": query, + "catalog_query": query, + "results": results, + "total": total, + "page": page, + "total_pages": total_pages, + "shown_count": len(results), + "in_library_count": sum(1 for result in results if result["existing_id"]), + "error_message": error, + "pagination_base_url": (f"{base_url}?{urlencode({'q': query})}" if query else base_url), + "next_url": ( + base_url + "?" + urlencode({"q": query, "page": page + 1}) if page * 20 < total else "" + ), + "previous_url": ( + base_url + "?" + urlencode({"q": query, "page": page - 1}) if page > 1 else "" + ), + } + + +def _members(preview: StoryArcCatalogPreview) -> list[dict[str, str]]: + titles = {series.provider_id: series.title for series in preview.series} + return [ + { + "provider_id": issue.provider_id, + "series_name": titles.get( + issue.series_provider_id, f"Series {issue.series_provider_id}" + ), + "issue_number": issue.issue_number_text or format_issue_number(issue.issue_number), + "title": issue.title or "Untitled issue", + } + for issue in preview.issues + ] + + +@router.get("/story-arcs/catalog", include_in_schema=False) +async def story_arc_catalog_search( + request: Request, + user: AuthenticatedUser, + session: DbSession, + q: Annotated[str, Query(max_length=500)] = "", + page: Annotated[int, Query(ge=1, le=100)] = 1, +) -> Response: + username = user.username + context = await load_story_arc_catalog_search_context( + session, + q=q, + page=page, + base_url="/story-arcs/catalog", + session_factory=get_request_session_factory(request), + ) + return _render( + request, + username, + "partials/story_arc_catalog_results.html" + if request.headers.get("HX-Request") + else "pages/story_arc_catalog.html", + **context, + ) + + +@router.get("/story-arcs/catalog/{provider_id}", include_in_schema=False) +async def story_arc_catalog_preview( + provider_id: _ProviderId, + request: Request, + user: AuthenticatedUser, + session: DbSession, + error: str = Query(""), +) -> Response: + username = user.username + existing_id = await _existing(session, provider_id) + if existing_id is not None: + return _redirect(request, f"/story-arcs/{existing_id}") + preview = None + message = _ERRORS.get(error, "") + try: + async with _catalog_service( + session, + session_factory=get_request_session_factory(request), + ) as service: + preview = await service.preview(provider_id) + except (ComicVineError, StoryArcServiceError) as exc: + await session.rollback() + message = _ERRORS[_failure_code(exc)] + logger.warning("story_arc_catalog_preview_failed", category=_failure_code(exc)) + roots, truncated = await load_story_arc_placement_roots(session, selected_root_id=None) + managed_roots = tuple(root for root in roots if root.can_manage) + return _render( + request, + username, + "pages/story_arc_catalog_preview.html", + preview=preview, + members=_members(preview) if preview else [], + provider_id=provider_id, + error_message=message, + placement_roots=roots, + managed_roots=managed_roots, + placement_roots_truncated=truncated, + arc_file_defaults=await load_story_arc_file_defaults(session), + ) + + +@router.post("/story-arcs/catalog/{provider_id}", include_in_schema=False) +async def story_arc_catalog_add( + provider_id: _ProviderId, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + background_tasks: BackgroundTasks, + form: Annotated[StoryArcCatalogAddForm, Form()], +) -> Response: + existing_id = await _existing(session, provider_id) + if existing_id is not None: + return _redirect(request, f"/story-arcs/{existing_id}") + code = "validation" + try: + order = form.reviewed_order() + defaults = await load_story_arc_file_defaults(session) + if ( + form.file_defaults_fingerprint + and form.file_defaults_fingerprint != defaults.fingerprint + ): + code = "file-defaults" + raise StoryArcValidationError("File defaults changed") + policy = defaults.proposal() + async with _catalog_service( + session, + session_factory=get_request_session_factory(request), + ) as service: + preview = await service.preview(provider_id) + if preview.fingerprint != form.fingerprint: + code = "stale" + raise StoryArcValidationError("Preview changed") + arc = await service.add( + session, + preview, + ordered_issue_provider_ids=order, + skipped_issue_provider_ids=form.skipped_issue_provider_ids, + library_root_id=form.library_root_id, + monitored=form.monitored, + search_missing=form.monitored, + include_upcoming=form.monitored, + placement_policy=policy, + ) + arc_id = arc.id + initial_pending = _has_initial_work(arc) + search_on_add = arc.monitored and await load_search_on_add_default(session) + await session.commit() + except StoryArcPlacementIntegrationError: + await session.rollback() + return _redirect(request, f"/story-arcs/catalog/{provider_id}?error=file-defaults") + except (StoryArcServiceError, IntegrityError): + await session.rollback() + return _redirect(request, f"/story-arcs/catalog/{provider_id}?error={code}") + except ComicVineError: + await session.rollback() + return _redirect(request, f"/story-arcs/catalog/{provider_id}?error=provider") + if search_on_add: + from pullbox.tasks.story_arc_search_task import schedule_story_arc_search + + schedule_story_arc_search(arc_id) + if initial_pending: + from pullbox.services.story_arc_catalog_placement import run_catalog_initial_placements + + background_tasks.add_task( + run_catalog_initial_placements, + arc_id, + session_factory=request.app.state.db_session_factory, + ) + return _redirect(request, f"/story-arcs/{arc_id}?notice=catalog-added") + + +async def _provider_arc(session: DbSession, arc_id: int) -> StoryArc: + arc = await session.get(StoryArc, arc_id) + if arc is None or arc.comicvine_id is None or arc.lifecycle is not StoryArcLifecycle.ACTIVE: + raise HTTPException(status_code=404, detail="Active provider Story Arc not found") + return arc + + +def _has_initial_work(arc: StoryArc) -> bool: + marker = (arc.diagnostics or {}).get("catalog_initial_placements") + if not isinstance(marker, dict): + return False + return any(type(value := marker.get(key)) is int and value > 0 for key in ("pending", "failed")) + + +@router.post("/story-arcs/{story_arc_id}/initial-placements/retry", include_in_schema=False) +async def story_arc_catalog_initial_placements_retry( + story_arc_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + background_tasks: BackgroundTasks, +) -> Response: + """Resume only the frozen creation work, never migrate an established policy.""" + from pullbox.services.story_arc_catalog_placement import run_catalog_initial_placements + + arc = await _provider_arc(session, story_arc_id) + if not _has_initial_work(arc): + return _redirect(request, f"/story-arcs/{story_arc_id}") + await session.commit() + background_tasks.add_task( + run_catalog_initial_placements, + story_arc_id, + retry_failed=True, + session_factory=request.app.state.db_session_factory, + ) + return _redirect(request, f"/story-arcs/{story_arc_id}?notice=catalog-placements-started") + + +@router.get("/story-arcs/{story_arc_id}/catalog-refresh", include_in_schema=False) +async def story_arc_catalog_refresh_preview( + story_arc_id: int, + request: Request, + user: AuthenticatedUser, + session: DbSession, + error: str = Query(""), +) -> Response: + username = user.username + arc = await _provider_arc(session, story_arc_id) + name, provider_id = arc.name, str(arc.comicvine_id) + cover_src = build_story_arc_cover_url(arc) + comicvine_url = arc.comicvine_url or ( + f"https://comicvine.gamespot.com/story-arc/4045-{provider_id}/" + ) + catalog = (arc.diagnostics or {}).get("provider_catalog") + saved_root = catalog.get("canonical_library_root_id") if isinstance(catalog, dict) else None + needs_library_root = not (type(saved_root) is int and saved_root > 0) + preview = None + changes = None + message = _ERRORS.get(error, "") + try: + async with _catalog_service( + session, + session_factory=get_request_session_factory(request), + ) as service: + preview = await service.preview(provider_id) + if preview.membership_complete: + changes = await service.preview_refresh(session, story_arc_id, preview) + except (ComicVineError, StoryArcServiceError) as exc: + await session.rollback() + message = _ERRORS[_failure_code(exc)] + roots, truncated = await load_story_arc_placement_roots(session, selected_root_id=None) + managed_roots = tuple(root for root in roots if root.can_manage) + return _render( + request, + username, + "pages/story_arc_catalog_refresh.html", + story_arc_id=story_arc_id, + arc_name=name, + arc_cover_src=cover_src, + arc_comicvine_url=comicvine_url, + preview=preview, + changes=changes, + error_message=message, + members=_members(preview) if preview else [], + needs_library_root=needs_library_root, + placement_roots=managed_roots, + placement_roots_truncated=truncated, + ) + + +@router.post("/story-arcs/{story_arc_id}/catalog-refresh", include_in_schema=False) +async def story_arc_catalog_refresh( + story_arc_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + expected_revision: Annotated[int, Form(ge=1)], + fingerprint: Annotated[str, Form(max_length=128)], + confirm_refresh: bool = Form(False), + library_root_id: Annotated[int | None, Form(ge=1)] = None, +) -> Response: + arc = await _provider_arc(session, story_arc_id) + provider_id = str(arc.comicvine_id) + code = "conflict" + try: + if not confirm_refresh: + raise StoryArcValidationError("Confirm the refresh preview") + async with _catalog_service( + session, + session_factory=get_request_session_factory(request), + ) as service: + preview = await service.preview(provider_id) + if preview.fingerprint != fingerprint: + code = "stale" + raise StoryArcValidationError("Preview changed") + result = await service.refresh( + session, + story_arc_id, + preview, + expected_revision=expected_revision, + library_root_id=library_root_id, + ) + search_on_add = result.story_arc.monitored and await load_search_on_add_default(session) + await session.commit() + except (StoryArcServiceError, IntegrityError) as exc: + await session.rollback() + root_error = getattr(exc, "code", "") + if root_error in {"canonical_root_required", "canonical_root_unavailable"}: + code = root_error + return _redirect(request, f"/story-arcs/{story_arc_id}/catalog-refresh?error={code}") + except ComicVineError: + await session.rollback() + return _redirect(request, f"/story-arcs/{story_arc_id}/catalog-refresh?error=provider") + if search_on_add: + from pullbox.tasks.story_arc_search_task import schedule_story_arc_search + + schedule_story_arc_search(story_arc_id) + return _redirect(request, f"/story-arcs/{story_arc_id}?notice=catalog-refreshed") diff --git a/src/pullbox/ui/story_arc_local_issue_search.py b/src/pullbox/ui/story_arc_local_issue_search.py new file mode 100644 index 00000000..7cef5e0a --- /dev/null +++ b/src/pullbox/ui/story_arc_local_issue_search.py @@ -0,0 +1,142 @@ +"""Bounded local-library issue search for unresolved Story Arc entries.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from sqlalchemy import and_, case, exists, func, or_, select + +from pullbox.core.db_utils import escape_like +from pullbox.core.issue_numbers import format_issue_number, parse_issue_number_text +from pullbox.models.issue import Issue +from pullbox.models.library import LibraryFile +from pullbox.models.series import Series + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + + +LOCAL_ISSUE_RESULT_LIMIT = 25 + + +@dataclass(frozen=True, slots=True) +class StoryArcLocalIssueCandidate: + """One path-free canonical issue candidate safe to render in a picker.""" + + issue_id: int + series_title: str + issue_number_text: str + issue_title: str + status: str + canonical_file_available: bool + + +@dataclass(frozen=True, slots=True) +class StoryArcLocalIssueSearchResult: + """A fixed-size result page; local search intentionally has no full export.""" + + query: str + items: tuple[StoryArcLocalIssueCandidate, ...] + has_more: bool + limit: int = LOCAL_ISSUE_RESULT_LIMIT + + +async def search_story_arc_local_issues( + session: AsyncSession, + *, + query: str, + source_series_name: str | None, + source_issue_number_text: str | None, +) -> StoryArcLocalIssueSearchResult: + """Search canonical issues without providers or unbounded relationship loads.""" + search_text = query.strip() + if not search_text: + return StoryArcLocalIssueSearchResult(query="", items=(), has_more=False) + + escaped = escape_like(search_text) + pattern = f"%{escaped}%" + normalized = search_text.casefold() + exact_source_series = (source_series_name or "").strip().casefold() + exact_source_number = (source_issue_number_text or "").strip().casefold() + + issue_number_filters: list[ColumnElement[bool]] = [] + try: + numeric_query, exact_query = parse_issue_number_text(search_text) + except ValueError: + issue_number_filters.append(Issue.issue_number_text.ilike(pattern, escape="\\")) + else: + issue_number_filters.append(func.lower(Issue.issue_number_text) == exact_query.casefold()) + if exact_query == format_issue_number(numeric_query): + issue_number_filters.append( + and_( + Issue.issue_number_text.is_(None), + Issue.issue_number == numeric_query, + ) + ) + issue_number_match = or_(*issue_number_filters) + filters = or_( + Series.title.ilike(pattern, escape="\\"), + Issue.title.ilike(pattern, escape="\\"), + issue_number_match, + ) + has_file = exists(select(LibraryFile.id).where(LibraryFile.issue_id == Issue.id).limit(1)) + rows = ( + await session.execute( + select( + Issue.id, + Series.title, + Issue.issue_number_text, + Issue.issue_number, + Issue.title, + Issue.status, + has_file.label("canonical_file_available"), + ) + .join(Series, Series.id == Issue.series_id) + .where(filters) + .order_by( + case( + ( + func.lower(Series.title) == exact_source_series, + 0, + ), + else_=1, + ), + case( + ( + func.lower(Issue.issue_number_text) == exact_source_number, + 0, + ), + else_=1, + ), + case((func.lower(Series.title) == normalized, 0), else_=1), + case((func.lower(Issue.issue_number_text) == normalized, 0), else_=1), + Series.sort_title.asc(), + Issue.issue_number.asc(), + Issue.issue_number_text.asc(), + Issue.id.asc(), + ) + .limit(LOCAL_ISSUE_RESULT_LIMIT + 1) + ) + ).all() + items = tuple( + StoryArcLocalIssueCandidate( + issue_id=int(row.id), + series_title=str(row.title), + issue_number_text=( + str(row.issue_number_text) + if row.issue_number_text is not None + else format_issue_number(float(row.issue_number)) + ), + issue_title=str(row[4] or "Untitled issue"), + status=row.status.value, + canonical_file_available=bool(row.canonical_file_available), + ) + for row in rows[:LOCAL_ISSUE_RESULT_LIMIT] + ) + return StoryArcLocalIssueSearchResult( + query=search_text, + items=items, + has_more=len(rows) > LOCAL_ISSUE_RESULT_LIMIT, + ) diff --git a/src/pullbox/ui/story_arc_presenters.py b/src/pullbox/ui/story_arc_presenters.py new file mode 100644 index 00000000..749c278a --- /dev/null +++ b/src/pullbox/ui/story_arc_presenters.py @@ -0,0 +1,1383 @@ +"""Bounded query presenters for normal Story Arc management pages.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING +from urllib.parse import urlencode + +from sqlalchemy import case, func, select +from sqlalchemy.orm import joinedload + +from pullbox.core.db_utils import escape_like +from pullbox.models.issue import Issue, IssueStatus +from pullbox.models.publisher import Publisher +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcPlacement, + StoryArcPlacementState, + StoryArcResolutionState, +) +from pullbox.models.story_arc_sync import StoryArcSyncWork, StoryArcSyncWorkState +from pullbox.services.cover_url_service import build_story_arc_cover_url +from pullbox.services.library_root_management import list_library_roots +from pullbox.services.reading_query_service import ( + ReadingStateProjection, + load_story_arc_reading_aggregates, + load_visible_issue_states, +) +from pullbox.services.story_arc_editing_policy import can_manually_edit_arc +from pullbox.services.story_arc_membership_policy import ( + order_review_filter, + provider_issue_identity, + requires_order_review, +) +from pullbox.services.story_arc_placement_integration import ( + StoryArcPlacementPolicy, + StoryArcPlacementPolicyInput, + StoryArcPlacementPolicyMode, + StoryArcPlacementPreviewItem, + StoryArcPlacementSyncService, + StoryArcPlacementView, +) +from pullbox.services.story_arc_search_targets import load_story_arc_search_eligible_counts +from pullbox.ui.reading_presenters import IssueReadingView, present_issue_reading + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + from sqlalchemy.sql.elements import ColumnElement + from sqlalchemy.sql.selectable import Subquery + + +@dataclass(frozen=True, slots=True) +class StoryArcListItemView: + """One compact Story Arc row with aggregate entry state.""" + + id: int + name: str + description: str | None + lifecycle: str + source_kind: str + monitored: bool + sync_enabled: bool + revision: int + publisher_name: str | None + cover_src: str | None + cover_loading: str + cover_fetchpriority: str + cover_decoding: str + membership_count: int + resolved_count: int + owned_count: int + readable_count: int + read_count: int + completion_pct: int + acquisition_tone: str + system_tone: str + pending_count: int + missing_count: int + ambiguous_count: int + conflict_count: int + placement_problem_count: int + sync_failure_count: int + review_count: int + review_tone: str + review_summary: str + search_eligible_count: int + completion_label: str + + +@dataclass(frozen=True, slots=True) +class StoryArcListPageView: + """A bounded Story Arc registry page.""" + + items: tuple[StoryArcListItemView, ...] + total: int + page: int + per_page: int + total_pages: int + active_count: int + archived_count: int + monitored_count: int + membership_count: int + resolved_count: int + owned_count: int + review_count: int + pending_count: int + missing_count: int + ambiguous_count: int + conflict_count: int + + +@dataclass(frozen=True, slots=True) +class StoryArcMembershipView: + """One ordered membership with fully loaded canonical context.""" + + id: int + issue_id: int | None + position: int + sequence_number: int + source_ordinal: int + exact_issue_number: str + series_name: str + issue_title: str + resolution_state: str + resolution_label: str + issue_status: str | None + reading: IssueReadingView | None + can_resolve: bool + order_review_required: bool + local_search_query: str + metadata_add_url: str + is_first: bool + is_last: bool + can_rematch: bool = True + + +@dataclass(frozen=True, slots=True) +class StoryArcInitialPlacementView: + """Aggregate initial-file progress without exposing internal path snapshots.""" + + state: str + total: int + completed: int + failed: int + pending: int + + +@dataclass(frozen=True, slots=True) +class StoryArcDetailView: + """Story Arc metadata and one bounded ordered membership page.""" + + id: int + name: str + description: str | None + lifecycle: str + source_kind: str + monitored: bool + search_missing: bool + include_upcoming: bool + sync_enabled: bool + revision: int + membership_count: int + resolved_count: int + owned_count: int + readable_count: int + read_count: int + acquisition_pct: int + pending_count: int + missing_count: int + ambiguous_count: int + conflict_count: int + placement_problem_count: int + sync_failure_count: int + review_count: int + review_summary: str + memberships: tuple[StoryArcMembershipView, ...] + page: int + per_page: int + total_pages: int + next_sequence_number: int + comicvine_id: int | None = None + comicvine_url: str | None = None + publisher_name: str | None = None + cover_src: str | None = None + catalog_removed_count: int = 0 + catalog_added_review_count: int = 0 + catalog_refresh_error: str | None = None + initial_placements: StoryArcInitialPlacementView | None = None + manual_editable: bool = True + + @property + def active(self) -> bool: + """Whether membership and automation controls remain editable.""" + return self.lifecycle == StoryArcLifecycle.ACTIVE.value + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementRootView: + """One bounded root option approved by the library-root registry.""" + + id: int + name: str + path: str + enabled: bool + can_manage: bool + can_reference: bool + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPolicyView: + """Complete effective or candidate policy rendered without hidden defaults.""" + + configured: bool + revision: int + mode: str + mode_label: str + layout_label: str + summary_label: str + synchronization_label: str + target_library_root_id: int | None + destination_root: str + folder_template: str + file_template: str + symlink_style: str + synchronize: bool + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPreviewItemView: + """One ownership-aware read-only preview row.""" + + membership_id: int + sequence_number: int + issue_number_text: str + mode: str + method_label: str + state: str + classification: str + target_path: str + collision: str + inspection_code: str + reason: str + required_bytes: int + required_bytes_label: str + proposed_ownership: str + ownership_label: str + placement_id: int | None + current_ownership: str + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementPreviewPageView: + """Bounded preview rows plus explicit coverage metadata.""" + + items: tuple[StoryArcPlacementPreviewItemView, ...] + total: int + page: int + per_page: int + total_pages: int + has_more: bool + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementStateView: + """One durable placement record detached from the ORM session.""" + + id: int + membership_id: int + placement_path: str + mode: str + mode_label: str + ownership: str + state: str + last_checked_label: str + can_retry: bool + can_repair: bool + can_remove: bool + safety_block_reason: str + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementStatePageView: + """Bounded durable placement state page.""" + + items: tuple[StoryArcPlacementStateView, ...] + total: int + page: int + per_page: int + total_pages: int + has_more: bool + + +@dataclass(frozen=True, slots=True) +class StoryArcSyncWorkSummaryView: + """Fixed-cardinality aggregate of durable automatic placement work.""" + + queued: int + running: int + retry_wait: int + failed: int + completed: int + cancelled: int + total: int + + +@dataclass(frozen=True, slots=True) +class StoryArcPlacementContextView: + """Normal-product policy, preview, root options, and durable state.""" + + policy: StoryArcPlacementPolicyView + roots: tuple[StoryArcPlacementRootView, ...] + roots_truncated: bool + preview: StoryArcPlacementPreviewPageView + placements: StoryArcPlacementStatePageView + sync_work: StoryArcSyncWorkSummaryView + page: int + total_pages: int + preview_only: bool + + +_placement_service = StoryArcPlacementSyncService() +_PLACEMENT_UI_PAGE_SIZE = 10 +_PLACEMENT_ROOT_OPTION_LIMIT = 100 + + +def _membership_counts_subquery() -> Subquery: + """Aggregate membership state once for a bounded registry query.""" + return ( + select( + IssueStoryArc.story_arc_id.label("story_arc_id"), + func.count(IssueStoryArc.id).label("membership_count"), + func.sum( + case( + (IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, 1), + else_=0, + ) + ).label("resolved_count"), + func.sum( + case( + ( + (IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED) + & (Issue.status == IssueStatus.OWNED), + 1, + ), + else_=0, + ) + ).label("owned_count"), + func.sum( + case( + ( + (IssueStoryArc.resolution_state == StoryArcResolutionState.PENDING) + | ( + (IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED) + & order_review_filter() + ), + 1, + ), + else_=0, + ) + ).label("pending_count"), + func.sum( + case( + (IssueStoryArc.resolution_state == StoryArcResolutionState.MISSING, 1), + else_=0, + ) + ).label("missing_count"), + func.sum( + case( + (IssueStoryArc.resolution_state == StoryArcResolutionState.AMBIGUOUS, 1), + else_=0, + ) + ).label("ambiguous_count"), + func.sum( + case( + (IssueStoryArc.resolution_state == StoryArcResolutionState.CONFLICT, 1), + else_=0, + ) + ).label("conflict_count"), + ) + .outerjoin(Issue, Issue.id == IssueStoryArc.issue_id) + .group_by(IssueStoryArc.story_arc_id) + .subquery() + ) + + +def _placement_problem_counts_subquery() -> Subquery: + return ( + select( + IssueStoryArc.story_arc_id.label("story_arc_id"), + func.count(StoryArcPlacement.id).label("placement_problem_count"), + ) + .join(StoryArcPlacement, StoryArcPlacement.issue_story_arc_id == IssueStoryArc.id) + .where( + StoryArcPlacement.state.in_( + ( + StoryArcPlacementState.MISSING, + StoryArcPlacementState.DRIFTED, + StoryArcPlacementState.FAILED, + ) + ) + ) + .group_by(IssueStoryArc.story_arc_id) + .subquery() + ) + + +def _sync_failure_counts_subquery() -> Subquery: + return ( + select( + IssueStoryArc.story_arc_id.label("story_arc_id"), + func.count(StoryArcSyncWork.id).label("sync_failure_count"), + ) + .join(StoryArcSyncWork, StoryArcSyncWork.issue_story_arc_id == IssueStoryArc.id) + .where(StoryArcSyncWork.state == StoryArcSyncWorkState.FAILED) + .group_by(IssueStoryArc.story_arc_id) + .subquery() + ) + + +def _review_summary( + *, + pending: int, + missing: int, + ambiguous: int, + conflicts: int, + placements: int, + sync_failures: int, +) -> str: + labels: tuple[tuple[int, str, str], ...] = ( + (pending, "pending", "pending"), + (missing, "missing match", "missing matches"), + (ambiguous, "ambiguous", "ambiguous"), + (conflicts, "identity conflict", "identity conflicts"), + (placements, "placement problem", "placement problems"), + (sync_failures, "sync failure", "sync failures"), + ) + parts = [ + f"{count} {singular if count == 1 else plural}" + for count, singular, plural in labels + if count + ] + return " · ".join(parts) if parts else "No review needed" + + +def _completion_label(*, total: int, resolved: int) -> str: + if total == 0: + return "Empty" + return f"{resolved} of {total} resolved" + + +async def load_story_arc_list_page( + session: AsyncSession, + *, + q: str | None, + lifecycle: StoryArcLifecycle | None, + monitored: bool | None, + page: int, + per_page: int, + user_id: int, +) -> StoryArcListPageView: + """Load one stable Story Arc registry page with literal search semantics.""" + filters: list[ColumnElement[bool]] = [] + if q is not None and (search_text := q.strip()): + filters.append(StoryArc.name.ilike(f"%{escape_like(search_text)}%", escape="\\")) + if lifecycle is not None: + filters.append(StoryArc.lifecycle == lifecycle) + if monitored is not None: + filters.append(StoryArc.monitored.is_(monitored)) + + counts = _membership_counts_subquery() + placement_counts = _placement_problem_counts_subquery() + sync_counts = _sync_failure_counts_subquery() + registry_counts = ( + await session.execute( + select( + func.count(StoryArc.id), + func.coalesce( + func.sum(case((StoryArc.lifecycle == StoryArcLifecycle.ACTIVE, 1), else_=0)), + 0, + ), + func.coalesce( + func.sum(case((StoryArc.lifecycle == StoryArcLifecycle.ARCHIVED, 1), else_=0)), + 0, + ), + func.coalesce( + func.sum(case((StoryArc.monitored.is_(True), 1), else_=0)), + 0, + ), + func.coalesce(func.sum(counts.c.membership_count), 0), + func.coalesce(func.sum(counts.c.resolved_count), 0), + func.coalesce(func.sum(counts.c.owned_count), 0), + func.coalesce(func.sum(counts.c.pending_count), 0), + func.coalesce(func.sum(counts.c.missing_count), 0), + func.coalesce(func.sum(counts.c.ambiguous_count), 0), + func.coalesce(func.sum(counts.c.conflict_count), 0), + func.coalesce(func.sum(placement_counts.c.placement_problem_count), 0), + func.coalesce(func.sum(sync_counts.c.sync_failure_count), 0), + ) + .outerjoin(counts, counts.c.story_arc_id == StoryArc.id) + .outerjoin(placement_counts, placement_counts.c.story_arc_id == StoryArc.id) + .outerjoin(sync_counts, sync_counts.c.story_arc_id == StoryArc.id) + .where(*filters) + ) + ).one() + ( + total, + active_count, + archived_count, + monitored_count, + membership_count, + resolved_count, + owned_count, + pending_count, + missing_count, + ambiguous_count, + conflict_count, + placement_problem_count, + sync_failure_count, + ) = (int(value or 0) for value in registry_counts) + total_pages = max(1, (total + per_page - 1) // per_page) + safe_page = min(page, total_pages) + first_issue_id = ( + select(IssueStoryArc.issue_id) + .where( + IssueStoryArc.story_arc_id == StoryArc.id, + IssueStoryArc.issue_id.is_not(None), + ) + .order_by( + IssueStoryArc.sequence_number, + IssueStoryArc.source_ordinal, + IssueStoryArc.id, + ) + .limit(1) + .correlate(StoryArc) + .scalar_subquery() + ) + rows = ( + await session.execute( + select( + StoryArc, + Publisher.name, + first_issue_id, + func.coalesce(counts.c.membership_count, 0), + func.coalesce(counts.c.resolved_count, 0), + func.coalesce(counts.c.owned_count, 0), + func.coalesce(counts.c.pending_count, 0), + func.coalesce(counts.c.missing_count, 0), + func.coalesce(counts.c.ambiguous_count, 0), + func.coalesce(counts.c.conflict_count, 0), + func.coalesce(placement_counts.c.placement_problem_count, 0), + func.coalesce(sync_counts.c.sync_failure_count, 0), + ) + .outerjoin(counts, counts.c.story_arc_id == StoryArc.id) + .outerjoin(Publisher, Publisher.id == StoryArc.publisher_id) + .outerjoin(placement_counts, placement_counts.c.story_arc_id == StoryArc.id) + .outerjoin(sync_counts, sync_counts.c.story_arc_id == StoryArc.id) + .where(*filters) + .order_by(StoryArc.normalized_name.asc(), StoryArc.id.asc()) + .limit(per_page) + .offset((safe_page - 1) * per_page) + ) + ).all() + visible_arc_ids = tuple(int(row[0].id) for row in rows) + reading_aggregates = await load_story_arc_reading_aggregates( + session, + user_id=user_id, + story_arc_ids=visible_arc_ids, + ) + search_eligible_counts = await load_story_arc_search_eligible_counts( + session, + visible_arc_ids, + ) + items: list[StoryArcListItemView] = [] + for index, row in enumerate(rows): + ( + arc, + publisher_name, + fallback_issue_id, + item_membership_count, + item_resolved_count, + item_owned_count, + item_pending_count, + item_missing_count, + item_ambiguous_count, + item_conflict_count, + item_placement_problem_count, + item_sync_failure_count, + ) = row + membership_total = int(item_membership_count) + resolved_total = int(item_resolved_count) + owned_total = int(item_owned_count) + completion_pct = round((owned_total / membership_total) * 100) if membership_total else 0 + acquisition_tone = ( + "green" if completion_pct >= 80 else "amber" if completion_pct >= 35 else "red" + ) + pending_total = int(item_pending_count) + missing_total = int(item_missing_count) + ambiguous_total = int(item_ambiguous_count) + conflict_total = int(item_conflict_count) + placement_total = int(item_placement_problem_count) + sync_failure_total = int(item_sync_failure_count) + review_total = ( + pending_total + + missing_total + + ambiguous_total + + conflict_total + + placement_total + + sync_failure_total + ) + reading = reading_aggregates.get(arc.id) + cover_src = build_story_arc_cover_url(arc) + if cover_src is None and fallback_issue_id is not None: + cover_src = f"/api/v1/issues/{int(fallback_issue_id)}/cover" + items.append( + StoryArcListItemView( + id=arc.id, + name=arc.name, + description=arc.description, + lifecycle=arc.lifecycle.value, + source_kind=arc.source_kind.value, + monitored=arc.monitored, + sync_enabled=arc.sync_enabled, + revision=arc.revision, + publisher_name=publisher_name, + cover_src=cover_src, + cover_loading="eager" if index < 8 else "lazy", + cover_fetchpriority="high" if index < 4 else "auto", + cover_decoding="sync" if index < 2 else "async", + membership_count=membership_total, + resolved_count=resolved_total, + owned_count=owned_total, + readable_count=reading.readable_count if reading is not None else 0, + read_count=reading.completed_count if reading is not None else 0, + completion_pct=completion_pct, + acquisition_tone=acquisition_tone, + system_tone=( + "off" + if not arc.monitored or arc.lifecycle is StoryArcLifecycle.ARCHIVED + else "green" + if completion_pct >= 80 + else "amber" + ), + pending_count=pending_total, + missing_count=missing_total, + ambiguous_count=ambiguous_total, + conflict_count=conflict_total, + placement_problem_count=placement_total, + sync_failure_count=sync_failure_total, + review_count=review_total, + review_tone=( + "error" + if conflict_total or placement_total or sync_failure_total + else "warning" + if review_total + else "success" + ), + review_summary=_review_summary( + pending=pending_total, + missing=missing_total, + ambiguous=ambiguous_total, + conflicts=conflict_total, + placements=placement_total, + sync_failures=sync_failure_total, + ), + search_eligible_count=search_eligible_counts.get(arc.id, 0), + completion_label=_completion_label( + total=membership_total, + resolved=resolved_total, + ), + ) + ) + return StoryArcListPageView( + items=tuple(items), + total=total, + page=safe_page, + per_page=per_page, + total_pages=total_pages, + active_count=active_count, + archived_count=archived_count, + monitored_count=monitored_count, + membership_count=membership_count, + resolved_count=resolved_count, + owned_count=owned_count, + review_count=( + pending_count + + missing_count + + ambiguous_count + + conflict_count + + placement_problem_count + + sync_failure_count + ), + pending_count=pending_count, + missing_count=missing_count, + ambiguous_count=ambiguous_count, + conflict_count=conflict_count, + ) + + +def _resolution_label(state: StoryArcResolutionState) -> str: + return { + StoryArcResolutionState.PENDING: "Pending review", + StoryArcResolutionState.RESOLVED: "Resolved", + StoryArcResolutionState.MISSING: "Missing", + StoryArcResolutionState.AMBIGUOUS: "Ambiguous", + StoryArcResolutionState.CONFLICT: "Conflict", + StoryArcResolutionState.SKIPPED: "Skipped", + }[state] + + +def _present_membership( + membership: IssueStoryArc, + *, + position: int, + membership_total: int, + reading: IssueReadingView | None, +) -> StoryArcMembershipView: + issue = membership.issue + exact_issue_number = membership.source_issue_number_text + if exact_issue_number is None and issue is not None: + exact_issue_number = issue.effective_issue_number_text + exact_issue_number = exact_issue_number or "Unknown" + series_name = membership.source_series_name + if series_name is None and issue is not None: + series_name = issue.series.title + issue_title = membership.source_issue_title + if issue_title is None and issue is not None: + issue_title = issue.title + source_series_query = (membership.source_series_name or "").strip() + if not source_series_query and issue is not None: + source_series_query = issue.series.title + metadata_query = source_series_query + return StoryArcMembershipView( + id=membership.id, + issue_id=membership.issue_id, + position=position, + sequence_number=membership.sequence_number, + source_ordinal=membership.source_ordinal, + exact_issue_number=exact_issue_number, + series_name=series_name or "Series not matched", + issue_title=issue_title or "Untitled issue", + resolution_state=membership.resolution_state.value, + resolution_label=_resolution_label(membership.resolution_state), + issue_status=( + issue.status.value + if issue is not None and membership.resolution_state is StoryArcResolutionState.RESOLVED + else None + ), + reading=reading, + can_resolve=( + membership.issue_id is None + or ( + membership.resolution_state is StoryArcResolutionState.RESOLVED + and requires_order_review(membership) + ) + or membership.resolution_state + in { + StoryArcResolutionState.PENDING, + StoryArcResolutionState.MISSING, + StoryArcResolutionState.AMBIGUOUS, + StoryArcResolutionState.CONFLICT, + } + ), + order_review_required=requires_order_review(membership), + local_search_query=source_series_query or exact_issue_number, + metadata_add_url=( + f"/series/add?{urlencode({'q': metadata_query})}" if metadata_query else "/series/add" + ), + is_first=position == 1, + is_last=position == membership_total, + can_rematch=membership.issue_id is None or provider_issue_identity(membership) is None, + ) + + +async def load_story_arc_detail( + session: AsyncSession, + *, + story_arc_id: int, + page: int, + per_page: int, + user_id: int, +) -> StoryArcDetailView | None: + """Load one arc and one bounded, relationship-complete membership page.""" + arc_row = ( + await session.execute( + select(StoryArc, Publisher.name) + .outerjoin(Publisher, Publisher.id == StoryArc.publisher_id) + .where(StoryArc.id == story_arc_id) + ) + ).one_or_none() + if arc_row is None: + return None + arc, publisher_name = arc_row + + count_row = ( + await session.execute( + select( + func.count(IssueStoryArc.id), + func.coalesce( + func.sum( + case( + ( + IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED, + 1, + ), + else_=0, + ) + ), + 0, + ), + func.coalesce( + func.sum( + case( + ( + IssueStoryArc.resolution_state == StoryArcResolutionState.MISSING, + 1, + ), + else_=0, + ) + ), + 0, + ), + func.coalesce( + func.sum( + case( + ( + IssueStoryArc.resolution_state == StoryArcResolutionState.CONFLICT, + 1, + ), + else_=0, + ) + ), + 0, + ), + func.coalesce( + func.sum( + case( + ( + (IssueStoryArc.resolution_state == StoryArcResolutionState.PENDING) + | ( + ( + IssueStoryArc.resolution_state + == StoryArcResolutionState.RESOLVED + ) + & order_review_filter() + ), + 1, + ), + else_=0, + ) + ), + 0, + ), + func.coalesce( + func.sum( + case( + ( + IssueStoryArc.resolution_state == StoryArcResolutionState.AMBIGUOUS, + 1, + ), + else_=0, + ) + ), + 0, + ), + func.coalesce( + func.sum( + case( + ( + (IssueStoryArc.resolution_state == StoryArcResolutionState.RESOLVED) + & (Issue.status == IssueStatus.OWNED), + 1, + ), + else_=0, + ) + ), + 0, + ), + ) + .outerjoin(Issue, Issue.id == IssueStoryArc.issue_id) + .where(IssueStoryArc.story_arc_id == story_arc_id) + ) + ).one() + membership_total = int(count_row[0]) + total_pages = max(1, (membership_total + per_page - 1) // per_page) + safe_page = min(page, total_pages) + offset = (safe_page - 1) * per_page + memberships = list( + ( + await session.scalars( + select(IssueStoryArc) + .where(IssueStoryArc.story_arc_id == story_arc_id) + .options( + joinedload(IssueStoryArc.issue).joinedload(Issue.series), + joinedload(IssueStoryArc.issue).joinedload(Issue.library_file), + ) + .order_by( + IssueStoryArc.sequence_number.asc(), + IssueStoryArc.source_ordinal.asc(), + IssueStoryArc.id.asc(), + ) + .limit(per_page) + .offset(offset) + ) + ).unique() + ) + issue_ids = tuple( + membership.issue_id for membership in memberships if membership.issue_id is not None + ) + reading_states: dict[int, ReadingStateProjection] = {} + for batch_start in range(0, len(issue_ids), 50): + reading_states.update( + await load_visible_issue_states( + session, + user_id=user_id, + issue_ids=issue_ids[batch_start : batch_start + 50], + ) + ) + membership_views = tuple( + _present_membership( + membership, + position=offset + index, + membership_total=membership_total, + reading=( + present_issue_reading( + reading_states.get(membership.issue_id), + readable=membership.issue is not None + and membership.issue.library_file is not None, + ) + if membership.issue_id is not None + else None + ), + ) + for index, membership in enumerate(memberships, start=1) + ) + reading = ( + await load_story_arc_reading_aggregates( + session, + user_id=user_id, + story_arc_ids=(story_arc_id,), + ) + ).get(story_arc_id) + placement_problem_count = int( + await session.scalar( + select(func.count(StoryArcPlacement.id)) + .join(IssueStoryArc, IssueStoryArc.id == StoryArcPlacement.issue_story_arc_id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + StoryArcPlacement.state.in_( + ( + StoryArcPlacementState.MISSING, + StoryArcPlacementState.DRIFTED, + StoryArcPlacementState.FAILED, + ) + ), + ) + ) + or 0 + ) + sync_failure_count = int( + await session.scalar( + select(func.count(StoryArcSyncWork.id)) + .join(IssueStoryArc, IssueStoryArc.id == StoryArcSyncWork.issue_story_arc_id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + StoryArcSyncWork.state == StoryArcSyncWorkState.FAILED, + ) + ) + or 0 + ) + pending_count = int(count_row[4]) + ambiguous_count = int(count_row[5]) + owned_count = int(count_row[6]) + review_count = ( + pending_count + + int(count_row[2]) + + ambiguous_count + + int(count_row[3]) + + placement_problem_count + + sync_failure_count + ) + cover_src = build_story_arc_cover_url(arc) + if cover_src is None: + fallback_issue_id = await session.scalar( + select(IssueStoryArc.issue_id) + .where( + IssueStoryArc.story_arc_id == story_arc_id, + IssueStoryArc.issue_id.is_not(None), + ) + .order_by( + IssueStoryArc.sequence_number, + IssueStoryArc.source_ordinal, + IssueStoryArc.id, + ) + .limit(1) + ) + if fallback_issue_id is not None: + cover_src = f"/api/v1/issues/{int(fallback_issue_id)}/cover" + return StoryArcDetailView( + id=arc.id, + name=arc.name, + description=arc.description, + lifecycle=arc.lifecycle.value, + source_kind=arc.source_kind.value, + monitored=arc.monitored, + search_missing=arc.search_missing, + include_upcoming=arc.include_upcoming, + sync_enabled=arc.sync_enabled, + revision=arc.revision, + membership_count=membership_total, + resolved_count=int(count_row[1]), + owned_count=owned_count, + readable_count=reading.readable_count if reading is not None else 0, + read_count=reading.completed_count if reading is not None else 0, + acquisition_pct=(round((owned_count / membership_total) * 100) if membership_total else 0), + pending_count=pending_count, + missing_count=int(count_row[2]), + ambiguous_count=ambiguous_count, + conflict_count=int(count_row[3]), + placement_problem_count=placement_problem_count, + sync_failure_count=sync_failure_count, + review_count=review_count, + review_summary=_review_summary( + pending=pending_count, + missing=int(count_row[2]), + ambiguous=ambiguous_count, + conflicts=int(count_row[3]), + placements=placement_problem_count, + sync_failures=sync_failure_count, + ), + memberships=membership_views, + page=safe_page, + per_page=per_page, + total_pages=total_pages, + next_sequence_number=membership_total + 1, + comicvine_id=arc.comicvine_id, + comicvine_url=( + arc.comicvine_url + or ( + f"https://comicvine.gamespot.com/story-arc/4045-{arc.comicvine_id}/" + if arc.comicvine_id is not None + else None + ) + ), + publisher_name=publisher_name, + cover_src=cover_src, + catalog_removed_count=_catalog_diagnostic_count(arc, "removed_issue_provider_ids"), + catalog_added_review_count=pending_count if arc.comicvine_id else 0, + catalog_refresh_error=_catalog_refresh_error(arc), + initial_placements=_initial_placement_view(arc), + manual_editable=can_manually_edit_arc(arc), + ) + + +def _catalog_refresh_error(arc: StoryArc) -> str | None: + error = (arc.diagnostics or {}).get("provider_refresh_error") + if not isinstance(error, dict): + return None + if error.get("code") in {"canonical_root_required", "canonical_root_unavailable"}: + return ( + "New member discovery needs an available managed library root. " + "Check for updates to choose a root, or restore the saved root in Settings." + ) + return ( + "Comic Vine member refresh didn't finish. Existing members are unchanged; " + "check for updates to retry." + ) + + +def _catalog_diagnostic_count(arc: StoryArc, key: str) -> int: + catalog = (arc.diagnostics or {}).get("provider_catalog") + if not isinstance(catalog, dict): + return 0 + values = catalog.get(key) + return len(values) if isinstance(values, list) else 0 + + +def _initial_placement_view(arc: StoryArc) -> StoryArcInitialPlacementView | None: + marker = (arc.diagnostics or {}).get("catalog_initial_placements") + if not isinstance(marker, dict): + return None + state = str(marker.get("state", "pending")) + if state not in {"pending", "running", "failed", "complete", "blocked"}: + state = "blocked" + counts = { + key: value if type(value := marker.get(key)) is int and value >= 0 else 0 + for key in ("total", "completed", "failed", "pending") + } + return StoryArcInitialPlacementView(state=state, **counts) + + +def _placement_mode_label(mode: str) -> str: + return { + "logical": "Logical only", + "reference_only": "Reference only", + "copy": "Copy", + "hardlink": "Hardlink", + "symlink": "Symlink", + }.get(mode, mode.replace("_", " ").title()) + + +def _placement_policy_view(policy: StoryArcPlacementPolicy) -> StoryArcPlacementPolicyView: + summary_label = { + StoryArcPlacementPolicyMode.LOGICAL: "No separate folder", + StoryArcPlacementPolicyMode.COPY: "Copied to arc folder", + StoryArcPlacementPolicyMode.HARDLINK: "Hardlinked into arc folder", + StoryArcPlacementPolicyMode.SYMLINK: "Symlinked into arc folder", + StoryArcPlacementPolicyMode.REFERENCE_ONLY: "Existing files referenced", + }[policy.mode] + return StoryArcPlacementPolicyView( + configured=policy.configured, + revision=policy.revision, + mode=policy.mode.value, + mode_label=_placement_mode_label(policy.mode.value), + layout_label=( + "Logical only" + if policy.mode is StoryArcPlacementPolicyMode.LOGICAL + else "Separate Story Arc folder" + ), + summary_label=summary_label, + synchronization_label=( + "No file synchronization" + if policy.mode is StoryArcPlacementPolicyMode.LOGICAL + else "Synchronized" + if policy.synchronize + else "Manual" + ), + target_library_root_id=policy.target_library_root_id, + destination_root=policy.destination_root or "", + folder_template=policy.folder_template, + file_template=policy.file_template, + symlink_style=policy.symlink_style.value if policy.symlink_style is not None else "", + synchronize=policy.synchronize, + ) + + +def _bytes_label(value: int) -> str: + return f"{value:,} bytes" + + +def _preview_ownership_label(item: StoryArcPlacementPreviewItem) -> str: + if item.classification == "untracked_identical": + return "User-owned (untracked)" + if item.classification == "different_content": + return "User-owned (different content)" + ownership = item.current_ownership or item.proposed_ownership + if ownership == "referenced": + return "Referenced (user-owned)" + return ownership.replace("_", " ").title() + + +def _preview_item_view(item: StoryArcPlacementPreviewItem) -> StoryArcPlacementPreviewItemView: + # Placement integration returns a frozen data object. Keeping this mapper + # field-explicit makes template output independent of ORM/session lifetime. + return StoryArcPlacementPreviewItemView( + membership_id=item.membership_id, + sequence_number=item.sequence_number, + issue_number_text=item.issue_number_text, + mode=item.mode, + method_label=_placement_mode_label(item.mode), + state=item.state, + classification=item.classification, + target_path=item.target_path or "Not applicable", + collision=item.collision, + inspection_code=item.inspection_code or "", + reason=item.reason or "", + required_bytes=item.required_bytes, + required_bytes_label=_bytes_label(item.required_bytes), + proposed_ownership=item.proposed_ownership, + ownership_label=_preview_ownership_label(item), + placement_id=item.placement_id, + current_ownership=item.current_ownership or "", + ) + + +def _placement_state_view(item: StoryArcPlacementView) -> StoryArcPlacementStateView: + """Detach durable state and suppress writes after an ownership safety block.""" + last_result = dict(item.last_result) + removal_safety_blocked = ( + item.ownership.value == "managed" + and item.state.value == "drifted" + and last_result.get("operation") == "remove" + and last_result.get("error_category") in {"safety", "collision", "ownership"} + ) + return StoryArcPlacementStateView( + id=item.id, + membership_id=item.issue_story_arc_id, + placement_path=item.placement_path, + mode=item.mode.value, + mode_label=_placement_mode_label(item.mode.value), + ownership=item.ownership.value, + state=item.state.value, + last_checked_label=( + item.last_checked_at.isoformat() if item.last_checked_at is not None else "Not checked" + ), + can_retry=item.state.value != "current" and not removal_safety_blocked, + can_repair=( + item.ownership.value == "managed" + and item.state.value in {"missing", "failed"} + and not removal_safety_blocked + ), + can_remove=not removal_safety_blocked, + safety_block_reason=( + "Pullbox blocked changes because this managed artifact no longer matches its " + "recorded ownership evidence. Review the artifact manually; the canonical " + "library file was not changed." + if removal_safety_blocked + else "" + ), + ) + + +async def load_story_arc_placement_roots( + session: AsyncSession, + *, + selected_root_id: int | None, +) -> tuple[tuple[StoryArcPlacementRootView, ...], bool]: + states = await list_library_roots(session) + usable = [ + root + for root in states + if bool(root["enabled"]) + and bool(root["available"]) + and bool(root["readable"]) + and ( + (bool(root["allow_managed_writes"]) and bool(root["writable"])) + or bool(root["allow_referenced_registrations"]) + ) + ] + usable.sort( + key=lambda root: ( + not bool(root["is_default_managed_destination"]), + str(root["name"]).casefold(), + int(root["id"]), + ) + ) + usable_ids = {int(root["id"]) for root in usable} + options = list(usable) + if selected_root_id is not None and selected_root_id not in usable_ids: + selected = next( + (root for root in states if int(root["id"]) == selected_root_id), + None, + ) + if selected is not None: + options.append(selected) + truncated = len(options) > _PLACEMENT_ROOT_OPTION_LIMIT + return ( + tuple( + StoryArcPlacementRootView( + id=int(root["id"]), + name=str(root["name"]), + path=str(root["path"]), + enabled=int(root["id"]) in usable_ids, + can_manage=( + int(root["id"]) in usable_ids + and bool(root["allow_managed_writes"]) + and bool(root["writable"]) + ), + can_reference=( + int(root["id"]) in usable_ids and bool(root["allow_referenced_registrations"]) + ), + ) + for root in options[:_PLACEMENT_ROOT_OPTION_LIMIT] + ), + truncated, + ) + + +async def _load_sync_work_summary( + session: AsyncSession, + *, + story_arc_id: int, +) -> StoryArcSyncWorkSummaryView: + rows = ( + await session.execute( + select(StoryArcSyncWork.state, func.count(StoryArcSyncWork.id)) + .join( + IssueStoryArc, + StoryArcSyncWork.issue_story_arc_id == IssueStoryArc.id, + ) + .where(IssueStoryArc.story_arc_id == story_arc_id) + .group_by(StoryArcSyncWork.state) + ) + ).all() + counts = {state.value: int(count) for state, count in rows} + return StoryArcSyncWorkSummaryView( + queued=counts.get(StoryArcSyncWorkState.QUEUED.value, 0), + running=counts.get(StoryArcSyncWorkState.RUNNING.value, 0), + retry_wait=counts.get(StoryArcSyncWorkState.RETRY_WAIT.value, 0), + failed=counts.get(StoryArcSyncWorkState.FAILED.value, 0), + completed=counts.get(StoryArcSyncWorkState.COMPLETED.value, 0), + cancelled=counts.get(StoryArcSyncWorkState.CANCELLED.value, 0), + total=sum(counts.values()), + ) + + +async def load_story_arc_placement_policy_view( + session: AsyncSession, story_arc_id: int +) -> StoryArcPlacementPolicyView: + """Read saved file settings without probing placement files on page loads.""" + return _placement_policy_view(await _placement_service.get_policy(session, story_arc_id)) + + +async def load_story_arc_placement_context( + session: AsyncSession, + *, + story_arc_id: int, + page: int, + proposal: StoryArcPlacementPolicyInput | None = None, +) -> StoryArcPlacementContextView: + """Load one bounded policy preview and durable-state page for normal UI.""" + offset = (page - 1) * _PLACEMENT_UI_PAGE_SIZE + policy = ( + await _placement_service.validate_policy(session, story_arc_id, proposal) + if proposal is not None + else await _placement_service.get_policy(session, story_arc_id) + ) + preview = await _placement_service.preview_arc( + session, + story_arc_id, + limit=_PLACEMENT_UI_PAGE_SIZE, + offset=offset, + proposal=proposal, + ) + placements = await _placement_service.list_placements( + session, + story_arc_id, + limit=_PLACEMENT_UI_PAGE_SIZE, + offset=offset, + ) + preview_pages = max(1, (preview.total + _PLACEMENT_UI_PAGE_SIZE - 1) // _PLACEMENT_UI_PAGE_SIZE) + placement_pages = max( + 1, + (placements.total + _PLACEMENT_UI_PAGE_SIZE - 1) // _PLACEMENT_UI_PAGE_SIZE, + ) + total_pages = max(preview_pages, placement_pages) + safe_page = min(page, total_pages) + if safe_page != page: + offset = (safe_page - 1) * _PLACEMENT_UI_PAGE_SIZE + preview = await _placement_service.preview_arc( + session, + story_arc_id, + limit=_PLACEMENT_UI_PAGE_SIZE, + offset=offset, + proposal=proposal, + ) + placements = await _placement_service.list_placements( + session, + story_arc_id, + limit=_PLACEMENT_UI_PAGE_SIZE, + offset=offset, + ) + roots, roots_truncated = await load_story_arc_placement_roots( + session, + selected_root_id=policy.target_library_root_id, + ) + sync_work = await _load_sync_work_summary(session, story_arc_id=story_arc_id) + return StoryArcPlacementContextView( + policy=_placement_policy_view(policy), + roots=roots, + roots_truncated=roots_truncated, + preview=StoryArcPlacementPreviewPageView( + items=tuple(_preview_item_view(item) for item in preview.items), + total=preview.total, + page=safe_page, + per_page=_PLACEMENT_UI_PAGE_SIZE, + total_pages=preview_pages, + has_more=preview.has_more, + ), + placements=StoryArcPlacementStatePageView( + items=tuple(_placement_state_view(item) for item in placements.items), + total=placements.total, + page=safe_page, + per_page=_PLACEMENT_UI_PAGE_SIZE, + total_pages=placement_pages, + has_more=placements.has_more, + ), + sync_work=sync_work, + page=safe_page, + total_pages=total_pages, + preview_only=proposal is not None, + ) diff --git a/src/pullbox/ui/story_arc_routes.py b/src/pullbox/ui/story_arc_routes.py new file mode 100644 index 00000000..5994f8be --- /dev/null +++ b/src/pullbox/ui/story_arc_routes.py @@ -0,0 +1,1118 @@ +"""Normal-navigation Story Arc management UI routes.""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Annotated, Literal +from urllib.parse import urlencode + +from fastapi import APIRouter, Form, HTTPException, Query, Request +from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.templating import Jinja2Templates +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError +from starlette.responses import Response + +from pullbox.api.deps import ( + AuthenticatedUser, + DbSession, + get_request_session_factory, +) +from pullbox.config import get_settings +from pullbox.core.story_arc_naming import ( + DEFAULT_STORY_ARC_FILE_TEMPLATE, + DEFAULT_STORY_ARC_FOLDER_TEMPLATE, +) +from pullbox.models.story_arc import ( + IssueStoryArc, + StoryArc, + StoryArcLifecycle, + StoryArcSourceKind, +) +from pullbox.services.story_arc_editing_policy import ( + StoryArcManualEditingDisabledError, + require_manual_arc_edit, +) +from pullbox.services.story_arc_file_defaults import load_story_arc_file_defaults +from pullbox.services.story_arc_managed_reorder import ( + StoryArcManagedReorderError, + StoryArcManagedReorderService, + StoryArcReorderPreview, +) +from pullbox.services.story_arc_placement_integration import ( + STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION, + StoryArcPlacementIntegrationError, + StoryArcPlacementPolicyInput, + StoryArcPlacementPolicyMode, + StoryArcPlacementSyncService, + validate_story_arc_placement_policy_input, +) +from pullbox.services.story_arc_service import ( + StoryArcConflictError, + StoryArcNotFoundError, + StoryArcProviderIdentityError, + StoryArcService, + StoryArcServiceError, + StoryArcValidationError, +) +from pullbox.ui import story_arc_catalog_routes +from pullbox.ui.story_arc_local_issue_search import search_story_arc_local_issues +from pullbox.ui.story_arc_presenters import ( + load_story_arc_detail, + load_story_arc_list_page, + load_story_arc_placement_context, + load_story_arc_placement_policy_view, +) + +router = APIRouter() +router.include_router(story_arc_catalog_routes.router) + +_GetTemplates = Callable[[], Jinja2Templates] +_BuildContext = Callable[..., dict[str, object]] + +_get_templates: _GetTemplates | None = None +_build_context: _BuildContext | None = None +_story_arc_service = StoryArcService() +_placement_service = StoryArcPlacementSyncService() +_managed_reorder_service = StoryArcManagedReorderService() +_STORY_ARC_VIEW_MODES = {"list", "grid"} + + +@dataclass(frozen=True, slots=True) +class _StoryArcTemplateUser: + """Session-independent user fields needed by the shared page shell.""" + + username: str + + +_NOTICE_MESSAGES = { + "catalog-added": ( + "Story arc added with your reviewed reading order. " + "Canonical files stay in their series folders." + ), + "catalog-refreshed": ( + "Provider changes saved. New members need review; existing order, memberships, " + "and files were preserved." + ), + "search-started": ( + "Missing-issue search started. Check each issue for results and download status." + ), + "search-running": "A missing-issue search is already running for this arc.", + "catalog-placements-started": ( + "Initial arc-file work queued. Refresh to check progress; canonical files stay unchanged." + ), + "created": "Story arc created. Add issues when you are ready.", + "updated": "Story arc monitoring updated.", + "membership-added": "Issue added to the story arc.", + "moved-up": "Issue moved up in the reading order.", + "moved-down": "Issue moved down in the reading order.", + "already-first": "That issue is already first in the reading order.", + "already-last": "That issue is already last in the reading order.", + "resolved": "Story arc entry matched to the canonical issue.", + "membership-removed": "Issue removed from the story arc. Its canonical record was preserved.", + "archived": "Story arc archived. Canonical issues and files were preserved.", + "placement-policy-updated": "Story Arc placement policy saved.", + "placement-synchronized": "Story Arc placement synchronized.", + "placement-retried": "Story Arc placement retry completed.", + "placement-repaired": "Managed Story Arc placement repaired.", + "placement-managed-removed": ( + "Pullbox-managed Story Arc artifact removed. The canonical library file was preserved." + ), + "placement-reference-forgotten": ( + "Story Arc reference forgotten. The user-owned artifact was preserved." + ), +} +_ERROR_MESSAGES = { + "provider-managed": ( + "This story arc is managed by its metadata provider. Manual membership edits " + "are not enabled." + ), + "provider-identity": ( + "That issue does not match this member's provider identity. Review the provider " + "changes instead." + ), + "conflict": "This story arc changed in another tab. Review the latest state and try again.", + "not-found": "That story arc entry is no longer available.", + "validation": "That change could not be saved. Review the fields and try again.", + "placement": "Placement failed. Review the policy and placement state.", + "reorder": "That reorder could not be completed. Review the placement state and try again.", + "reorder-recovery": ( + "This reorder has unfinished managed-file work. Use the same preview below to retry " + "recovery; canonical and referenced files remain protected." + ), +} + + +def configure_story_arc_routes( + *, + get_templates: _GetTemplates, + build_context: _BuildContext, +) -> None: + """Provide shared UI runtime dependencies from the facade module.""" + global _get_templates, _build_context + _get_templates = get_templates + _build_context = build_context + story_arc_catalog_routes.configure_story_arc_catalog_routes( + get_templates=get_templates, build_context=build_context + ) + + +def _templates() -> Jinja2Templates: + if _get_templates is None: + raise RuntimeError("story arc routes have not been configured with templates") + return _get_templates() + + +def _ctx(request: Request, user: object | None = None, **kwargs: object) -> dict[str, object]: + if _build_context is None: + raise RuntimeError("story arc routes have not been configured with a context builder") + context: Mapping[str, object] = _build_context(request, user, **kwargs) + return dict(context) + + +def _redirect(request: Request, url: str) -> Response: + """Redirect both progressive-enhancement and plain form submissions.""" + if request.headers.get("HX-Request"): + return Response(status_code=204, headers={"HX-Redirect": url}) + # All callers use fixed local routes with integer IDs; _detail_url and + # _list_url encode query values so they cannot replace the destination. + # codeql[py/url-redirection] + return RedirectResponse(url=url, status_code=303) + + +def _detail_url( + story_arc_id: int, + *, + notice: str | None = None, + error: str | None = None, + page: int | None = None, + per_page: int | None = None, + placement_page: int | None = None, +) -> str: + params: list[tuple[str, str | int]] = [] + if page is not None: + params.append(("page", page)) + if per_page is not None: + params.append(("per_page", per_page)) + if placement_page is not None: + params.append(("placement_page", placement_page)) + if error is not None: + params.append(("error", error)) + if notice is not None: + params.append(("notice", notice)) + query = urlencode(params) + return f"/story-arcs/{story_arc_id}" + (f"?{query}" if query else "") + + +def _list_url(*, error: str | None = None) -> str: + return f"/story-arcs?{urlencode({'error': error})}" if error is not None else "/story-arcs" + + +def _add_url(*, error: str | None = None) -> str: + return ( + f"/story-arcs/add?{urlencode({'error': error})}" if error is not None else "/story-arcs/add" + ) + + +def _error_code( + exc: StoryArcServiceError | StoryArcPlacementIntegrationError | IntegrityError, +) -> str: + if isinstance(exc, StoryArcManualEditingDisabledError): + return "provider-managed" + if isinstance(exc, StoryArcProviderIdentityError): + return "provider-identity" + if isinstance(exc, (StoryArcConflictError, IntegrityError)): + return "conflict" + if isinstance(exc, StoryArcNotFoundError): + return "not-found" + if isinstance(exc, StoryArcPlacementIntegrationError): + return "placement" + return "validation" + + +def _optional_issue_id(value: str) -> int | None: + """Parse a browser's blank optional number field without turning it into a 422.""" + text = value.strip() + if not text: + return None + try: + issue_id = int(text) + except ValueError as exc: + raise StoryArcValidationError("Issue ID must be a positive integer") from exc + if issue_id < 1: + raise StoryArcValidationError("Issue ID must be a positive integer") + return issue_id + + +def resolve_story_arc_view(request: Request, view_mode: str | None) -> str: + """Resolve the active registry view from the query or saved preference.""" + if view_mode in _STORY_ARC_VIEW_MODES: + return view_mode + cookie_view = request.cookies.get("story_arc_view") + return cookie_view if cookie_view in _STORY_ARC_VIEW_MODES else "list" + + +def _placement_policy_input( + *, + mode: str, + target_library_root_id: str, + destination_root: str, + folder_template: str, + file_template: str, + symlink_style: str, + synchronize: bool, +) -> StoryArcPlacementPolicyInput: + """Normalize browser blanks while keeping the complete policy explicit.""" + try: + policy_mode = StoryArcPlacementPolicyMode(mode) + except ValueError as exc: + raise StoryArcPlacementIntegrationError( + "unsupported_mode", + "Unsupported Story Arc placement mode", + ) from exc + root_id = _optional_issue_id(target_library_root_id) + destination = destination_root.strip() or None + style = symlink_style.strip() or None + if policy_mode is StoryArcPlacementPolicyMode.LOGICAL: + root_id = None + destination = None + style = None + synchronize = False + elif policy_mode is not StoryArcPlacementPolicyMode.SYMLINK: + style = None + return StoryArcPlacementPolicyInput( + mode=policy_mode, + target_library_root_id=root_id, + destination_root=destination, + folder_template=folder_template, + file_template=file_template, + symlink_style=style, + synchronize=synchronize, + ) + + +async def _render_story_arc_detail( + *, + story_arc_id: int, + request: Request, + username: str, + user_id: int, + session: DbSession, + page: int, + per_page: int, + placement_page: int, + notice_message: str = "", + error_message: str = "", + placement_proposal: StoryArcPlacementPolicyInput | None = None, + placement_message: str = "", + reorder_preview: StoryArcReorderPreview | None = None, + reorder_message: str = "", +) -> Response: + if reorder_preview is None: + try: + reorder_preview = await _managed_reorder_service.load_pending_preview( + session, + story_arc_id, + ) + except StoryArcManagedReorderError: + # A malformed/incomplete journal is still important recovery truth + # even when it cannot safely mint a confirmation form. + error_message = error_message or _ERROR_MESSAGES["reorder-recovery"] + detail = await load_story_arc_detail( + session, + story_arc_id=story_arc_id, + page=page, + per_page=per_page, + user_id=user_id, + ) + if detail is None: + raise HTTPException(status_code=404, detail="Story arc not found") + placement_ui = ( + await load_story_arc_placement_context( + session, story_arc_id=story_arc_id, page=placement_page, proposal=placement_proposal + ) + if placement_proposal is not None + else None + ) + placement_policy = ( + placement_ui.policy + if placement_ui is not None + else await load_story_arc_placement_policy_view(session, story_arc_id) + ) + context = _ctx( + request, + _StoryArcTemplateUser(username=username), + story_arc=detail, + placement_ui=placement_ui, + placement_policy=placement_policy, + pagination_base_url=f"/story-arcs/{story_arc_id}?{urlencode({'per_page': per_page})}", + placement_pagination_base_url=( + f"/story-arcs/{story_arc_id}?{urlencode({'page': page, 'per_page': per_page})}" + ), + notice_message=notice_message, + error_message=error_message, + placement_message=placement_message, + reorder_preview=reorder_preview, + reorder_message=reorder_message, + show_placement_diagnostics=placement_proposal is not None, + ) + return _templates().TemplateResponse(request, "pages/story_arc_detail.html", context) + + +async def _nested_memberships( + session: DbSession, + *, + story_arc_id: int, + membership_id: int, +) -> tuple[list[IssueStoryArc], int]: + """Load stable order and reject cross-arc nested mutations.""" + memberships = await _story_arc_service.list_memberships(session, story_arc_id) + for index, membership in enumerate(memberships): + if membership.id == membership_id: + return memberships, index + raise StoryArcNotFoundError(f"Story-arc membership {membership_id} was not found") + + +async def _bounded_nested_membership( + session: DbSession, + *, + story_arc_id: int, + membership_id: int, +) -> IssueStoryArc: + """Load one membership while rejecting cross-arc nested access.""" + membership = await session.scalar( + select(IssueStoryArc).where( + IssueStoryArc.id == membership_id, + IssueStoryArc.story_arc_id == story_arc_id, + ) + ) + if membership is None: + raise HTTPException(status_code=404, detail="Story arc entry not found") + return membership + + +@router.get("/story-arcs", response_class=HTMLResponse, include_in_schema=False) +async def story_arc_list( + request: Request, + user: AuthenticatedUser, + session: DbSession, + q: Annotated[str | None, Query(max_length=500)] = None, + lifecycle: Annotated[str | None, Query(max_length=20)] = None, + monitored: Annotated[str | None, Query(max_length=5)] = None, + page: Annotated[int, Query(ge=1)] = 1, + per_page: Annotated[int, Query(ge=1, le=100)] = 25, + view_mode: str | None = Query(None), + error: str | None = Query(None), +) -> Response: + """Render a bounded, searchable Story Arc registry.""" + active_view = resolve_story_arc_view(request, view_mode) + lifecycle_value = ( + StoryArcLifecycle(lifecycle) + if lifecycle in {item.value for item in StoryArcLifecycle} + else None + ) + monitored_value = monitored == "true" if monitored in {"true", "false"} else None + result = await load_story_arc_list_page( + session, + q=q, + lifecycle=lifecycle_value, + monitored=monitored_value, + page=page, + per_page=per_page, + user_id=user.id, + ) + base_params: list[tuple[str, str | int]] = [ + ("per_page", per_page), + ("view_mode", active_view), + ] + if q is not None and q.strip(): + base_params.append(("q", q.strip())) + if lifecycle_value is not None: + base_params.append(("lifecycle", lifecycle_value.value)) + if monitored_value is not None: + base_params.append(("monitored", str(monitored_value).lower())) + context = _ctx( + request, + user, + story_arc_page=result, + query=q or "", + lifecycle_filter=lifecycle_value.value if lifecycle_value is not None else "", + monitored_filter=(str(monitored_value).lower() if monitored_value is not None else ""), + active_view=active_view, + pagination_base_url=f"/story-arcs?{urlencode(base_params)}", + error_message=_ERROR_MESSAGES.get(error or "", ""), + ) + template = ( + "partials/story_arc_results_bundle.html" + if request.headers.get("HX-Request") + else "pages/story_arcs.html" + ) + response = _templates().TemplateResponse(request, template, context) + if view_mode in _STORY_ARC_VIEW_MODES: + response.set_cookie( + "story_arc_view", + active_view, + max_age=31_536_000, + path="/", + samesite="lax", + ) + return response + + +@router.get("/story-arcs/add", response_class=HTMLResponse, include_in_schema=False) +async def story_arc_add( + request: Request, + user: AuthenticatedUser, + session: DbSession, + q: Annotated[str, Query(max_length=500)] = "", + page: Annotated[int, Query(ge=1, le=100)] = 1, + error: str | None = Query(None), +) -> Response: + """Render provider-first Story Arc discovery on its own add page.""" + template_user = _StoryArcTemplateUser(username=user.username) + manual_create_enabled = get_settings().story_arc_manual_create_enabled + search_context = await story_arc_catalog_routes.load_story_arc_catalog_search_context( + session, + q=q, + page=page, + base_url="/story-arcs/add", + session_factory=get_request_session_factory(request), + ) + search_context["error_message"] = _ERROR_MESSAGES.get(error or "", "") or str( + search_context["error_message"] + ) + context = _ctx( + request, + template_user, + **search_context, + arc_file_defaults=await load_story_arc_file_defaults(session), + story_arc_manual_create_enabled=manual_create_enabled, + ) + template = ( + "partials/story_arc_add_results_bundle.html" + if request.headers.get("HX-Request") + else "pages/story_arc_add.html" + ) + return _templates().TemplateResponse(request, template, context) + + +@router.post("/story-arcs", include_in_schema=False) +async def story_arc_create( + request: Request, + _user: AuthenticatedUser, + session: DbSession, + name: str = Form(""), + description: str = Form(""), + monitored: bool = Form(False), + search_missing: bool = Form(False), + include_upcoming: bool = Form(False), + file_defaults_fingerprint: Annotated[str, Form(max_length=128)] = "", +) -> Response: + """Create an empty arc with a snapshot of the global, source-preserving defaults.""" + if not get_settings().story_arc_manual_create_enabled: + raise HTTPException(status_code=404, detail="Not found") + try: + defaults = await load_story_arc_file_defaults(session) + if file_defaults_fingerprint and file_defaults_fingerprint != defaults.fingerprint: + raise StoryArcValidationError("Story Arc file defaults changed. Review them again.") + proposal = defaults.proposal() + # Validate before creating anything, so an invalid destination cannot leave an arc behind. + policy = await validate_story_arc_placement_policy_input(session, proposal, revision=1) + arc = await _story_arc_service.create( + session, + name=name, + description=description.strip() or None, + monitored=monitored, + search_missing=search_missing, + include_upcoming=include_upcoming, + source_kind=StoryArcSourceKind.PULLBOX, + ) + story_arc_id = arc.id + arc.target_library_root_id = policy.target_library_root_id + arc.policy_snapshot = policy.snapshot + arc.policy_schema_version = STORY_ARC_PLACEMENT_POLICY_SCHEMA_VERSION + arc.sync_enabled = policy.synchronize + await session.commit() + except (StoryArcServiceError, StoryArcPlacementIntegrationError, IntegrityError) as exc: + await session.rollback() + return _redirect(request, _add_url(error=_error_code(exc))) + return _redirect(request, _detail_url(story_arc_id, notice="created")) + + +@router.get("/story-arcs/{story_arc_id}", response_class=HTMLResponse, include_in_schema=False) +async def story_arc_detail( + story_arc_id: int, + request: Request, + user: AuthenticatedUser, + session: DbSession, + page: Annotated[int, Query(ge=1)] = 1, + per_page: Annotated[int, Query(ge=1, le=100)] = 25, + placement_page: Annotated[int, Query(ge=1)] = 1, + notice: str | None = Query(None), + error: str | None = Query(None), +) -> Response: + """Render arc metadata and one ordered, bounded membership page.""" + return await _render_story_arc_detail( + story_arc_id=story_arc_id, + request=request, + username=user.username, + user_id=user.id, + session=session, + page=page, + per_page=per_page, + placement_page=placement_page, + notice_message=_NOTICE_MESSAGES.get(notice or "", ""), + error_message=_ERROR_MESSAGES.get(error or "", ""), + ) + + +@router.post( + "/story-arcs/{story_arc_id}/placement-policy/preview", + response_class=HTMLResponse, + include_in_schema=False, +) +async def story_arc_placement_policy_preview( + story_arc_id: int, + request: Request, + user: AuthenticatedUser, + session: DbSession, + expected_revision: Annotated[int, Form(ge=1)], + mode: Annotated[str, Form(max_length=50)], + target_library_root_id: str = Form(""), + destination_root: Annotated[str, Form(max_length=1000)] = "", + folder_template: Annotated[str, Form(max_length=1024)] = DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + file_template: Annotated[str, Form(max_length=1024)] = DEFAULT_STORY_ARC_FILE_TEMPLATE, + symlink_style: Annotated[str, Form(max_length=50)] = "", + synchronize: bool = Form(False), +) -> Response: + """Render a bounded candidate preview without persisting policy or files.""" + del expected_revision # Preview is deliberately write-free; save enforces the revision. + try: + proposal = _placement_policy_input( + mode=mode, + target_library_root_id=target_library_root_id, + destination_root=destination_root, + folder_template=folder_template, + file_template=file_template, + symlink_style=symlink_style, + synchronize=synchronize, + ) + return await _render_story_arc_detail( + story_arc_id=story_arc_id, + request=request, + username=user.username, + user_id=user.id, + session=session, + page=1, + per_page=25, + placement_page=1, + placement_proposal=proposal, + placement_message="Preview only — no policy or files were changed.", + ) + except (StoryArcPlacementIntegrationError, StoryArcValidationError) as exc: + await session.rollback() + return await _render_story_arc_detail( + story_arc_id=story_arc_id, + request=request, + username=user.username, + user_id=user.id, + session=session, + page=1, + per_page=25, + placement_page=1, + placement_message=str(exc), + ) + + +@router.post("/story-arcs/{story_arc_id}/placement-policy", include_in_schema=False) +async def story_arc_placement_policy_update( + story_arc_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + expected_revision: Annotated[int, Form(ge=1)], + mode: Annotated[str, Form(max_length=50)], + target_library_root_id: str = Form(""), + destination_root: Annotated[str, Form(max_length=1000)] = "", + folder_template: Annotated[str, Form(max_length=1024)] = DEFAULT_STORY_ARC_FOLDER_TEMPLATE, + file_template: Annotated[str, Form(max_length=1024)] = DEFAULT_STORY_ARC_FILE_TEMPLATE, + symlink_style: Annotated[str, Form(max_length=50)] = "", + synchronize: bool = Form(False), +) -> Response: + """Freeze one complete policy using the arc revision shown by the form.""" + try: + await _placement_service.update_policy( + session, + story_arc_id, + expected_revision=expected_revision, + proposal=_placement_policy_input( + mode=mode, + target_library_root_id=target_library_root_id, + destination_root=destination_root, + folder_template=folder_template, + file_template=file_template, + symlink_style=symlink_style, + synchronize=synchronize, + ), + ) + except (StoryArcPlacementIntegrationError, StoryArcValidationError, IntegrityError): + await session.rollback() + return _redirect(request, _detail_url(story_arc_id, error="placement")) + return _redirect( + request, + _detail_url(story_arc_id, notice="placement-policy-updated"), + ) + + +@router.post( + "/story-arcs/{story_arc_id}/memberships/{membership_id}/placement-sync", + include_in_schema=False, +) +async def story_arc_placement_sync( + story_arc_id: int, + membership_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + adopt_identical_existing: bool = Form(False), +) -> Response: + """Synchronize one preview row without changing its canonical file.""" + try: + await _placement_service.sync_membership( + session, + story_arc_id, + membership_id, + adopt_identical_existing=adopt_identical_existing, + ) + except StoryArcPlacementIntegrationError: + await session.rollback() + return _redirect(request, _detail_url(story_arc_id, error="placement")) + return _redirect(request, _detail_url(story_arc_id, notice="placement-synchronized")) + + +@router.post( + "/story-arcs/{story_arc_id}/placements/{placement_id}/retry", + include_in_schema=False, +) +async def story_arc_placement_retry( + story_arc_id: int, + placement_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + adopt_identical_existing: bool = Form(False), +) -> Response: + """Retry one durable placement while preserving referenced ownership.""" + try: + await _placement_service.retry_placement( + session, + story_arc_id, + placement_id, + adopt_identical_existing=adopt_identical_existing, + ) + except StoryArcPlacementIntegrationError: + await session.rollback() + return _redirect(request, _detail_url(story_arc_id, error="placement")) + return _redirect(request, _detail_url(story_arc_id, notice="placement-retried")) + + +@router.post( + "/story-arcs/{story_arc_id}/placements/{placement_id}/repair", + include_in_schema=False, +) +async def story_arc_placement_repair( + story_arc_id: int, + placement_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, +) -> Response: + """Repair only a Pullbox-managed placement with durable ownership evidence.""" + try: + await _placement_service.repair_placement(session, story_arc_id, placement_id) + except StoryArcPlacementIntegrationError: + await session.rollback() + return _redirect(request, _detail_url(story_arc_id, error="placement")) + return _redirect(request, _detail_url(story_arc_id, notice="placement-repaired")) + + +@router.post( + "/story-arcs/{story_arc_id}/placements/{placement_id}/remove", + include_in_schema=False, +) +async def story_arc_placement_remove( + story_arc_id: int, + placement_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + confirm_managed_artifact_removal: bool = Form(False), +) -> Response: + """Remove managed artifacts or forget references under ownership safeguards.""" + try: + removed = await _placement_service.remove_placement( + session, + story_arc_id, + placement_id, + confirm_managed_artifact_removal=confirm_managed_artifact_removal, + ) + except StoryArcPlacementIntegrationError: + await session.rollback() + return _redirect(request, _detail_url(story_arc_id, error="placement")) + notice = ( + "placement-reference-forgotten" + if removed.referenced_artifact_preserved + else "placement-managed-removed" + ) + return _redirect(request, _detail_url(story_arc_id, notice=notice)) + + +@router.post("/story-arcs/{story_arc_id}/monitor", include_in_schema=False) +async def story_arc_monitor( + story_arc_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + expected_revision: Annotated[int, Form(ge=1)], + monitored: bool = Form(False), +) -> Response: + """Change monitoring without editing metadata, storage, or parent series.""" + try: + await _story_arc_service.update( + session, + story_arc_id, + expected_revision=expected_revision, + monitored=monitored, + ) + await session.commit() + except (StoryArcServiceError, IntegrityError) as exc: + await session.rollback() + return _redirect( + request, + _detail_url(story_arc_id, error=_error_code(exc)), + ) + return _redirect(request, _detail_url(story_arc_id, notice="updated")) + + +@router.post("/story-arcs/{story_arc_id}/search", include_in_schema=False) +async def story_arc_search( + story_arc_id: int, request: Request, _user: AuthenticatedUser, session: DbSession +) -> Response: + """Use the shared acquisition flow, scoped to this arc's eligible canonical issues.""" + from pullbox.tasks.story_arc_search_task import schedule_story_arc_search + + arc = await session.get(StoryArc, story_arc_id) + if arc is None or arc.lifecycle is not StoryArcLifecycle.ACTIVE: + raise HTTPException(status_code=404, detail="Active story arc not found") + await session.commit() + started = schedule_story_arc_search(story_arc_id) + return _redirect( + request, _detail_url(story_arc_id, notice="search-started" if started else "search-running") + ) + + +@router.post("/story-arcs/{story_arc_id}/memberships", include_in_schema=False) +async def story_arc_add_membership( + story_arc_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + sequence_number: Annotated[int, Form(ge=0)], + issue_id: str = Form(""), + source_issue_number_text: Annotated[str | None, Form(max_length=320)] = None, +) -> Response: + """Add one resolved or unresolved logical membership.""" + try: + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise StoryArcNotFoundError("Story arc not found") + require_manual_arc_edit(arc) + await _story_arc_service.add_membership( + session, + story_arc_id, + issue_id=_optional_issue_id(issue_id), + sequence_number=sequence_number, + source_issue_number_text=( + source_issue_number_text.strip() if source_issue_number_text else None + ), + source_kind=StoryArcSourceKind.PULLBOX, + ) + await session.commit() + except (StoryArcServiceError, IntegrityError) as exc: + await session.rollback() + return _redirect(request, _detail_url(story_arc_id, error=_error_code(exc))) + return _redirect(request, _detail_url(story_arc_id, notice="membership-added")) + + +@router.post( + "/story-arcs/{story_arc_id}/memberships/{membership_id}/move", + include_in_schema=False, +) +async def story_arc_move_membership( + story_arc_id: int, + membership_id: int, + request: Request, + user: AuthenticatedUser, + session: DbSession, + direction: Annotated[Literal["up", "down"], Form()], + expected_revision: Annotated[int, Form(ge=1)], + return_page: Annotated[int | None, Form(ge=1)] = None, + return_per_page: Annotated[int | None, Form(ge=1, le=100)] = None, + preview_token: Annotated[str, Form(max_length=200_000)] = "", +) -> Response: + """Apply a chevron move through the same durable, source-preserving plan.""" + username, user_id = user.username, user.id + inline = request.headers.get("HX-Target") == "story-arc-detail-page" + notice, error = "", "" + page, per_page = return_page or 1, return_per_page or 25 + try: + if not preview_token: + preview = await _managed_reorder_service.preview_adjacent_move( + session, + story_arc_id, + membership_id, + direction=direction, + expected_revision=expected_revision, + ) + preview_token = preview.preview_token + await _managed_reorder_service.confirm_adjacent_move( + session, + story_arc_id=story_arc_id, + membership_id=membership_id, + direction=direction, + expected_revision=expected_revision, + preview_token=preview_token, + ) + notice = f"moved-{direction}" + # Follow the moved member across a pagination boundary using saved order, + # not a client-supplied row number or potentially sparse sequence value. + ranked = ( + select( + IssueStoryArc.id, + func.row_number() + .over( + order_by=( + IssueStoryArc.sequence_number, + IssueStoryArc.source_ordinal, + IssueStoryArc.id, + ) + ) + .label("position"), + ) + .where(IssueStoryArc.story_arc_id == story_arc_id) + .subquery() + ) + position = await session.scalar( + select(ranked.c.position).where(ranked.c.id == membership_id) + ) + if position is not None: + page = (int(position) - 1) // per_page + 1 + except StoryArcManagedReorderError as exc: + await session.rollback() + if exc.code in {"already_first", "already_last"}: + notice = exc.code.replace("_", "-") + else: + error = ( + "conflict" + if exc.category == "conflict" + else "not-found" + if exc.category == "not_found" + else "reorder-recovery" + if exc.category == "recovery" + else "reorder" + ) + except IntegrityError: + await session.rollback() + error = "conflict" + + url = _detail_url( + story_arc_id, + notice=notice or None, + error=error or None, + page=page if inline or return_page is not None or return_per_page is not None else None, + per_page=per_page if inline or return_per_page is not None else None, + ) + if not inline: + return _redirect(request, url) + response = await _render_story_arc_detail( + story_arc_id=story_arc_id, + request=request, + username=username, + user_id=user_id, + session=session, + page=page, + per_page=per_page, + placement_page=1, + error_message=_ERROR_MESSAGES.get(error, ""), + reorder_message=_NOTICE_MESSAGES.get(notice, ""), + ) + response.headers["HX-Replace-Url"] = _detail_url(story_arc_id, page=page, per_page=per_page) + response.headers["HX-Trigger-After-Settle"] = json.dumps( + { + "story-arc-reordered": { + "membershipId": membership_id, + "direction": direction, + "pageChanged": page != (return_page or 1), + } + } + ) + return response + + +@router.post( + "/story-arcs/{story_arc_id}/memberships/{membership_id}/resolve", + include_in_schema=False, +) +async def story_arc_resolve_membership( + story_arc_id: int, + membership_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + issue_id: Annotated[int, Form(ge=1)], + return_page: Annotated[int | None, Form(ge=1)] = None, + return_per_page: Annotated[int | None, Form(ge=1, le=100)] = None, +) -> Response: + """Resolve one entry to an existing canonical issue record.""" + try: + await _nested_memberships( + session, + story_arc_id=story_arc_id, + membership_id=membership_id, + ) + await _story_arc_service.resolve_membership(session, membership_id, issue_id=issue_id) + await session.commit() + except (StoryArcServiceError, IntegrityError) as exc: + await session.rollback() + return _redirect( + request, + _detail_url( + story_arc_id, + error=_error_code(exc), + page=return_page, + per_page=return_per_page, + ), + ) + return _redirect( + request, + _detail_url( + story_arc_id, + notice="resolved", + page=return_page, + per_page=return_per_page, + ), + ) + + +@router.get( + "/story-arcs/{story_arc_id}/memberships/{membership_id}/local-issues", + response_class=HTMLResponse, + include_in_schema=False, +) +async def story_arc_local_issue_search( + story_arc_id: int, + membership_id: int, + request: Request, + user: AuthenticatedUser, + session: DbSession, + q: Annotated[str, Query(min_length=1, max_length=500)], + return_page: Annotated[int, Query(ge=1)] = 1, + return_per_page: Annotated[int, Query(ge=1, le=100)] = 25, +) -> Response: + """Return a bounded, provider-free local issue picker for one entry.""" + membership = await _bounded_nested_membership( + session, + story_arc_id=story_arc_id, + membership_id=membership_id, + ) + result = await search_story_arc_local_issues( + session, + query=q, + source_series_name=membership.source_series_name, + source_issue_number_text=membership.source_issue_number_text, + ) + return _templates().TemplateResponse( + request, + "partials/story_arc_local_issue_results.html", + _ctx( + request, + user, + story_arc_id=story_arc_id, + membership_id=membership_id, + result=result, + return_page=return_page, + return_per_page=return_per_page, + ), + ) + + +@router.post( + "/story-arcs/{story_arc_id}/memberships/{membership_id}/remove", + include_in_schema=False, +) +async def story_arc_remove_membership( + story_arc_id: int, + membership_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + return_page: Annotated[int | None, Form(ge=1)] = None, + return_per_page: Annotated[int | None, Form(ge=1, le=100)] = None, +) -> Response: + """Remove an association while preserving its canonical issue and file.""" + try: + await _bounded_nested_membership( + session, + story_arc_id=story_arc_id, + membership_id=membership_id, + ) + arc = await session.get(StoryArc, story_arc_id) + if arc is None: + raise StoryArcNotFoundError("Story arc not found") + require_manual_arc_edit(arc) + await _story_arc_service.remove_membership(session, membership_id) + await session.commit() + except (StoryArcServiceError, IntegrityError) as exc: + await session.rollback() + return _redirect( + request, + _detail_url( + story_arc_id, + error=_error_code(exc), + page=return_page, + per_page=return_per_page, + ), + ) + return _redirect( + request, + _detail_url( + story_arc_id, + notice="membership-removed", + page=return_page, + per_page=return_per_page, + ), + ) + + +@router.post("/story-arcs/{story_arc_id}/archive", include_in_schema=False) +async def story_arc_archive( + story_arc_id: int, + request: Request, + _user: AuthenticatedUser, + session: DbSession, + expected_revision: Annotated[int, Form(ge=1)], +) -> Response: + """Soft-archive an arc without deleting canonical issues, files, or memberships.""" + try: + await _story_arc_service.archive( + session, + story_arc_id, + expected_revision=expected_revision, + ) + await session.commit() + except (StoryArcServiceError, IntegrityError) as exc: + await session.rollback() + return _redirect(request, _detail_url(story_arc_id, error=_error_code(exc))) + return _redirect(request, _detail_url(story_arc_id, notice="archived")) diff --git a/src/pullbox/ui/templates/base.html b/src/pullbox/ui/templates/base.html index b0278226..316d8328 100644 --- a/src/pullbox/ui/templates/base.html +++ b/src/pullbox/ui/templates/base.html @@ -286,15 +286,8 @@ - {#- Global primary action -#} - - - - + {#- Global catalog creation action -#} + {% include "components/header_add_menu.html" %} {#- Page-specific header actions -#} {% block header_actions %}{% endblock %} @@ -988,6 +981,8 @@

+ + diff --git a/src/pullbox/ui/templates/components/comicvine_result_card.html b/src/pullbox/ui/templates/components/comicvine_result_card.html new file mode 100644 index 00000000..7ed2a321 --- /dev/null +++ b/src/pullbox/ui/templates/components/comicvine_result_card.html @@ -0,0 +1,26 @@ +{% macro comicvine_result_card(card_id, testid, is_static=false) %} +
+ {{ caller("cover") }} +
+
+
+ {{ caller("details") }} +
+
+ {{ caller("actions") }} +
+
+
+
+{% endmacro %} + +{% macro comicvine_in_library_badge() %} + + + In Library + +{% endmacro %} diff --git a/src/pullbox/ui/templates/components/comicvine_result_cover.html b/src/pullbox/ui/templates/components/comicvine_result_cover.html new file mode 100644 index 00000000..b5ff6471 --- /dev/null +++ b/src/pullbox/ui/templates/components/comicvine_result_cover.html @@ -0,0 +1,28 @@ +{% macro comicvine_result_cover(cover_url, title, href=none, link_testid=none) -%} + +{%- endmacro %} diff --git a/src/pullbox/ui/templates/components/comicvine_search_loading.html b/src/pullbox/ui/templates/components/comicvine_search_loading.html new file mode 100644 index 00000000..a913ebf1 --- /dev/null +++ b/src/pullbox/ui/templates/components/comicvine_search_loading.html @@ -0,0 +1,21 @@ +{% macro comicvine_search_loading(element_id, testid, local_catalog=false) -%} +
+
+ +
+

{{ 'Searching local catalog' if local_catalog else 'Searching ComicVine' }}

+

Large catalogs can take a moment.

+
+
+
+{%- endmacro %} diff --git a/src/pullbox/ui/templates/components/dropdown_select.html b/src/pullbox/ui/templates/components/dropdown_select.html index e0dbbd44..3bc643ff 100644 --- a/src/pullbox/ui/templates/components/dropdown_select.html +++ b/src/pullbox/ui/templates/components/dropdown_select.html @@ -46,6 +46,8 @@ wrap_options=false, local_model="", local_on_change="", + options_expression="", + disabled_expression="", on_change="" ) %} {%- set normalized_size = namespace(value=size) -%} @@ -83,14 +85,16 @@ diff --git a/src/pullbox/ui/templates/components/header_add_menu.html b/src/pullbox/ui/templates/components/header_add_menu.html new file mode 100644 index 00000000..d4a5d2d5 --- /dev/null +++ b/src/pullbox/ui/templates/components/header_add_menu.html @@ -0,0 +1,167 @@ +{#- Persistent selector plus action for creating catalog content. -#} +
+ + + + + +
diff --git a/src/pullbox/ui/templates/components/import_log_viewer.html b/src/pullbox/ui/templates/components/import_log_viewer.html index 682ff4d4..9939166e 100644 --- a/src/pullbox/ui/templates/components/import_log_viewer.html +++ b/src/pullbox/ui/templates/components/import_log_viewer.html @@ -62,6 +62,7 @@ total_pages_expr="totalPages", prev_action="prevPage()", next_action="nextPage()", + pagination_status_text_expr="footerStatusText", footer_status_text_expr="footerStatusText", footer_status_show_expr="isLive", footer_status_label_expr="'Live updates'", diff --git a/src/pullbox/ui/templates/components/log_viewer.html b/src/pullbox/ui/templates/components/log_viewer.html index 32d07c74..5165ca30 100644 --- a/src/pullbox/ui/templates/components/log_viewer.html +++ b/src/pullbox/ui/templates/components/log_viewer.html @@ -69,6 +69,7 @@ total_pages_expr="totalPages", prev_action="prevPage()", next_action="nextPage()", + pagination_status_text_expr="", footer_status_text_expr="", footer_status_show_expr="", footer_status_label_expr="", @@ -231,7 +232,10 @@ {% if body_testid %}data-testid="{{ body_testid }}"{% endif %} {% if body_scroll_action %}@scroll="{{ body_scroll_action }}"{% endif %} > -