From 86cd81f3a31713df995e3150120c902aa934c0b3 Mon Sep 17 00:00:00 2001 From: Sanjin <102841251+duckduckgrayduck@users.noreply.github.com> Date: Tue, 22 Sep 2026 10:37:56 -0500 Subject: [PATCH 1/3] Add a page count to documentcloud uploads --- config/aws/lambda/template_params.yaml | 1 + .../processing/info_and_image/main.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/config/aws/lambda/template_params.yaml b/config/aws/lambda/template_params.yaml index 8e03318d..8c6041d3 100644 --- a/config/aws/lambda/template_params.yaml +++ b/config/aws/lambda/template_params.yaml @@ -38,6 +38,7 @@ Globals: SIDEKICK_PREPROCESS_TOPIC: "{{resolve:ssm:/dc/{$ENV$}/lambdas/sidekick_preprocess/topic:latest}}" RETRY_ERROR_TOPIC: "{{resolve:ssm:/dc/{$ENV$}/lambdas/retry_errors/topic:latest}}" PDF_SIZE_LIMIT: "{{resolve:ssm:/dc/{$ENV$}/params/pdf_size_limit:latest}}" + PAGE_COUNT_LIMIT: "{{resolve:ssm:/dc/{$ENV$}/params/page_count_limit:latest}}" BLOCK_SIZE: "{{resolve:ssm:/dc/{$ENV$}/params/block_size:latest}}" IMPORT_DIR: "{{resolve:ssm:/dc/{$ENV$}/lambdas/config/import_dir:latest}}" IMPORT_BUCKET: "{{resolve:ssm:/dc/{$ENV$}/lambdas/config/import_bucket:latest}}" diff --git a/documentcloud/documents/processing/info_and_image/main.py b/documentcloud/documents/processing/info_and_image/main.py index 6290b426..19a19e44 100755 --- a/documentcloud/documents/processing/info_and_image/main.py +++ b/documentcloud/documents/processing/info_and_image/main.py @@ -91,7 +91,12 @@ TEXT_POSITION_BATCH = env.int( "TEXT_POSITION_BATCH", 3 ) # Number of pages to pull text positions from with each function + +# PDF upload size limit is 500MB PDF_SIZE_LIMIT = env.int("PDF_SIZE_LIMIT", 501 * 1024 * 1024) +# PDF page count limit +PAGE_COUNT_LIMIT = env.int("PAGE_COUNT_LIMIT", 2000) + BLOCK_SIZE = env.int( "BLOCK_SIZE", 8 * 1024 * 1024 ) # Block size to use for reading chunks of the PDF @@ -170,6 +175,10 @@ class PdfSizeError(Exception): pass +class PageCountError(Exception): + pass + + def initialize_redis_page_data(doc_id, page_count): """Initialize Redis fields to manage page dimensions and processing""" dimensions_field = redis_fields.dimensions(doc_id) @@ -719,6 +728,15 @@ def process_pdf(data, _context=None): # Extract the page count and store it in Redis page_count = extract_pagecount(doc_id, slug) + + # Ensure page count is within the limit + if page_count > PAGE_COUNT_LIMIT: + # If it is beyond the limit + # delete it from storage + # and raise PageCountError + storage.delete(path.path(doc_id)) + raise PageCountError() + initialize_redis_page_data(doc_id, page_count) # Update the model with the page count From 31172e610624420726cc5685181c1f29bb47c9c3 Mon Sep 17 00:00:00 2001 From: Sanjin <102841251+duckduckgrayduck@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:36:05 -0500 Subject: [PATCH 2/3] Add message to PageCountError so that it populates in errors on the document --- documentcloud/documents/processing/info_and_image/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/documentcloud/documents/processing/info_and_image/main.py b/documentcloud/documents/processing/info_and_image/main.py index 19a19e44..25ee6220 100755 --- a/documentcloud/documents/processing/info_and_image/main.py +++ b/documentcloud/documents/processing/info_and_image/main.py @@ -94,7 +94,7 @@ # PDF upload size limit is 500MB PDF_SIZE_LIMIT = env.int("PDF_SIZE_LIMIT", 501 * 1024 * 1024) -# PDF page count limit +# PDF page count limit, default 2000 PAGE_COUNT_LIMIT = env.int("PAGE_COUNT_LIMIT", 2000) BLOCK_SIZE = env.int( @@ -735,7 +735,10 @@ def process_pdf(data, _context=None): # delete it from storage # and raise PageCountError storage.delete(path.path(doc_id)) - raise PageCountError() + raise PageCountError( + f"This document has {page_count} pages, which exceeds the " + f"{PAGE_COUNT_LIMIT}-page limit." + ) initialize_redis_page_data(doc_id, page_count) From 5746027576f78ad478442d561a8ea93ebe38307b Mon Sep 17 00:00:00 2001 From: Sanjin <102841251+duckduckgrayduck@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:23:28 -0500 Subject: [PATCH 3/3] Add PDFSizeError message so it shows on documents/pk/errors and provide guidance --- .../documents/processing/info_and_image/main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/documentcloud/documents/processing/info_and_image/main.py b/documentcloud/documents/processing/info_and_image/main.py index 25ee6220..3e9afd1e 100755 --- a/documentcloud/documents/processing/info_and_image/main.py +++ b/documentcloud/documents/processing/info_and_image/main.py @@ -719,7 +719,13 @@ def process_pdf(data, _context=None): if storage.size(doc_path) > PDF_SIZE_LIMIT: # If not, remove the PDF storage.delete(path.path(doc_id)) - raise PdfSizeError() + # Display 500MB instead of 501MB so -1 + raise PdfSizeError( + f"This document is larger than the " + f"{(PDF_SIZE_LIMIT // (1024 * 1024)) - 1}MB limit. " + f"Please either compress or split the PDF to conform" + f" to the limit." + ) # files are always uploaded to S3 as private, set to public on S3 # if uploaded publicly to DocumentCloud @@ -737,7 +743,8 @@ def process_pdf(data, _context=None): storage.delete(path.path(doc_id)) raise PageCountError( f"This document has {page_count} pages, which exceeds the " - f"{PAGE_COUNT_LIMIT}-page limit." + f"{PAGE_COUNT_LIMIT}-page limit. " + f"Please split the PDF before upload." ) initialize_redis_page_data(doc_id, page_count)