From af7950ae4e9e2e9ebe7cd1c6d0b85cd618ceef26 Mon Sep 17 00:00:00 2001 From: Luka Pajukanta Date: Tue, 7 Jul 2026 16:41:55 +0300 Subject: [PATCH 1/7] feat: pseudonymize db --- .../management/commands/pseudonymize_db.py | 299 ++++++++++++++++++ scripts/make-pseudonymized-dump.sh | 41 +++ 2 files changed, 340 insertions(+) create mode 100644 kompassi/core/management/commands/pseudonymize_db.py create mode 100755 scripts/make-pseudonymized-dump.sh diff --git a/kompassi/core/management/commands/pseudonymize_db.py b/kompassi/core/management/commands/pseudonymize_db.py new file mode 100644 index 000000000..0e90055ce --- /dev/null +++ b/kompassi/core/management/commands/pseudonymize_db.py @@ -0,0 +1,299 @@ +import logging +from datetime import date + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +logger = logging.getLogger(__name__) + +INVALID_SSN = "000000A0000" + +# Finnish SSNs and Finnish PIA-protected identifiers — replace with clearly invalid placeholder. +FORM_DATA_SSN_SLUGS = frozenset( + [ + "hetu", + "henkilotunnus", + "ssn", + "social_security_number", + "socialSecurityNumber", + "personal_identification_number", + "personalIdentificationNumber", + ] +) + +# Standard PII + GDPR Art. 9 special category health data — clear to empty string. +FORM_DATA_CLEAR_SLUGS = frozenset( + [ + "first_name", + "firstName", + "last_name", + "lastName", + "official_first_names", + "name", + "email", + "phone", + "phone_number", + "phoneNumber", + "address", + "discord", + "discord_handle", + "discordHandle", + "nick", + # Special diet / health data (GDPR Art. 9) + "special_diet_other", + "erityisruokavalio", + "allergies", + "dietary_restrictions", + "dietaryRestrictions", + "special_diets", + ] +) + + +class Command(BaseCommand): + help = ( + "Pseudonymize all personal data in the database. " + "Intended to run against a temporary copy of the production database " + "created by scripts/make-pseudonymized-dump.sh. " + "The database name must contain 'pseudo'. " + "Note: form_data PII scrubbing is best-effort based on known field slugs; " + "custom slugs outside the known list will not be scrubbed." + ) + + def add_arguments(self, parser): + parser.add_argument( + "--yes", + action="store_true", + dest="yes", + help="Confirm that you want to pseudonymize this database.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + dest="dry_run", + help="Print counts of affected records without writing anything.", + ) + + def handle(self, *args, **options): + self._check_safety(options["yes"]) + + if options["dry_run"]: + self.stdout.write(self.style.WARNING("DRY RUN — no changes will be written.")) + self._report_counts() + return + + with transaction.atomic(): + self._pseudonymize_persons() + self._pseudonymize_tickets_v2_orders() + self._pseudonymize_forms_responses() + self._pseudonymize_involvement_invitations() + self._pseudonymize_signup_extras() + self._pseudonymize_zombie_enrollment() + self._pseudonymize_zombie_tickets_customers() + self._pseudonymize_zombie_programme_feedback() + self._delete_one_time_tokens() + self._delete_smtp_passwords() + self._delete_keypairs() + self._delete_email_aliases() + + self.stdout.write(self.style.SUCCESS("Pseudonymization complete.")) + + def _check_safety(self, yes: bool) -> None: + if not yes: + raise CommandError("Pass --yes to confirm you want to pseudonymize this database.") + db_name = settings.DATABASES["default"]["NAME"] + if "pseudo" not in db_name: + raise CommandError( + f"Database name {db_name!r} does not contain 'pseudo'. " + "This command is intended to run against a temporary copy created by " + "scripts/make-pseudonymized-dump.sh. " + "If you set up the temp database manually, rename it to include 'pseudo'." + ) + + def _pseudonymize_persons(self) -> None: + from kompassi.core.models import Person + + User = get_user_model() + count = 0 + + for person in Person.objects.select_related("user").iterator(chunk_size=500): + pk = person.pk + person.first_name = f"Person{pk}" + person.official_first_names = f"Person{pk}" + person.surname = "Testinen" + person.nick = f"person{pk}" if person.nick else "" + person.discord_handle = f"person{pk}" if person.discord_handle else "" + person.email = f"person{pk}@example.com" + person.phone = f"+358000{pk:06d}" + person.muncipality = "Testilä" + person.notes = "" + if person.birth_date: + person.birth_date = date(person.birth_date.year, 1, 1) + # person.save() syncs first_name/surname/email to person.user + person.save() + if person.user_id: + person.user.set_unusable_password() + person.user.save(update_fields=["password"]) + count += 1 + + orphan_count = User.objects.filter(person__isnull=True).update(first_name="", last_name="", email="") + for user in User.objects.filter(person__isnull=True).iterator(chunk_size=500): + user.set_unusable_password() + user.save(update_fields=["password"]) + + self.stdout.write(f" Persons: {count}, orphan users: {orphan_count}") + + def _pseudonymize_tickets_v2_orders(self) -> None: + from kompassi.tickets_v2.models.order import Order + from kompassi.tickets_v2.models.receipt import Receipt + + Order.objects.update(first_name="Order", last_name="Testinen", phone="") + for order in Order.objects.only("id", "order_number").iterator(chunk_size=1000): + Order.objects.filter(id=order.id).update(email=f"order{order.order_number}@example.com") + + receipt_count = Receipt.objects.update(email="") + self.stdout.write(f" Orders: {Order.objects.count()}, receipts cleared: {receipt_count}") + + def _pseudonymize_forms_responses(self) -> None: + from kompassi.forms.models.response import Response + + Response.objects.update(ip_address="127.0.0.1", cached_key_fields={}) + + count = 0 + scrubbed = 0 + for response in Response.objects.only("id", "form_data").iterator(chunk_size=500): + new_data = dict(response.form_data) + changed = False + for slug in FORM_DATA_SSN_SLUGS: + if slug in new_data: + new_data[slug] = INVALID_SSN + changed = True + for slug in FORM_DATA_CLEAR_SLUGS: + if slug in new_data: + new_data[slug] = "" + changed = True + if changed: + response.form_data = new_data + response.save(update_fields=["form_data"]) + scrubbed += 1 + count += 1 + + self.stdout.write(f" Responses: {count} processed, {scrubbed} form_data scrubbed") + + def _pseudonymize_involvement_invitations(self) -> None: + from kompassi.involvement.models.invitation import Invitation + + count = 0 + for invitation in Invitation.objects.only("id").iterator(chunk_size=500): + Invitation.objects.filter(id=invitation.id).update(email=f"invitation-{str(invitation.id)[:8]}@example.com") + count += 1 + + self.stdout.write(f" Invitations: {count}") + + def _pseudonymize_signup_extras(self) -> None: + from django.apps import apps + + from kompassi.labour.models.signup_extras import SignupExtraMixin + + for model in apps.get_models(): + if not (isinstance(model, type) and issubclass(model, SignupExtraMixin)): + continue + if model.get_special_diet_field(): + for instance in model.objects.iterator(chunk_size=500): + instance.special_diet.clear() # type: ignore[attr-defined] + self.stdout.write(f" {model.__name__} special_diet cleared") + if model.get_special_diet_other_field(): + count = model.objects.update(special_diet_other="") + self.stdout.write(f" {model.__name__} special_diet_other cleared: {count}") + + def _pseudonymize_zombie_enrollment(self) -> None: + from kompassi.zombies.enrollment.models.enrollment import Enrollment + + count = Enrollment.objects.update( + personal_identification_number=INVALID_SSN, + address="", + zip_code="", + city="", + ) + for enrollment in Enrollment.objects.iterator(chunk_size=500): + enrollment.special_diet.clear() + + self.stdout.write(f" Zombie enrollments: {count}") + + def _pseudonymize_zombie_tickets_customers(self) -> None: + from kompassi.zombies.tickets.models.LEGACY_TICKETSV1_customer import Customer + + count = 0 + for customer in Customer.objects.only("pk").iterator(chunk_size=500): + Customer.objects.filter(pk=customer.pk).update( + first_name="Customer", + last_name="Testinen", + email=f"customer{customer.pk}@example.com", + phone_number="", + ) + count += 1 + + self.stdout.write(f" Legacy customers: {count}") + + def _pseudonymize_zombie_programme_feedback(self) -> None: + from kompassi.zombies.programme.models.programme_feedback import ProgrammeFeedback + + count = ProgrammeFeedback.objects.update(author_ip_address="127.0.0.1", author_external_username="") + self.stdout.write(f" Programme feedback: {count}") + + def _delete_one_time_tokens(self) -> None: + from kompassi.core.models.email_verification_token import EmailVerificationToken + from kompassi.core.models.password_reset_token import PasswordResetToken + from kompassi.tickets_v2.models.order_cancellation_token import OrderCancellationToken + + n1, _ = EmailVerificationToken.objects.all().delete() + n2, _ = PasswordResetToken.objects.all().delete() + n3, _ = OrderCancellationToken.objects.all().delete() + self.stdout.write(f" Deleted tokens: {n1} email verification, {n2} password reset, {n3} order cancellation") + + def _delete_smtp_passwords(self) -> None: + from kompassi.access.models.smtp_password import SMTPPassword + + count, _ = SMTPPassword.objects.all().delete() + self.stdout.write(f" Deleted SMTP passwords: {count}") + + def _delete_keypairs(self) -> None: + from kompassi.forms.models.keypair import KeyPair + + count, _ = KeyPair.objects.all().delete() + self.stdout.write(f" Deleted keypairs: {count}") + + def _delete_email_aliases(self) -> None: + from kompassi.access.models.email_alias import EmailAlias + + count, _ = EmailAlias.objects.all().delete() + self.stdout.write(f" Deleted email aliases: {count}") + + def _report_counts(self) -> None: + from kompassi.access.models.email_alias import EmailAlias + from kompassi.access.models.smtp_password import SMTPPassword + from kompassi.core.models import Person + from kompassi.core.models.email_verification_token import EmailVerificationToken + from kompassi.core.models.password_reset_token import PasswordResetToken + from kompassi.forms.models.keypair import KeyPair + from kompassi.forms.models.response import Response + from kompassi.involvement.models.invitation import Invitation + from kompassi.tickets_v2.models.order import Order + from kompassi.tickets_v2.models.order_cancellation_token import OrderCancellationToken + from kompassi.zombies.enrollment.models.enrollment import Enrollment + from kompassi.zombies.tickets.models.LEGACY_TICKETSV1_customer import Customer + + self.stdout.write(f" Persons: {Person.objects.count()}") + self.stdout.write(f" Orders: {Order.objects.count()}") + self.stdout.write(f" Responses: {Response.objects.count()}") + self.stdout.write(f" Invitations: {Invitation.objects.count()}") + self.stdout.write(f" Zombie enrollments: {Enrollment.objects.count()}") + self.stdout.write(f" Legacy customers: {Customer.objects.count()}") + self.stdout.write(f" Email verification tokens: {EmailVerificationToken.objects.count()}") + self.stdout.write(f" Password reset tokens: {PasswordResetToken.objects.count()}") + self.stdout.write(f" Order cancellation tokens: {OrderCancellationToken.objects.count()}") + self.stdout.write(f" SMTP passwords: {SMTPPassword.objects.count()}") + self.stdout.write(f" Keypairs: {KeyPair.objects.count()}") + self.stdout.write(f" Email aliases: {EmailAlias.objects.count()}") diff --git a/scripts/make-pseudonymized-dump.sh b/scripts/make-pseudonymized-dump.sh new file mode 100755 index 000000000..d0f804d45 --- /dev/null +++ b/scripts/make-pseudonymized-dump.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Usage: scripts/make-pseudonymized-dump.sh +# +# Creates a pseudonymized dump of the production database. +# Run on the production server (SSH in or docker exec into the app container). +# +# Authentication uses PostgreSQL standard environment variables and ~/.pgpass — +# set PGHOST, PGPORT, PGUSER, PGPASSWORD as needed, or configure ~/.pgpass. +# PGDATABASE must be set to the production database name. +# +# The pseudonymized dump is written to /tmp and its path is printed at the end. +set -euo pipefail + +ORIG_DB="${PGDATABASE:?PGDATABASE must be set to the production database name}" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +PSEUDO_DB="${ORIG_DB}_pseudo_${TIMESTAMP}" +OUTPUT_FILE="/tmp/${PSEUDO_DB}.sql.gz" + +echo "==> Creating temp database ${PSEUDO_DB}" +createdb "${PSEUDO_DB}" + +echo "==> Copying ${ORIG_DB} -> ${PSEUDO_DB} (pg_dump -Fc | pg_restore)" +# Binary custom format: compressed, faster than plain SQL, safe for same-server copies. +# For large databases with a maintenance window available, replace this with: +# createdb -T "${ORIG_DB}" "${PSEUDO_DB}" +# For maximum throughput without downtime, use directory format with --jobs=N instead of piping. +pg_dump --format=custom "${ORIG_DB}" \ + | pg_restore --dbname="${PSEUDO_DB}" --no-owner --no-privileges + +echo "==> Pseudonymizing ${PSEUDO_DB}" +# Override POSTGRES_DATABASE so Django connects to the temp copy, not the original. +# PGDATABASE is intentionally left unchanged so pg_* tools still default to the original DB. +POSTGRES_DATABASE="${PSEUDO_DB}" python manage.py pseudonymize_db --yes + +echo "==> Dumping pseudonymized database" +pg_dump "${PSEUDO_DB}" | gzip >"${OUTPUT_FILE}" + +echo "==> Cleaning up ${PSEUDO_DB}" +dropdb "${PSEUDO_DB}" + +echo "Done. Pseudonymized dump: ${OUTPUT_FILE}" From 993cb981b44103ba08afff72a0b4c2f5782adf51 Mon Sep 17 00:00:00 2001 From: Luka Pajukanta Date: Tue, 29 Sep 2026 20:41:32 +0300 Subject: [PATCH 2/7] feat(core): rule-based pseudonymization with in-cluster dump and local loader pseudonymize_db now applies declarative rules from core/pseudonymization.py, and a test fails when a field that may hold personal data is neither covered by a rule nor reviewed as safe. Tables that no installed model owns are dropped. scripts/pseudonymized-dump.sh pseudonymizes a copy of the deployed database inside a short-lived pod and streams out only the result; scripts/load-dump.sh loads it into the docker compose database and recreates the development credentials. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 9 + .../management/commands/pseudonymize_db.py | 294 +---------- kompassi/core/pseudonymization.py | 471 ++++++++++++++++++ kompassi/core/test_pseudonymization.py | 101 ++++ scripts/load-dump.sh | 42 ++ scripts/make-pseudonymized-dump.sh | 41 -- scripts/pseudonymized-dump.sh | 102 ++++ 7 files changed, 745 insertions(+), 315 deletions(-) create mode 100644 kompassi/core/pseudonymization.py create mode 100644 kompassi/core/test_pseudonymization.py create mode 100755 scripts/load-dump.sh delete mode 100755 scripts/make-pseudonymized-dump.sh create mode 100755 scripts/pseudonymized-dump.sh diff --git a/CLAUDE.md b/CLAUDE.md index c9b8d7616..bd457abe2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,15 @@ export NEXT_PUBLIC_KOMPASSI_BASE_URL=http://localhost:8000 npm run dev # starts Next.js at localhost:3000 with GraphQL codegen watch ``` +To develop against production data, fetch a pseudonymized copy and load it into the docker compose database (replaces it; `mahti`/`mahti` is recreated): + +```bash +scripts/pseudonymized-dump.sh > kompassi.pgdump # needs kubectl access to kompassi-production +scripts/load-dump.sh kompassi.pgdump +``` + +Never load a raw production dump locally. The pseudonymization rules live in `kompassi/core/pseudonymization.py`; a new model field that could hold personal data fails `kompassi/core/test_pseudonymization.py` until it has a rule or an entry in `NOT_PERSONAL` (`python manage.py pseudonymize_db --check` lists them). + ## Backend commands All backend commands assume the virtualenv is active (`source .venv/bin/activate`) or are run via Docker. diff --git a/kompassi/core/management/commands/pseudonymize_db.py b/kompassi/core/management/commands/pseudonymize_db.py index 0e90055ce..b0eeef66d 100644 --- a/kompassi/core/management/commands/pseudonymize_db.py +++ b/kompassi/core/management/commands/pseudonymize_db.py @@ -1,299 +1,45 @@ -import logging -from datetime import date - from django.conf import settings -from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError from django.db import transaction -logger = logging.getLogger(__name__) - -INVALID_SSN = "000000A0000" - -# Finnish SSNs and Finnish PIA-protected identifiers — replace with clearly invalid placeholder. -FORM_DATA_SSN_SLUGS = frozenset( - [ - "hetu", - "henkilotunnus", - "ssn", - "social_security_number", - "socialSecurityNumber", - "personal_identification_number", - "personalIdentificationNumber", - ] -) - -# Standard PII + GDPR Art. 9 special category health data — clear to empty string. -FORM_DATA_CLEAR_SLUGS = frozenset( - [ - "first_name", - "firstName", - "last_name", - "lastName", - "official_first_names", - "name", - "email", - "phone", - "phone_number", - "phoneNumber", - "address", - "discord", - "discord_handle", - "discordHandle", - "nick", - # Special diet / health data (GDPR Art. 9) - "special_diet_other", - "erityisruokavalio", - "allergies", - "dietary_restrictions", - "dietaryRestrictions", - "special_diets", - ] -) - class Command(BaseCommand): help = ( - "Pseudonymize all personal data in the database. " - "Intended to run against a temporary copy of the production database " - "created by scripts/make-pseudonymized-dump.sh. " - "The database name must contain 'pseudo'. " - "Note: form_data PII scrubbing is best-effort based on known field slugs; " - "custom slugs outside the known list will not be scrubbed." + "Pseudonymize all personal data and delete all secrets in the database. " + "Refuses to run unless the database name contains 'pseudo'." ) def add_arguments(self, parser): parser.add_argument( "--yes", action="store_true", - dest="yes", help="Confirm that you want to pseudonymize this database.", ) parser.add_argument( - "--dry-run", + "--check", action="store_true", - dest="dry_run", - help="Print counts of affected records without writing anything.", + help="List fields that may hold personal data but are neither pseudonymized nor reviewed as safe.", ) def handle(self, *args, **options): - self._check_safety(options["yes"]) - - if options["dry_run"]: - self.stdout.write(self.style.WARNING("DRY RUN — no changes will be written.")) - self._report_counts() + from kompassi.core.pseudonymization import find_stale_classifications, find_unclassified_fields, pseudonymize + + if options["check"]: + problems = find_unclassified_fields() + [f"{name} (stale)" for name in find_stale_classifications()] + for problem in problems: + self.stdout.write(problem) + if problems: + raise CommandError(f"{len(problems)} fields need a rule or an entry in NOT_PERSONAL") return + database_name = settings.DATABASES["default"]["NAME"] + if "pseudo" not in database_name: + raise CommandError(f"Database name {database_name!r} does not contain 'pseudo'.") + if not options["yes"]: + raise CommandError(f"Pass --yes to confirm you want to pseudonymize {database_name!r}.") + with transaction.atomic(): - self._pseudonymize_persons() - self._pseudonymize_tickets_v2_orders() - self._pseudonymize_forms_responses() - self._pseudonymize_involvement_invitations() - self._pseudonymize_signup_extras() - self._pseudonymize_zombie_enrollment() - self._pseudonymize_zombie_tickets_customers() - self._pseudonymize_zombie_programme_feedback() - self._delete_one_time_tokens() - self._delete_smtp_passwords() - self._delete_keypairs() - self._delete_email_aliases() + for model_label, summary in pseudonymize(): + self.stdout.write(f"{model_label}: {summary}") self.stdout.write(self.style.SUCCESS("Pseudonymization complete.")) - - def _check_safety(self, yes: bool) -> None: - if not yes: - raise CommandError("Pass --yes to confirm you want to pseudonymize this database.") - db_name = settings.DATABASES["default"]["NAME"] - if "pseudo" not in db_name: - raise CommandError( - f"Database name {db_name!r} does not contain 'pseudo'. " - "This command is intended to run against a temporary copy created by " - "scripts/make-pseudonymized-dump.sh. " - "If you set up the temp database manually, rename it to include 'pseudo'." - ) - - def _pseudonymize_persons(self) -> None: - from kompassi.core.models import Person - - User = get_user_model() - count = 0 - - for person in Person.objects.select_related("user").iterator(chunk_size=500): - pk = person.pk - person.first_name = f"Person{pk}" - person.official_first_names = f"Person{pk}" - person.surname = "Testinen" - person.nick = f"person{pk}" if person.nick else "" - person.discord_handle = f"person{pk}" if person.discord_handle else "" - person.email = f"person{pk}@example.com" - person.phone = f"+358000{pk:06d}" - person.muncipality = "Testilä" - person.notes = "" - if person.birth_date: - person.birth_date = date(person.birth_date.year, 1, 1) - # person.save() syncs first_name/surname/email to person.user - person.save() - if person.user_id: - person.user.set_unusable_password() - person.user.save(update_fields=["password"]) - count += 1 - - orphan_count = User.objects.filter(person__isnull=True).update(first_name="", last_name="", email="") - for user in User.objects.filter(person__isnull=True).iterator(chunk_size=500): - user.set_unusable_password() - user.save(update_fields=["password"]) - - self.stdout.write(f" Persons: {count}, orphan users: {orphan_count}") - - def _pseudonymize_tickets_v2_orders(self) -> None: - from kompassi.tickets_v2.models.order import Order - from kompassi.tickets_v2.models.receipt import Receipt - - Order.objects.update(first_name="Order", last_name="Testinen", phone="") - for order in Order.objects.only("id", "order_number").iterator(chunk_size=1000): - Order.objects.filter(id=order.id).update(email=f"order{order.order_number}@example.com") - - receipt_count = Receipt.objects.update(email="") - self.stdout.write(f" Orders: {Order.objects.count()}, receipts cleared: {receipt_count}") - - def _pseudonymize_forms_responses(self) -> None: - from kompassi.forms.models.response import Response - - Response.objects.update(ip_address="127.0.0.1", cached_key_fields={}) - - count = 0 - scrubbed = 0 - for response in Response.objects.only("id", "form_data").iterator(chunk_size=500): - new_data = dict(response.form_data) - changed = False - for slug in FORM_DATA_SSN_SLUGS: - if slug in new_data: - new_data[slug] = INVALID_SSN - changed = True - for slug in FORM_DATA_CLEAR_SLUGS: - if slug in new_data: - new_data[slug] = "" - changed = True - if changed: - response.form_data = new_data - response.save(update_fields=["form_data"]) - scrubbed += 1 - count += 1 - - self.stdout.write(f" Responses: {count} processed, {scrubbed} form_data scrubbed") - - def _pseudonymize_involvement_invitations(self) -> None: - from kompassi.involvement.models.invitation import Invitation - - count = 0 - for invitation in Invitation.objects.only("id").iterator(chunk_size=500): - Invitation.objects.filter(id=invitation.id).update(email=f"invitation-{str(invitation.id)[:8]}@example.com") - count += 1 - - self.stdout.write(f" Invitations: {count}") - - def _pseudonymize_signup_extras(self) -> None: - from django.apps import apps - - from kompassi.labour.models.signup_extras import SignupExtraMixin - - for model in apps.get_models(): - if not (isinstance(model, type) and issubclass(model, SignupExtraMixin)): - continue - if model.get_special_diet_field(): - for instance in model.objects.iterator(chunk_size=500): - instance.special_diet.clear() # type: ignore[attr-defined] - self.stdout.write(f" {model.__name__} special_diet cleared") - if model.get_special_diet_other_field(): - count = model.objects.update(special_diet_other="") - self.stdout.write(f" {model.__name__} special_diet_other cleared: {count}") - - def _pseudonymize_zombie_enrollment(self) -> None: - from kompassi.zombies.enrollment.models.enrollment import Enrollment - - count = Enrollment.objects.update( - personal_identification_number=INVALID_SSN, - address="", - zip_code="", - city="", - ) - for enrollment in Enrollment.objects.iterator(chunk_size=500): - enrollment.special_diet.clear() - - self.stdout.write(f" Zombie enrollments: {count}") - - def _pseudonymize_zombie_tickets_customers(self) -> None: - from kompassi.zombies.tickets.models.LEGACY_TICKETSV1_customer import Customer - - count = 0 - for customer in Customer.objects.only("pk").iterator(chunk_size=500): - Customer.objects.filter(pk=customer.pk).update( - first_name="Customer", - last_name="Testinen", - email=f"customer{customer.pk}@example.com", - phone_number="", - ) - count += 1 - - self.stdout.write(f" Legacy customers: {count}") - - def _pseudonymize_zombie_programme_feedback(self) -> None: - from kompassi.zombies.programme.models.programme_feedback import ProgrammeFeedback - - count = ProgrammeFeedback.objects.update(author_ip_address="127.0.0.1", author_external_username="") - self.stdout.write(f" Programme feedback: {count}") - - def _delete_one_time_tokens(self) -> None: - from kompassi.core.models.email_verification_token import EmailVerificationToken - from kompassi.core.models.password_reset_token import PasswordResetToken - from kompassi.tickets_v2.models.order_cancellation_token import OrderCancellationToken - - n1, _ = EmailVerificationToken.objects.all().delete() - n2, _ = PasswordResetToken.objects.all().delete() - n3, _ = OrderCancellationToken.objects.all().delete() - self.stdout.write(f" Deleted tokens: {n1} email verification, {n2} password reset, {n3} order cancellation") - - def _delete_smtp_passwords(self) -> None: - from kompassi.access.models.smtp_password import SMTPPassword - - count, _ = SMTPPassword.objects.all().delete() - self.stdout.write(f" Deleted SMTP passwords: {count}") - - def _delete_keypairs(self) -> None: - from kompassi.forms.models.keypair import KeyPair - - count, _ = KeyPair.objects.all().delete() - self.stdout.write(f" Deleted keypairs: {count}") - - def _delete_email_aliases(self) -> None: - from kompassi.access.models.email_alias import EmailAlias - - count, _ = EmailAlias.objects.all().delete() - self.stdout.write(f" Deleted email aliases: {count}") - - def _report_counts(self) -> None: - from kompassi.access.models.email_alias import EmailAlias - from kompassi.access.models.smtp_password import SMTPPassword - from kompassi.core.models import Person - from kompassi.core.models.email_verification_token import EmailVerificationToken - from kompassi.core.models.password_reset_token import PasswordResetToken - from kompassi.forms.models.keypair import KeyPair - from kompassi.forms.models.response import Response - from kompassi.involvement.models.invitation import Invitation - from kompassi.tickets_v2.models.order import Order - from kompassi.tickets_v2.models.order_cancellation_token import OrderCancellationToken - from kompassi.zombies.enrollment.models.enrollment import Enrollment - from kompassi.zombies.tickets.models.LEGACY_TICKETSV1_customer import Customer - - self.stdout.write(f" Persons: {Person.objects.count()}") - self.stdout.write(f" Orders: {Order.objects.count()}") - self.stdout.write(f" Responses: {Response.objects.count()}") - self.stdout.write(f" Invitations: {Invitation.objects.count()}") - self.stdout.write(f" Zombie enrollments: {Enrollment.objects.count()}") - self.stdout.write(f" Legacy customers: {Customer.objects.count()}") - self.stdout.write(f" Email verification tokens: {EmailVerificationToken.objects.count()}") - self.stdout.write(f" Password reset tokens: {PasswordResetToken.objects.count()}") - self.stdout.write(f" Order cancellation tokens: {OrderCancellationToken.objects.count()}") - self.stdout.write(f" SMTP passwords: {SMTPPassword.objects.count()}") - self.stdout.write(f" Keypairs: {KeyPair.objects.count()}") - self.stdout.write(f" Email aliases: {EmailAlias.objects.count()}") diff --git a/kompassi/core/pseudonymization.py b/kompassi/core/pseudonymization.py new file mode 100644 index 000000000..3d4af2207 --- /dev/null +++ b/kompassi/core/pseudonymization.py @@ -0,0 +1,471 @@ +""" +Turns a copy of the production database into one that can be handed to developers. + +Every field that `is_suspicious` flags must be covered by a rule in `RULES` or listed in +`NOT_PERSONAL`. `core/tests.py` fails otherwise, so a new model or field holding personal data +cannot slip past the pseudonymizer unnoticed. `manage.py pseudonymize_db --check` prints the +fields that still need a decision. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any + +from django.apps import apps +from django.contrib.auth.hashers import UNUSABLE_PASSWORD_PREFIX +from django.contrib.postgres.fields import ArrayField, HStoreField +from django.db import connection, models +from django.db.models import Case, CharField, OuterRef, Subquery, Value, When +from django.db.models.expressions import Combinable +from django.db.models.functions import Cast, Coalesce, Concat, LPad, TruncYear + +from kompassi.core.models import Person +from kompassi.forms.models.field import FieldType +from kompassi.labour.models.signup_extras import SignupExtraMixin + +REDACTED = "(pseudonymized)" +UNUSABLE_PASSWORD = f"{UNUSABLE_PASSWORD_PREFIX}pseudonymized" + +ALL_FIELDS = frozenset({"*"}) + + +@dataclass(frozen=True) +class Rule: + model_label: str + fields: frozenset[str] + """Fields this rule makes safe to hand out. `ALL_FIELDS` means the rule deletes every row.""" + + apply: Callable[[type[models.Model]], str] + """Performs the rule on the model and returns a summary for the operator.""" + + @property + def model(self) -> type[models.Model]: + return apps.get_model(self.model_label) + + +def delete_all(model_label: str) -> Rule: + def apply(model: type[models.Model]) -> str: + count, _ = model.objects.all().delete() + return f"deleted {count}" + + return Rule(model_label, ALL_FIELDS, apply) + + +def update(model_label: str, **values: Any) -> Rule: + def apply(model: type[models.Model]) -> str: + return f"updated {model.objects.update(**values)}" + + return Rule(model_label, frozenset(values), apply) + + +def numbered(prefix: str, suffix: str = "", field: str = "pk") -> Combinable: + return Concat(Value(prefix), Cast(field, output_field=CharField()), Value(suffix)) + + +def unless_blank(field: str, expression: Combinable) -> Combinable: + return Case(When(**{field: ""}, then=Value("")), default=expression) + + +def person_of(foreign_key: str, field: str) -> Subquery: + return Subquery(Person.objects.filter(pk=OuterRef(foreign_key)).values(field)[:1]) + + +def user_person(field: str) -> Subquery: + return Subquery(Person.objects.filter(user=OuterRef("pk")).values(field)[:1]) + + +def remove_json_keys(model_label: str, field: str, keys: list[str]) -> Rule: + def apply(model: type[models.Model]) -> str: + table = connection.ops.quote_name(model._meta.db_table) + column = connection.ops.quote_name(model._meta.get_field(field).column) + with connection.cursor() as cursor: + cursor.execute( + f"UPDATE {table} SET {column} = {column} - %s::text[] WHERE {column} ?| %s::text[]", + [keys, keys], + ) + return f"removed {', '.join(keys)} from {cursor.rowcount}" + + return Rule(model_label, frozenset({field}), apply) + + +def is_free_text(field: models.Field) -> bool: + return isinstance(field, (models.CharField, models.TextField)) and not field.choices and not field.primary_key + + +def signup_extra_rule(model: type[models.Model]) -> Rule: + free_text_fields = [field.name for field in model._meta.concrete_fields if is_free_text(field)] + + def apply(model: type[models.Model]) -> str: + count = model.objects.update(**{name: Value("") for name in free_text_fields}) + summary = f"cleared {', '.join(free_text_fields) or 'nothing'} on {count}" + if special_diet_field := model.get_special_diet_field(): # type: ignore[attr-defined] + deleted, _ = special_diet_field.remote_field.through.objects.all().delete() + summary += f", {deleted} special diets" + return summary + + return Rule(model._meta.label, frozenset(free_text_fields), apply) + + +def signup_extra_models() -> Iterable[type[models.Model]]: + for model in apps.get_models(): + if issubclass(model, SignupExtraMixin): + yield model + + +# Values that are a choice among options the organizers defined, not something the respondent wrote. +KEPT_FORM_FIELD_TYPES = frozenset( + { + FieldType.SINGLE_CHECKBOX, + FieldType.TRISTATE, + FieldType.SINGLE_SELECT, + FieldType.MULTI_SELECT, + FieldType.RADIO_MATRIX, + FieldType.DIMENSION_SINGLE_SELECT, + FieldType.DIMENSION_MULTI_SELECT, + FieldType.DIMENSION_SINGLE_CHECKBOX, + FieldType.NUMBER_FIELD, + FieldType.DECIMAL_FIELD, + FieldType.TIME_FIELD, + FieldType.DATE_TIME_FIELD, + } +) +REDACTED_FORM_FIELD_TYPES = frozenset( + { + FieldType.SINGLE_LINE_TEXT, + FieldType.MULTI_LINE_TEXT, + FieldType.MARKDOWN_TEXT, + } +) + + +def pseudonymize_form_data(fields: list[dict[str, Any]], form_data: dict[str, Any]) -> dict[str, Any]: + """ + Keeps answers that are choices, replaces free text with `REDACTED` and drops everything else, + including date fields (birth dates), file uploads and answers to fields no longer on the form. + Encrypted answers are dropped whatever the field type. + """ + result = {} + for field in fields: + slug = field.get("slug") + type = field.get("type") + if not slug or field.get("encryptTo"): + continue + + for key, value in form_data.items(): + if key != slug and not key.startswith(f"{slug}."): + continue + if type in KEPT_FORM_FIELD_TYPES: + result[key] = value + elif type in REDACTED_FORM_FIELD_TYPES: + result[key] = REDACTED if value else value + + return result + + +def pseudonymize_responses(model: type[models.Model]) -> str: + from kompassi.forms.models.form import Form + from kompassi.forms.models.response import Response + + count = 0 + for form in Form.objects.select_related("survey").only("id", "cached_enriched_fields", "survey").iterator(): + responses = [] + for response in Response.objects.filter(form=form).only("id", "form", "form_data"): + response.form = form + response.form_data = pseudonymize_form_data(form.cached_enriched_fields, response.form_data) + response.cached_key_fields = response._build_cached_key_fields(form.validated_fields) + response.ip_address = "" + responses.append(response) + Response.objects.bulk_update(responses, ["form_data", "cached_key_fields", "ip_address"], batch_size=500) + count += len(responses) + + return f"pseudonymized {count}" + + +RULES: list[Rule] = [ + # Credentials and one-time secrets + delete_all("sessions.Session"), + delete_all("oauth2_provider.AccessToken"), + delete_all("oauth2_provider.RefreshToken"), + delete_all("oauth2_provider.IDToken"), + delete_all("oauth2_provider.Grant"), + delete_all("oauth2_provider.DeviceGrant"), + update("oauth2_provider.Application", client_secret=Value("")), + delete_all("core.EmailVerificationToken"), + delete_all("core.PasswordResetToken"), + delete_all("tickets_v2.OrderCancellationToken"), + delete_all("desuprofile_integration.ConfirmationCode"), + delete_all("access.SMTPPassword"), + delete_all("forms.KeyPair"), + update("payments.PaymentsOrganizationMeta", checkout_password=Value("")), + update("lippukala.Code", code=numbered("pseudonymized"), literate_code=Value("")), + update("paikkala.Ticket", name=Value(""), email=Value(""), phone=Value(""), key=LPad(numbered(""), 8, Value("0"))), + # Queued tasks may send email to real addresses when a developer starts a worker. + delete_all("task_queue.QueuedTask"), + # Persons first: the rules after this one copy their pseudonymized names. + update( + "core.Person", + first_name=numbered("Person"), + official_first_names=numbered("Person"), + surname=Value("Testinen"), + nick=unless_blank("nick", numbered("person")), + discord_handle=unless_blank("discord_handle", numbered("person")), + email=numbered("person", "@example.com"), + # Finnish numbers never start with 000, so nobody gets called or texted by accident. + phone=Concat(Value("+358000"), LPad(numbered(""), 7, Value("0"))), + muncipality=Value("Testilä"), + notes=Value(""), + birth_date=TruncYear("birth_date"), + ), + update( + "auth.User", + username=numbered("user"), + first_name=Coalesce(user_person("first_name"), Value("")), + last_name=Coalesce(user_person("surname"), Value("")), + email=Coalesce(user_person("email"), Value("")), + password=Value(UNUSABLE_PASSWORD), + ), + update( + "badges.Badge", + first_name=Coalesce(person_of("person_id", "first_name"), Value("Badge")), + surname=Coalesce(person_of("person_id", "surname"), Value("Testinen")), + nick=unless_blank("nick", Coalesce(person_of("person_id", "nick"), Value(""))), + notes=Value(""), + ), + # Admin log entries name the objects edited, and event log entries record request details. + delete_all("admin.LogEntry"), + remove_json_keys( + "event_log_v2.Entry", + "other_fields", + ["ip_address", "context", "search_term", "user", "feedback_message"], + ), + delete_all("event_log.Entry"), + update("desuprofile_integration.Connection", desuprofile_username=numbered("desuprofile")), + update("labour.Signup", notes=Value("")), + update("labour.Shift", notes=Value("")), + *(signup_extra_rule(model) for model in signup_extra_models()), + update("involvement.Invitation", email=numbered("invitation-", "@example.com")), + update("membership.Membership", message=Value("")), + Rule( + "forms.Response", + frozenset({"form_data", "cached_key_fields", "ip_address"}), + pseudonymize_responses, + ), + update("messages_v2.MessageRecipient", email=numbered("recipient-", "@example.com"), subject=Value(REDACTED)), + update("messages_v2.MessageBody", text=Value(REDACTED)), + delete_all("mailings.PersonMessage"), + delete_all("mailings.PersonMessageSubject"), + delete_all("mailings.PersonMessageBody"), + delete_all("access.EmailAlias"), + update("access.InternalEmailAlias", target_emails=Value("")), + update( + "tickets_v2.Order", + first_name=Value("Order"), + last_name=Value("Testinen"), + email=numbered("order", "@example.com", field="order_number"), + phone=Value(""), + ), + update("tickets_v2.Receipt", email=Value("")), + update("tickets_v2.PaymentStamp", data=Value({}, output_field=models.JSONField())), + update( + "tickets.Customer", + first_name=Value("Customer"), + last_name=Value("Testinen"), + email=numbered("customer", "@example.com"), + phone_number=Value(""), + ), + update("payments.CheckoutPayment", customer=Value({}, output_field=models.JSONField())), + update("tickets.Order", ip_address=Value("")), + update("lippukala.Order", address_text=Value(""), free_text=Value(""), comment=Value("")), + update( + "programme.Programme", + notes=Value(""), + notes_from_host=Value(""), + solmukohta2024_other_emails=Value(""), + ), + delete_all("programme.ProgrammeFeedback"), +] + + +# Fields `is_suspicious` flags that were reviewed and hold no personal data or secrets. +NOT_PERSONAL: dict[str, frozenset[str]] = { + label: frozenset(fields.split()) + for label, fields in { + # Configuration written by organizers or administrators + "auth.Permission": "codename", + "oauth2_provider.Application": "client_id redirect_uris post_logout_redirect_uris name allowed_origins", + "core.Organization": "muncipality", + "access.CBACEntry": "claims", + "access.InternalEmailAlias": "email_address", + "access.SMTPServer": "ssh_username password_file_path_on_server", + "intra.Team": "email", + "labour.LabourEventMeta": "monitor_email contact_email", + "labour.PersonnelClass": "perks", + "program_v2.ProgramV2EventMeta": "contact_email", + "tickets_v2.TicketsV2EventMeta": "contact_email", + "programme.ProgrammeEventMeta": "contact_email", + "tickets.TicketsEventMeta": "contact_email", + "tickets.Product": "notify_email code", + "programme.AlternativeProgrammeForm": "programme_form_code v2_dimensions", + "programme.Category": "notes v2_dimensions", + "programme.Room": "notes v2_dimensions", + "programme.Role": "perks", + "programme.SpecialReservation": "code", + "programme.Tag": "v2_dimensions", + "dimensions.UniverseAnnotation": "form_fields", + "involvement.InvolvementToGroupMapping": "required_dimensions", + "involvement.InvolvementToBadgeMapping": "required_dimensions annotations", + "badges.SurveyToBadgeMapping": "required_dimensions annotations", + "event_log_v2.Entry": "entry_type", + "event_log_v2.Subscription": "entry_type", + "forms.Survey": "slug cached_key_fields cached_default_response_dimensions cached_default_involvement_dimensions", + "forms.Form": "title description thank_you_message fields cached_enriched_fields", + "forms.Projection": ( + "default_language_code splats required_dimensions projected_dimensions filterable_dimensions" + " order_by special_fields" + ), + # Organizers write these to many recipients at once, so they are not about any one person. + "messages_v2.Message": "subject body recipient_filters", + "messages_v2.MessageReplyTo": "email", + # Programs, including their hosts, are published on the public schedule. + "program_v2.Program": ( + "title slug description annotations cached_dimensions cached_combined_dimensions cached_color" + ), + "program_v2.ScheduleItem": ( + "cached_dimensions cached_combined_dimensions cached_location annotations cached_combined_annotations" + ), + # Roles, perks and dimensions that organizers assign + "involvement.Invitation": "cached_dimensions", + "involvement.Involvement": "title cached_dimensions annotations", + "messages_v2.MessageRecipient": "cached_dimensions", + "forms.Response": "cached_dimensions", + "labour.Signup": "xxx_interim_shifts job_title override_formatted_perks", + "labour.ArchivedSignup": "job_title", + "badges.Badge": "job_title perks", + "intra.TeamMember": "override_job_title", + "programme.ProgrammeRole": "override_perks", + "paikkala.Ticket": "qualifier_text_cache", + "tickets_v2.Order": "product_data", + "payments.CheckoutPayment": "items", + }.items() +} + + +def find_orphan_tables() -> list[str]: + """ + Tables that no installed model owns, such as those left behind by removed apps. No rule can + cover them, and they may hold personal data. + """ + known_tables = set(connection.introspection.django_table_names(include_views=False)) | {"django_migrations"} + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT c.relname + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND NOT c.relispartition + ORDER BY c.relname + """ + ) + return [table for (table,) in cursor.fetchall() if table not in known_tables] + + +def drop_orphan_tables() -> str: + tables = find_orphan_tables() + with connection.cursor() as cursor: + for table in tables: + cursor.execute(f"DROP TABLE {connection.ops.quote_name(table)} CASCADE") + return f"dropped {', '.join(tables) or 'nothing'}" + + +def pseudonymize() -> Iterable[tuple[str, str]]: + """Applies every rule in order, yielding the model label and a summary as each finishes.""" + yield "orphan tables", drop_orphan_tables() + for rule in RULES: + yield rule.model_label, rule.apply(rule.model) + + +PERSONAL_FIELD_NAME = re.compile( + r"e_?mail|phone|first_name|surname|last_name|full_name|nick|address|(^|_)ip($|_)|token|secret|password" + r"|hetu|ssn|identification|birth|diet|allerg|notes?$|handle|city|zip|muncipality|iban|key|code|username", + re.IGNORECASE, +) +TEXT_LIKE_FIELDS = ( + models.CharField, + models.TextField, + models.JSONField, + models.BinaryField, + models.GenericIPAddressField, + HStoreField, + ArrayField, +) + + +def is_linked_to_person(model: type[models.Model]) -> bool: + person_models = (Person, apps.get_model("auth.User")) + return model in person_models or any( + field.is_relation and field.related_model in person_models for field in model._meta.concrete_fields + ) + + +def is_suspicious(model: type[models.Model], field: models.Field) -> bool: + """ + Could this field hold personal data or a secret? Free text on a row about a person could hold + anything, so every such field is suspicious. Elsewhere only fields named like personal data are. + """ + if field.primary_key or field.choices or isinstance(field, (models.SlugField, models.UUIDField)): + return False + if isinstance(field, models.DateField) and not isinstance(field, models.DateTimeField): + return bool(PERSONAL_FIELD_NAME.search(field.name)) + if not isinstance(field, TEXT_LIKE_FIELDS): + return False + return ( + isinstance(field, models.JSONField) + or is_linked_to_person(model) + or bool(PERSONAL_FIELD_NAME.search(field.name)) + ) + + +def managed_models() -> Iterable[type[models.Model]]: + for model in apps.get_models(): + if model._meta.managed and not model._meta.proxy: + yield model + + +def find_unclassified_fields() -> list[str]: + covered: dict[str, frozenset[str]] = {} + for rule in RULES: + covered[rule.model_label] = covered.get(rule.model_label, frozenset()) | rule.fields + + unclassified = [] + for model in managed_models(): + label = model._meta.label + rule_fields = covered.get(label, frozenset()) + if rule_fields == ALL_FIELDS: + continue + reviewed_fields = rule_fields | NOT_PERSONAL.get(label, frozenset()) + unclassified.extend( + f"{label}.{field.name}" + for field in model._meta.concrete_fields + if field.name not in reviewed_fields and is_suspicious(model, field) + ) + + return unclassified + + +def find_stale_classifications() -> list[str]: + """Rules and `NOT_PERSONAL` entries that name a model or field that no longer exists.""" + classified = [(rule.model_label, rule.fields - ALL_FIELDS) for rule in RULES] + list(NOT_PERSONAL.items()) + + stale = [] + for label, field_names in classified: + try: + model = apps.get_model(label) + except LookupError: + stale.append(label) + continue + existing = {field.name for field in model._meta.concrete_fields} + stale.extend(f"{label}.{name}" for name in sorted(field_names - existing)) + + return stale diff --git a/kompassi/core/test_pseudonymization.py b/kompassi/core/test_pseudonymization.py new file mode 100644 index 000000000..2cc972c90 --- /dev/null +++ b/kompassi/core/test_pseudonymization.py @@ -0,0 +1,101 @@ +import pytest +from django.db import connection + +from kompassi.core.models import Event, Person +from kompassi.core.pseudonymization import ( + REDACTED, + find_orphan_tables, + find_stale_classifications, + find_unclassified_fields, + pseudonymize, + pseudonymize_form_data, +) +from kompassi.forms.models.form import Form +from kompassi.forms.models.response import Response +from kompassi.forms.models.survey import Survey + + +def test_every_suspicious_field_is_classified(): + """ + A field that may hold personal data needs a rule in RULES or an entry in NOT_PERSONAL. + `python manage.py pseudonymize_db --check` lists the fields that need one. + """ + assert find_unclassified_fields() == [] + + +def test_no_classification_names_a_missing_field(): + assert find_stale_classifications() == [] + + +def test_pseudonymize_form_data(): + fields = [ + dict(slug="name", type="SingleLineText"), + dict(slug="empty", type="SingleLineText"), + dict(slug="shirt", type="SingleSelect"), + dict(slug="days", type="MultiSelect"), + dict(slug="birthday", type="DateField"), + dict(slug="photo", type="FileUpload"), + dict(slug="hetu", type="SingleLineText", encryptTo=["someone"]), + ] + form_data = { + "name": "Markku Mahtinen", + "empty": "", + "shirt": "xl", + "days.friday": "on", + "birthday": "1984-01-01", + "photo": ["https://example.com/photo.jpg"], + "hetu": "eyJhbGciOi...", + # answer to a field that has since been removed from the form + "phone": "+358501234567", + } + + assert pseudonymize_form_data(fields, form_data) == { + "name": REDACTED, + "empty": "", + "shirt": "xl", + "days.friday": "on", + } + + +@pytest.mark.django_db +def test_pseudonymize(): + """ + Runs every rule against a database that has a little data in it, so that each rule's SQL + gets executed at least once. + """ + person, _ = Person.get_or_create_dummy() + event, _ = Event.get_or_create_dummy() + survey = Survey.objects.create(event=event, slug="pseudonymize-survey") + form = Form.objects.create( + event=event, + survey=survey, + language="en", + fields=[dict(slug="name", type="SingleLineText", isKeyField=True)], + ) + survey.refresh_cached_key_fields(form) + response = Response.objects.create(form=form, form_data={"name": "Markku"}, ip_address="192.0.2.1") + response.refresh_cached_fields() + with connection.cursor() as cursor: + # left behind by an app that has since been removed + cursor.execute("CREATE TABLE sms_smsmessageout (id serial PRIMARY KEY, message text)") + + list(pseudonymize()) + + assert find_orphan_tables() == [] + + person.refresh_from_db() + assert person.first_name == f"Person{person.pk}" + assert person.email == f"person{person.pk}@example.com" + assert person.birth_date.month == person.birth_date.day == 1 + + user = person.user + assert user + user.refresh_from_db() + assert user.username == f"user{user.pk}" + assert user.email == person.email + assert not user.has_usable_password() + + response.refresh_from_db() + assert response.form_data == {"name": REDACTED} + assert response.cached_key_fields == {"name": REDACTED} + assert response.ip_address == "" diff --git a/scripts/load-dump.sh b/scripts/load-dump.sh new file mode 100755 index 000000000..6bae2fec6 --- /dev/null +++ b/scripts/load-dump.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Replaces the docker compose development database with a dump from scripts/pseudonymized-dump.sh: +# +# scripts/load-dump.sh kompassi.pgdump +# +# Pseudonymization leaves no usable passwords or OAuth2 client secrets, so afterwards this +# recreates the dev superuser mahti/mahti and the OAuth2 client of the local V2 frontend. +set -eu + +if [ $# -ne 1 ]; then + echo "usage: $0 DUMP_FILE" >&2 + exit 2 +fi +dump="$1" + +cd "$(dirname "$0")/.." + +log() { + echo "==> $*" >&2 +} + +log "Stopping the services that hold database connections" +docker compose up -d router postgres redis +docker compose stop backend worker uvicorn + +log "Recreating the database" +until docker compose exec -T postgres pg_isready -q -U kompassi; do + sleep 1 +done +docker compose exec -T postgres dropdb -U kompassi --maintenance-db=postgres --if-exists --force kompassi +docker compose exec -T postgres createdb -U kompassi kompassi + +log "Restoring $dump" +# Not with --jobs: a parallel restore creates the triggers of a partitioned table while its +# partitions are still loading, and the triggers fail on the rows being copied. +docker compose exec -T postgres pg_restore -U kompassi -d kompassi --no-owner --no-acl --exit-on-error <"$dump" + +log "Migrating and recreating development credentials" +docker compose exec -T redis redis-cli -n 1 flushdb >/dev/null +docker compose run --rm --no-deps backend sh -c "python manage.py migrate && python manage.py setup_api_v2" + +log "Done. Start the rest with: docker compose up" diff --git a/scripts/make-pseudonymized-dump.sh b/scripts/make-pseudonymized-dump.sh deleted file mode 100755 index d0f804d45..000000000 --- a/scripts/make-pseudonymized-dump.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# Usage: scripts/make-pseudonymized-dump.sh -# -# Creates a pseudonymized dump of the production database. -# Run on the production server (SSH in or docker exec into the app container). -# -# Authentication uses PostgreSQL standard environment variables and ~/.pgpass — -# set PGHOST, PGPORT, PGUSER, PGPASSWORD as needed, or configure ~/.pgpass. -# PGDATABASE must be set to the production database name. -# -# The pseudonymized dump is written to /tmp and its path is printed at the end. -set -euo pipefail - -ORIG_DB="${PGDATABASE:?PGDATABASE must be set to the production database name}" -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -PSEUDO_DB="${ORIG_DB}_pseudo_${TIMESTAMP}" -OUTPUT_FILE="/tmp/${PSEUDO_DB}.sql.gz" - -echo "==> Creating temp database ${PSEUDO_DB}" -createdb "${PSEUDO_DB}" - -echo "==> Copying ${ORIG_DB} -> ${PSEUDO_DB} (pg_dump -Fc | pg_restore)" -# Binary custom format: compressed, faster than plain SQL, safe for same-server copies. -# For large databases with a maintenance window available, replace this with: -# createdb -T "${ORIG_DB}" "${PSEUDO_DB}" -# For maximum throughput without downtime, use directory format with --jobs=N instead of piping. -pg_dump --format=custom "${ORIG_DB}" \ - | pg_restore --dbname="${PSEUDO_DB}" --no-owner --no-privileges - -echo "==> Pseudonymizing ${PSEUDO_DB}" -# Override POSTGRES_DATABASE so Django connects to the temp copy, not the original. -# PGDATABASE is intentionally left unchanged so pg_* tools still default to the original DB. -POSTGRES_DATABASE="${PSEUDO_DB}" python manage.py pseudonymize_db --yes - -echo "==> Dumping pseudonymized database" -pg_dump "${PSEUDO_DB}" | gzip >"${OUTPUT_FILE}" - -echo "==> Cleaning up ${PSEUDO_DB}" -dropdb "${PSEUDO_DB}" - -echo "Done. Pseudonymized dump: ${OUTPUT_FILE}" diff --git a/scripts/pseudonymized-dump.sh b/scripts/pseudonymized-dump.sh new file mode 100755 index 000000000..b35680595 --- /dev/null +++ b/scripts/pseudonymized-dump.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# Writes a pseudonymized pg_dump (custom format) of a deployed Kompassi database to stdout: +# +# scripts/pseudonymized-dump.sh > kompassi.pgdump +# scripts/pseudonymized-dump.sh kompassi-staging > kompassi-staging.pgdump +# +# The personal data never leaves the cluster. A short-lived pod copies the database into a +# Postgres of its own, runs `manage.py pseudonymize_db` there, and only the result is streamed +# out. The pod is built from the pod template of the `kompassi` Deployment, so it runs the +# deployed image with the deployed configuration: the pseudonymization rules always match the +# schema they run against. Load the dump with scripts/load-dump.sh. +set -eu + +namespace="${1:-kompassi-production}" +postgres_image=postgres:18 +database=kompassi_pseudo +socket=/var/run/postgresql +pod="kompassi-pseudonymize-$(date +%Y%m%d%H%M%S)" + +if [ -t 1 ]; then + echo "$0: refusing to write a binary dump to a terminal; redirect stdout to a file" >&2 + exit 2 +fi + +log() { + echo "==> $*" >&2 +} + +deployment="$(kubectl -n "$namespace" get deployment kompassi -o json)" + +# The postgres container runs as the image's own postgres user because initdb needs the uid to +# have a passwd entry. It only listens on a socket shared with the pseudonymize container. +printf '%s' "$deployment" | jq \ + --arg pod "$pod" --arg postgres_image "$postgres_image" --arg database "$database" --arg socket "$socket" ' + .spec.template.spec as $spec + | ($spec.containers[] | select(.name == "master")) as $backend + | [$backend.env[] | select(.name | startswith("POSTGRES_"))] as $source + | { + apiVersion: "v1", + kind: "Pod", + metadata: {name: $pod, labels: {stack: "kompassi", component: "pseudonymize"}}, + spec: ($spec | del(.initContainers, .affinity, .terminationGracePeriodSeconds) + { + restartPolicy: "Never", + activeDeadlineSeconds: 7200, + volumes: ($spec.volumes + [ + {name: "postgres-data", emptyDir: {}}, + {name: "postgres-socket", emptyDir: {}} + ]), + containers: [ + { + name: "postgres", + image: $postgres_image, + command: ["sh", "-c", "initdb --auth=trust --username=postgres --encoding=UTF8 --locale-provider=icu --icu-locale=fi-FI -D /var/lib/postgresql/data && exec postgres -D /var/lib/postgresql/data -c listen_addresses= -c unix_socket_directories=\($socket)"], + env: [$source[] | .name |= sub("^POSTGRES_"; "SOURCE_")], + securityContext: ($backend.securityContext + {runAsUser: 999, runAsGroup: 999}), + volumeMounts: [ + {name: "postgres-data", mountPath: "/var/lib/postgresql"}, + {name: "postgres-socket", mountPath: $socket}, + {name: "kompassi-temp", mountPath: "/tmp"} + ] + }, + ($backend | del(.args, .ports, .startupProbe, .readinessProbe, .livenessProbe, .lifecycle) + { + name: "pseudonymize", + command: ["sleep", "infinity"], + env: ([.env[] | select(.name | startswith("POSTGRES_") | not)] + [ + {name: "POSTGRES_HOSTNAME", value: $socket}, + {name: "POSTGRES_DATABASE", value: $database}, + {name: "POSTGRES_USERNAME", value: "postgres"}, + {name: "POSTGRES_PASSWORD", value: ""}, + {name: "POSTGRES_SSLMODE", value: "disable"} + ]), + volumeMounts: (.volumeMounts + [{name: "postgres-socket", mountPath: $socket}]) + }) + ] + }) + } +' | kubectl -n "$namespace" apply -f - >/dev/null + +trap 'kubectl -n "$namespace" delete pod "$pod" --wait=false >/dev/null' EXIT + +log "Waiting for pod $namespace/$pod" +kubectl -n "$namespace" wait --for=condition=Ready "pod/$pod" --timeout=300s >/dev/null +until kubectl -n "$namespace" exec "$pod" -c postgres -- pg_isready -q -h "$socket"; do + sleep 2 +done + +log "Copying the $namespace database into the pod" +kubectl -n "$namespace" exec "$pod" -c postgres -- sh -c ' + set -eu + createdb -h "$1" "$2" + PGPASSWORD="$SOURCE_PASSWORD" pg_dump -Fc --no-owner --no-acl \ + "host=$SOURCE_HOSTNAME dbname=$SOURCE_DATABASE user=$SOURCE_USERNAME sslmode=$SOURCE_SSLMODE" \ + | pg_restore -h "$1" -d "$2" --no-owner --no-acl --exit-on-error +' sh "$socket" "$database" >&2 + +log "Pseudonymizing" +kubectl -n "$namespace" exec "$pod" -c pseudonymize -- python manage.py pseudonymize_db --yes >&2 + +log "Streaming the pseudonymized dump" +kubectl -n "$namespace" exec "$pod" -c postgres -- pg_dump -Fc --no-owner --no-acl -h "$socket" "$database" + +log "Done" From 5f412f56f85219f8c8e3d42cb9c9ef3f43e2fbb3 Mon Sep 17 00:00:00 2001 From: Luka Pajukanta Date: Tue, 29 Sep 2026 22:26:09 +0300 Subject: [PATCH 3/7] fix(core): let redaction win on form data key collisions, stop on failed source dump A form data key matching both a kept and a redacted field was kept or redacted depending on field order. The in-pod copy piped pg_dump into pg_restore, so a failing pg_dump did not fail the script; it now dumps to a file first. Co-Authored-By: Claude Opus 5.5 --- kompassi/core/pseudonymization.py | 3 ++- kompassi/core/test_pseudonymization.py | 10 ++++++++++ scripts/pseudonymized-dump.sh | 10 ++++++---- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/kompassi/core/pseudonymization.py b/kompassi/core/pseudonymization.py index 3d4af2207..edefd02e6 100644 --- a/kompassi/core/pseudonymization.py +++ b/kompassi/core/pseudonymization.py @@ -157,8 +157,9 @@ def pseudonymize_form_data(fields: list[dict[str, Any]], form_data: dict[str, An for key, value in form_data.items(): if key != slug and not key.startswith(f"{slug}."): continue + # A key can match several fields; a redacted value must win over a kept one. if type in KEPT_FORM_FIELD_TYPES: - result[key] = value + result.setdefault(key, value) elif type in REDACTED_FORM_FIELD_TYPES: result[key] = REDACTED if value else value diff --git a/kompassi/core/test_pseudonymization.py b/kompassi/core/test_pseudonymization.py index 2cc972c90..d1fcb55bb 100644 --- a/kompassi/core/test_pseudonymization.py +++ b/kompassi/core/test_pseudonymization.py @@ -57,6 +57,16 @@ def test_pseudonymize_form_data(): } +def test_pseudonymize_form_data_redaction_wins(): + """A key that matches both a kept and a redacted field is redacted whatever the field order.""" + kept = dict(slug="days", type="MultiSelect") + redacted = dict(slug="days.friday", type="SingleLineText") + form_data = {"days.friday": "Markku Mahtinen"} + + assert pseudonymize_form_data([redacted, kept], form_data) == {"days.friday": REDACTED} + assert pseudonymize_form_data([kept, redacted], form_data) == {"days.friday": REDACTED} + + @pytest.mark.django_db def test_pseudonymize(): """ diff --git a/scripts/pseudonymized-dump.sh b/scripts/pseudonymized-dump.sh index b35680595..384a19ae6 100755 --- a/scripts/pseudonymized-dump.sh +++ b/scripts/pseudonymized-dump.sh @@ -88,10 +88,12 @@ log "Copying the $namespace database into the pod" kubectl -n "$namespace" exec "$pod" -c postgres -- sh -c ' set -eu createdb -h "$1" "$2" - PGPASSWORD="$SOURCE_PASSWORD" pg_dump -Fc --no-owner --no-acl \ - "host=$SOURCE_HOSTNAME dbname=$SOURCE_DATABASE user=$SOURCE_USERNAME sslmode=$SOURCE_SSLMODE" \ - | pg_restore -h "$1" -d "$2" --no-owner --no-acl --exit-on-error -' sh "$socket" "$database" >&2 + # Through a file rather than a pipe, so that a failing pg_dump stops the script. + PGPASSWORD="$SOURCE_PASSWORD" pg_dump -Fc --no-owner --no-acl -f "$3" \ + "host=$SOURCE_HOSTNAME dbname=$SOURCE_DATABASE user=$SOURCE_USERNAME sslmode=$SOURCE_SSLMODE" + pg_restore -h "$1" -d "$2" --no-owner --no-acl --exit-on-error "$3" + rm "$3" +' sh "$socket" "$database" /var/lib/postgresql/source.pgdump >&2 log "Pseudonymizing" kubectl -n "$namespace" exec "$pod" -c pseudonymize -- python manage.py pseudonymize_db --yes >&2 From b33f217a3acf47e58552dd666e94416f4d383996 Mon Sep 17 00:00:00 2001 From: Luka Pajukanta Date: Tue, 29 Sep 2026 22:26:53 +0300 Subject: [PATCH 4/7] chore: gitignore *.pgdump --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8bb777c82..c09730785 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ # set !*.sql in folders where you want to include sql files *.sql +*.pgdump dev-secrets/ # Byte-compiled / optimized / DLL files From c1ffb21e58cfb964c8e4e5f9e52b1834bffe4372 Mon Sep 17 00:00:00 2001 From: Luka Pajukanta Date: Tue, 29 Sep 2026 22:30:58 +0300 Subject: [PATCH 5/7] feat(scripts): pseudonymized-dump.sh writes NAMESPACE-YYYYMMDD.pgdump itself The dump is written under a .partial name and renamed only when complete, so an interrupted run never leaves a truncated file that looks finished. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + CLAUDE.md | 4 ++-- scripts/load-dump.sh | 2 +- scripts/pseudonymized-dump.sh | 24 +++++++++++++----------- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index c09730785..f6cd83762 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # set !*.sql in folders where you want to include sql files *.sql *.pgdump +*.pgdump.partial dev-secrets/ # Byte-compiled / optimized / DLL files diff --git a/CLAUDE.md b/CLAUDE.md index bd457abe2..dd759170f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,8 +37,8 @@ npm run dev # starts Next.js at localhost:3000 with GraphQL codegen watch To develop against production data, fetch a pseudonymized copy and load it into the docker compose database (replaces it; `mahti`/`mahti` is recreated): ```bash -scripts/pseudonymized-dump.sh > kompassi.pgdump # needs kubectl access to kompassi-production -scripts/load-dump.sh kompassi.pgdump +scripts/pseudonymized-dump.sh # needs kubectl access; writes kompassi-production-YYYYMMDD.pgdump +scripts/load-dump.sh kompassi-production-YYYYMMDD.pgdump ``` Never load a raw production dump locally. The pseudonymization rules live in `kompassi/core/pseudonymization.py`; a new model field that could hold personal data fails `kompassi/core/test_pseudonymization.py` until it has a rule or an entry in `NOT_PERSONAL` (`python manage.py pseudonymize_db --check` lists them). diff --git a/scripts/load-dump.sh b/scripts/load-dump.sh index 6bae2fec6..9075be354 100755 --- a/scripts/load-dump.sh +++ b/scripts/load-dump.sh @@ -1,7 +1,7 @@ #!/bin/sh # Replaces the docker compose development database with a dump from scripts/pseudonymized-dump.sh: # -# scripts/load-dump.sh kompassi.pgdump +# scripts/load-dump.sh kompassi-production-20260929.pgdump # # Pseudonymization leaves no usable passwords or OAuth2 client secrets, so afterwards this # recreates the dev superuser mahti/mahti and the OAuth2 client of the local V2 frontend. diff --git a/scripts/pseudonymized-dump.sh b/scripts/pseudonymized-dump.sh index 384a19ae6..d26829c79 100755 --- a/scripts/pseudonymized-dump.sh +++ b/scripts/pseudonymized-dump.sh @@ -1,8 +1,9 @@ #!/bin/sh -# Writes a pseudonymized pg_dump (custom format) of a deployed Kompassi database to stdout: +# Writes a pseudonymized pg_dump (custom format) of a deployed Kompassi database to +# NAMESPACE-YYYYMMDD.pgdump in the current directory: # -# scripts/pseudonymized-dump.sh > kompassi.pgdump -# scripts/pseudonymized-dump.sh kompassi-staging > kompassi-staging.pgdump +# scripts/pseudonymized-dump.sh # kompassi-production-YYYYMMDD.pgdump +# scripts/pseudonymized-dump.sh kompassi-staging # kompassi-staging-YYYYMMDD.pgdump # # The personal data never leaves the cluster. A short-lived pod copies the database into a # Postgres of its own, runs `manage.py pseudonymize_db` there, and only the result is streamed @@ -16,11 +17,10 @@ postgres_image=postgres:18 database=kompassi_pseudo socket=/var/run/postgresql pod="kompassi-pseudonymize-$(date +%Y%m%d%H%M%S)" - -if [ -t 1 ]; then - echo "$0: refusing to write a binary dump to a terminal; redirect stdout to a file" >&2 - exit 2 -fi +output="$namespace-$(date +%Y%m%d).pgdump" +# Written under another name until complete, so an interrupted run never leaves a truncated +# dump that looks finished. +partial="$output.partial" log() { echo "==> $*" >&2 @@ -76,7 +76,7 @@ printf '%s' "$deployment" | jq \ } ' | kubectl -n "$namespace" apply -f - >/dev/null -trap 'kubectl -n "$namespace" delete pod "$pod" --wait=false >/dev/null' EXIT +trap 'rm -f "$partial"; kubectl -n "$namespace" delete pod "$pod" --wait=false >/dev/null' EXIT log "Waiting for pod $namespace/$pod" kubectl -n "$namespace" wait --for=condition=Ready "pod/$pod" --timeout=300s >/dev/null @@ -99,6 +99,8 @@ log "Pseudonymizing" kubectl -n "$namespace" exec "$pod" -c pseudonymize -- python manage.py pseudonymize_db --yes >&2 log "Streaming the pseudonymized dump" -kubectl -n "$namespace" exec "$pod" -c postgres -- pg_dump -Fc --no-owner --no-acl -h "$socket" "$database" +kubectl -n "$namespace" exec "$pod" -c postgres -- pg_dump -Fc --no-owner --no-acl -h "$socket" "$database" \ + >"$partial" +mv "$partial" "$output" -log "Done" +log "Wrote $output" From 7e46acf43ebeaa747b16c9bfc842307f4b360250 Mon Sep 17 00:00:00 2001 From: Luka Pajukanta Date: Tue, 29 Sep 2026 22:33:16 +0300 Subject: [PATCH 6/7] fix(scripts): silence pg_isready retries while the sidecar initializes Co-Authored-By: Claude Opus 5.5 --- scripts/pseudonymized-dump.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/pseudonymized-dump.sh b/scripts/pseudonymized-dump.sh index d26829c79..4f4d43433 100755 --- a/scripts/pseudonymized-dump.sh +++ b/scripts/pseudonymized-dump.sh @@ -80,7 +80,8 @@ trap 'rm -f "$partial"; kubectl -n "$namespace" delete pod "$pod" --wait=false > log "Waiting for pod $namespace/$pod" kubectl -n "$namespace" wait --for=condition=Ready "pod/$pod" --timeout=300s >/dev/null -until kubectl -n "$namespace" exec "$pod" -c postgres -- pg_isready -q -h "$socket"; do +# kubectl reports each failed attempt on stderr while initdb is still running. +until kubectl -n "$namespace" exec "$pod" -c postgres -- pg_isready -q -h "$socket" 2>/dev/null; do sleep 2 done From e7544b1c2bc03b0a397dba36a10540619970cc25 Mon Sep 17 00:00:00 2001 From: Luka Pajukanta Date: Tue, 29 Sep 2026 22:35:40 +0300 Subject: [PATCH 7/7] chore(core): pseudonymize_db takes --really like other destructive commands Co-Authored-By: Claude Opus 5.5 --- kompassi/core/management/commands/pseudonymize_db.py | 6 +++--- scripts/pseudonymized-dump.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kompassi/core/management/commands/pseudonymize_db.py b/kompassi/core/management/commands/pseudonymize_db.py index b0eeef66d..3a570019a 100644 --- a/kompassi/core/management/commands/pseudonymize_db.py +++ b/kompassi/core/management/commands/pseudonymize_db.py @@ -11,7 +11,7 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - "--yes", + "--really", action="store_true", help="Confirm that you want to pseudonymize this database.", ) @@ -35,8 +35,8 @@ def handle(self, *args, **options): database_name = settings.DATABASES["default"]["NAME"] if "pseudo" not in database_name: raise CommandError(f"Database name {database_name!r} does not contain 'pseudo'.") - if not options["yes"]: - raise CommandError(f"Pass --yes to confirm you want to pseudonymize {database_name!r}.") + if not options["really"]: + raise CommandError(f"Pass --really to confirm you want to pseudonymize {database_name!r}.") with transaction.atomic(): for model_label, summary in pseudonymize(): diff --git a/scripts/pseudonymized-dump.sh b/scripts/pseudonymized-dump.sh index 4f4d43433..e714d2dca 100755 --- a/scripts/pseudonymized-dump.sh +++ b/scripts/pseudonymized-dump.sh @@ -97,7 +97,7 @@ kubectl -n "$namespace" exec "$pod" -c postgres -- sh -c ' ' sh "$socket" "$database" /var/lib/postgresql/source.pgdump >&2 log "Pseudonymizing" -kubectl -n "$namespace" exec "$pod" -c pseudonymize -- python manage.py pseudonymize_db --yes >&2 +kubectl -n "$namespace" exec "$pod" -c pseudonymize -- python manage.py pseudonymize_db --really >&2 log "Streaming the pseudonymized dump" kubectl -n "$namespace" exec "$pod" -c postgres -- pg_dump -Fc --no-owner --no-acl -h "$socket" "$database" \