diff --git a/.gitignore b/.gitignore index 8bb777c82..f6cd83762 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ # set !*.sql in folders where you want to include sql files *.sql +*.pgdump +*.pgdump.partial dev-secrets/ # Byte-compiled / optimized / DLL files diff --git a/CLAUDE.md b/CLAUDE.md index c9b8d7616..dd759170f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,6 +34,15 @@ export NEXT_PUBLIC_KOMPASSI_BASE_URL=http://localhost:8000 npm run dev # starts Next.js at localhost:3000 with GraphQL codegen watch ``` +To develop against production data, fetch a pseudonymized copy and load it into the docker compose database (replaces it; `mahti`/`mahti` is recreated): + +```bash +scripts/pseudonymized-dump.sh # needs kubectl access; writes kompassi-production-YYYYMMDD.pgdump +scripts/load-dump.sh kompassi-production-YYYYMMDD.pgdump +``` + +Never load a raw production dump locally. The pseudonymization rules live in `kompassi/core/pseudonymization.py`; a new model field that could hold personal data fails `kompassi/core/test_pseudonymization.py` until it has a rule or an entry in `NOT_PERSONAL` (`python manage.py pseudonymize_db --check` lists them). + ## Backend commands All backend commands assume the virtualenv is active (`source .venv/bin/activate`) or are run via Docker. diff --git a/kompassi/core/management/commands/pseudonymize_db.py b/kompassi/core/management/commands/pseudonymize_db.py new file mode 100644 index 000000000..3a570019a --- /dev/null +++ b/kompassi/core/management/commands/pseudonymize_db.py @@ -0,0 +1,45 @@ +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + + +class Command(BaseCommand): + help = ( + "Pseudonymize all personal data and delete all secrets in the database. " + "Refuses to run unless the database name contains 'pseudo'." + ) + + def add_arguments(self, parser): + parser.add_argument( + "--really", + action="store_true", + help="Confirm that you want to pseudonymize this database.", + ) + parser.add_argument( + "--check", + action="store_true", + help="List fields that may hold personal data but are neither pseudonymized nor reviewed as safe.", + ) + + def handle(self, *args, **options): + from kompassi.core.pseudonymization import find_stale_classifications, find_unclassified_fields, pseudonymize + + if options["check"]: + problems = find_unclassified_fields() + [f"{name} (stale)" for name in find_stale_classifications()] + for problem in problems: + self.stdout.write(problem) + if problems: + raise CommandError(f"{len(problems)} fields need a rule or an entry in NOT_PERSONAL") + return + + database_name = settings.DATABASES["default"]["NAME"] + if "pseudo" not in database_name: + raise CommandError(f"Database name {database_name!r} does not contain 'pseudo'.") + if not options["really"]: + raise CommandError(f"Pass --really to confirm you want to pseudonymize {database_name!r}.") + + with transaction.atomic(): + for model_label, summary in pseudonymize(): + self.stdout.write(f"{model_label}: {summary}") + + self.stdout.write(self.style.SUCCESS("Pseudonymization complete.")) diff --git a/kompassi/core/pseudonymization.py b/kompassi/core/pseudonymization.py new file mode 100644 index 000000000..edefd02e6 --- /dev/null +++ b/kompassi/core/pseudonymization.py @@ -0,0 +1,472 @@ +""" +Turns a copy of the production database into one that can be handed to developers. + +Every field that `is_suspicious` flags must be covered by a rule in `RULES` or listed in +`NOT_PERSONAL`. `core/tests.py` fails otherwise, so a new model or field holding personal data +cannot slip past the pseudonymizer unnoticed. `manage.py pseudonymize_db --check` prints the +fields that still need a decision. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any + +from django.apps import apps +from django.contrib.auth.hashers import UNUSABLE_PASSWORD_PREFIX +from django.contrib.postgres.fields import ArrayField, HStoreField +from django.db import connection, models +from django.db.models import Case, CharField, OuterRef, Subquery, Value, When +from django.db.models.expressions import Combinable +from django.db.models.functions import Cast, Coalesce, Concat, LPad, TruncYear + +from kompassi.core.models import Person +from kompassi.forms.models.field import FieldType +from kompassi.labour.models.signup_extras import SignupExtraMixin + +REDACTED = "(pseudonymized)" +UNUSABLE_PASSWORD = f"{UNUSABLE_PASSWORD_PREFIX}pseudonymized" + +ALL_FIELDS = frozenset({"*"}) + + +@dataclass(frozen=True) +class Rule: + model_label: str + fields: frozenset[str] + """Fields this rule makes safe to hand out. `ALL_FIELDS` means the rule deletes every row.""" + + apply: Callable[[type[models.Model]], str] + """Performs the rule on the model and returns a summary for the operator.""" + + @property + def model(self) -> type[models.Model]: + return apps.get_model(self.model_label) + + +def delete_all(model_label: str) -> Rule: + def apply(model: type[models.Model]) -> str: + count, _ = model.objects.all().delete() + return f"deleted {count}" + + return Rule(model_label, ALL_FIELDS, apply) + + +def update(model_label: str, **values: Any) -> Rule: + def apply(model: type[models.Model]) -> str: + return f"updated {model.objects.update(**values)}" + + return Rule(model_label, frozenset(values), apply) + + +def numbered(prefix: str, suffix: str = "", field: str = "pk") -> Combinable: + return Concat(Value(prefix), Cast(field, output_field=CharField()), Value(suffix)) + + +def unless_blank(field: str, expression: Combinable) -> Combinable: + return Case(When(**{field: ""}, then=Value("")), default=expression) + + +def person_of(foreign_key: str, field: str) -> Subquery: + return Subquery(Person.objects.filter(pk=OuterRef(foreign_key)).values(field)[:1]) + + +def user_person(field: str) -> Subquery: + return Subquery(Person.objects.filter(user=OuterRef("pk")).values(field)[:1]) + + +def remove_json_keys(model_label: str, field: str, keys: list[str]) -> Rule: + def apply(model: type[models.Model]) -> str: + table = connection.ops.quote_name(model._meta.db_table) + column = connection.ops.quote_name(model._meta.get_field(field).column) + with connection.cursor() as cursor: + cursor.execute( + f"UPDATE {table} SET {column} = {column} - %s::text[] WHERE {column} ?| %s::text[]", + [keys, keys], + ) + return f"removed {', '.join(keys)} from {cursor.rowcount}" + + return Rule(model_label, frozenset({field}), apply) + + +def is_free_text(field: models.Field) -> bool: + return isinstance(field, (models.CharField, models.TextField)) and not field.choices and not field.primary_key + + +def signup_extra_rule(model: type[models.Model]) -> Rule: + free_text_fields = [field.name for field in model._meta.concrete_fields if is_free_text(field)] + + def apply(model: type[models.Model]) -> str: + count = model.objects.update(**{name: Value("") for name in free_text_fields}) + summary = f"cleared {', '.join(free_text_fields) or 'nothing'} on {count}" + if special_diet_field := model.get_special_diet_field(): # type: ignore[attr-defined] + deleted, _ = special_diet_field.remote_field.through.objects.all().delete() + summary += f", {deleted} special diets" + return summary + + return Rule(model._meta.label, frozenset(free_text_fields), apply) + + +def signup_extra_models() -> Iterable[type[models.Model]]: + for model in apps.get_models(): + if issubclass(model, SignupExtraMixin): + yield model + + +# Values that are a choice among options the organizers defined, not something the respondent wrote. +KEPT_FORM_FIELD_TYPES = frozenset( + { + FieldType.SINGLE_CHECKBOX, + FieldType.TRISTATE, + FieldType.SINGLE_SELECT, + FieldType.MULTI_SELECT, + FieldType.RADIO_MATRIX, + FieldType.DIMENSION_SINGLE_SELECT, + FieldType.DIMENSION_MULTI_SELECT, + FieldType.DIMENSION_SINGLE_CHECKBOX, + FieldType.NUMBER_FIELD, + FieldType.DECIMAL_FIELD, + FieldType.TIME_FIELD, + FieldType.DATE_TIME_FIELD, + } +) +REDACTED_FORM_FIELD_TYPES = frozenset( + { + FieldType.SINGLE_LINE_TEXT, + FieldType.MULTI_LINE_TEXT, + FieldType.MARKDOWN_TEXT, + } +) + + +def pseudonymize_form_data(fields: list[dict[str, Any]], form_data: dict[str, Any]) -> dict[str, Any]: + """ + Keeps answers that are choices, replaces free text with `REDACTED` and drops everything else, + including date fields (birth dates), file uploads and answers to fields no longer on the form. + Encrypted answers are dropped whatever the field type. + """ + result = {} + for field in fields: + slug = field.get("slug") + type = field.get("type") + if not slug or field.get("encryptTo"): + continue + + for key, value in form_data.items(): + if key != slug and not key.startswith(f"{slug}."): + continue + # A key can match several fields; a redacted value must win over a kept one. + if type in KEPT_FORM_FIELD_TYPES: + result.setdefault(key, value) + elif type in REDACTED_FORM_FIELD_TYPES: + result[key] = REDACTED if value else value + + return result + + +def pseudonymize_responses(model: type[models.Model]) -> str: + from kompassi.forms.models.form import Form + from kompassi.forms.models.response import Response + + count = 0 + for form in Form.objects.select_related("survey").only("id", "cached_enriched_fields", "survey").iterator(): + responses = [] + for response in Response.objects.filter(form=form).only("id", "form", "form_data"): + response.form = form + response.form_data = pseudonymize_form_data(form.cached_enriched_fields, response.form_data) + response.cached_key_fields = response._build_cached_key_fields(form.validated_fields) + response.ip_address = "" + responses.append(response) + Response.objects.bulk_update(responses, ["form_data", "cached_key_fields", "ip_address"], batch_size=500) + count += len(responses) + + return f"pseudonymized {count}" + + +RULES: list[Rule] = [ + # Credentials and one-time secrets + delete_all("sessions.Session"), + delete_all("oauth2_provider.AccessToken"), + delete_all("oauth2_provider.RefreshToken"), + delete_all("oauth2_provider.IDToken"), + delete_all("oauth2_provider.Grant"), + delete_all("oauth2_provider.DeviceGrant"), + update("oauth2_provider.Application", client_secret=Value("")), + delete_all("core.EmailVerificationToken"), + delete_all("core.PasswordResetToken"), + delete_all("tickets_v2.OrderCancellationToken"), + delete_all("desuprofile_integration.ConfirmationCode"), + delete_all("access.SMTPPassword"), + delete_all("forms.KeyPair"), + update("payments.PaymentsOrganizationMeta", checkout_password=Value("")), + update("lippukala.Code", code=numbered("pseudonymized"), literate_code=Value("")), + update("paikkala.Ticket", name=Value(""), email=Value(""), phone=Value(""), key=LPad(numbered(""), 8, Value("0"))), + # Queued tasks may send email to real addresses when a developer starts a worker. + delete_all("task_queue.QueuedTask"), + # Persons first: the rules after this one copy their pseudonymized names. + update( + "core.Person", + first_name=numbered("Person"), + official_first_names=numbered("Person"), + surname=Value("Testinen"), + nick=unless_blank("nick", numbered("person")), + discord_handle=unless_blank("discord_handle", numbered("person")), + email=numbered("person", "@example.com"), + # Finnish numbers never start with 000, so nobody gets called or texted by accident. + phone=Concat(Value("+358000"), LPad(numbered(""), 7, Value("0"))), + muncipality=Value("Testilä"), + notes=Value(""), + birth_date=TruncYear("birth_date"), + ), + update( + "auth.User", + username=numbered("user"), + first_name=Coalesce(user_person("first_name"), Value("")), + last_name=Coalesce(user_person("surname"), Value("")), + email=Coalesce(user_person("email"), Value("")), + password=Value(UNUSABLE_PASSWORD), + ), + update( + "badges.Badge", + first_name=Coalesce(person_of("person_id", "first_name"), Value("Badge")), + surname=Coalesce(person_of("person_id", "surname"), Value("Testinen")), + nick=unless_blank("nick", Coalesce(person_of("person_id", "nick"), Value(""))), + notes=Value(""), + ), + # Admin log entries name the objects edited, and event log entries record request details. + delete_all("admin.LogEntry"), + remove_json_keys( + "event_log_v2.Entry", + "other_fields", + ["ip_address", "context", "search_term", "user", "feedback_message"], + ), + delete_all("event_log.Entry"), + update("desuprofile_integration.Connection", desuprofile_username=numbered("desuprofile")), + update("labour.Signup", notes=Value("")), + update("labour.Shift", notes=Value("")), + *(signup_extra_rule(model) for model in signup_extra_models()), + update("involvement.Invitation", email=numbered("invitation-", "@example.com")), + update("membership.Membership", message=Value("")), + Rule( + "forms.Response", + frozenset({"form_data", "cached_key_fields", "ip_address"}), + pseudonymize_responses, + ), + update("messages_v2.MessageRecipient", email=numbered("recipient-", "@example.com"), subject=Value(REDACTED)), + update("messages_v2.MessageBody", text=Value(REDACTED)), + delete_all("mailings.PersonMessage"), + delete_all("mailings.PersonMessageSubject"), + delete_all("mailings.PersonMessageBody"), + delete_all("access.EmailAlias"), + update("access.InternalEmailAlias", target_emails=Value("")), + update( + "tickets_v2.Order", + first_name=Value("Order"), + last_name=Value("Testinen"), + email=numbered("order", "@example.com", field="order_number"), + phone=Value(""), + ), + update("tickets_v2.Receipt", email=Value("")), + update("tickets_v2.PaymentStamp", data=Value({}, output_field=models.JSONField())), + update( + "tickets.Customer", + first_name=Value("Customer"), + last_name=Value("Testinen"), + email=numbered("customer", "@example.com"), + phone_number=Value(""), + ), + update("payments.CheckoutPayment", customer=Value({}, output_field=models.JSONField())), + update("tickets.Order", ip_address=Value("")), + update("lippukala.Order", address_text=Value(""), free_text=Value(""), comment=Value("")), + update( + "programme.Programme", + notes=Value(""), + notes_from_host=Value(""), + solmukohta2024_other_emails=Value(""), + ), + delete_all("programme.ProgrammeFeedback"), +] + + +# Fields `is_suspicious` flags that were reviewed and hold no personal data or secrets. +NOT_PERSONAL: dict[str, frozenset[str]] = { + label: frozenset(fields.split()) + for label, fields in { + # Configuration written by organizers or administrators + "auth.Permission": "codename", + "oauth2_provider.Application": "client_id redirect_uris post_logout_redirect_uris name allowed_origins", + "core.Organization": "muncipality", + "access.CBACEntry": "claims", + "access.InternalEmailAlias": "email_address", + "access.SMTPServer": "ssh_username password_file_path_on_server", + "intra.Team": "email", + "labour.LabourEventMeta": "monitor_email contact_email", + "labour.PersonnelClass": "perks", + "program_v2.ProgramV2EventMeta": "contact_email", + "tickets_v2.TicketsV2EventMeta": "contact_email", + "programme.ProgrammeEventMeta": "contact_email", + "tickets.TicketsEventMeta": "contact_email", + "tickets.Product": "notify_email code", + "programme.AlternativeProgrammeForm": "programme_form_code v2_dimensions", + "programme.Category": "notes v2_dimensions", + "programme.Room": "notes v2_dimensions", + "programme.Role": "perks", + "programme.SpecialReservation": "code", + "programme.Tag": "v2_dimensions", + "dimensions.UniverseAnnotation": "form_fields", + "involvement.InvolvementToGroupMapping": "required_dimensions", + "involvement.InvolvementToBadgeMapping": "required_dimensions annotations", + "badges.SurveyToBadgeMapping": "required_dimensions annotations", + "event_log_v2.Entry": "entry_type", + "event_log_v2.Subscription": "entry_type", + "forms.Survey": "slug cached_key_fields cached_default_response_dimensions cached_default_involvement_dimensions", + "forms.Form": "title description thank_you_message fields cached_enriched_fields", + "forms.Projection": ( + "default_language_code splats required_dimensions projected_dimensions filterable_dimensions" + " order_by special_fields" + ), + # Organizers write these to many recipients at once, so they are not about any one person. + "messages_v2.Message": "subject body recipient_filters", + "messages_v2.MessageReplyTo": "email", + # Programs, including their hosts, are published on the public schedule. + "program_v2.Program": ( + "title slug description annotations cached_dimensions cached_combined_dimensions cached_color" + ), + "program_v2.ScheduleItem": ( + "cached_dimensions cached_combined_dimensions cached_location annotations cached_combined_annotations" + ), + # Roles, perks and dimensions that organizers assign + "involvement.Invitation": "cached_dimensions", + "involvement.Involvement": "title cached_dimensions annotations", + "messages_v2.MessageRecipient": "cached_dimensions", + "forms.Response": "cached_dimensions", + "labour.Signup": "xxx_interim_shifts job_title override_formatted_perks", + "labour.ArchivedSignup": "job_title", + "badges.Badge": "job_title perks", + "intra.TeamMember": "override_job_title", + "programme.ProgrammeRole": "override_perks", + "paikkala.Ticket": "qualifier_text_cache", + "tickets_v2.Order": "product_data", + "payments.CheckoutPayment": "items", + }.items() +} + + +def find_orphan_tables() -> list[str]: + """ + Tables that no installed model owns, such as those left behind by removed apps. No rule can + cover them, and they may hold personal data. + """ + known_tables = set(connection.introspection.django_table_names(include_views=False)) | {"django_migrations"} + with connection.cursor() as cursor: + cursor.execute( + """ + SELECT c.relname + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND NOT c.relispartition + ORDER BY c.relname + """ + ) + return [table for (table,) in cursor.fetchall() if table not in known_tables] + + +def drop_orphan_tables() -> str: + tables = find_orphan_tables() + with connection.cursor() as cursor: + for table in tables: + cursor.execute(f"DROP TABLE {connection.ops.quote_name(table)} CASCADE") + return f"dropped {', '.join(tables) or 'nothing'}" + + +def pseudonymize() -> Iterable[tuple[str, str]]: + """Applies every rule in order, yielding the model label and a summary as each finishes.""" + yield "orphan tables", drop_orphan_tables() + for rule in RULES: + yield rule.model_label, rule.apply(rule.model) + + +PERSONAL_FIELD_NAME = re.compile( + r"e_?mail|phone|first_name|surname|last_name|full_name|nick|address|(^|_)ip($|_)|token|secret|password" + r"|hetu|ssn|identification|birth|diet|allerg|notes?$|handle|city|zip|muncipality|iban|key|code|username", + re.IGNORECASE, +) +TEXT_LIKE_FIELDS = ( + models.CharField, + models.TextField, + models.JSONField, + models.BinaryField, + models.GenericIPAddressField, + HStoreField, + ArrayField, +) + + +def is_linked_to_person(model: type[models.Model]) -> bool: + person_models = (Person, apps.get_model("auth.User")) + return model in person_models or any( + field.is_relation and field.related_model in person_models for field in model._meta.concrete_fields + ) + + +def is_suspicious(model: type[models.Model], field: models.Field) -> bool: + """ + Could this field hold personal data or a secret? Free text on a row about a person could hold + anything, so every such field is suspicious. Elsewhere only fields named like personal data are. + """ + if field.primary_key or field.choices or isinstance(field, (models.SlugField, models.UUIDField)): + return False + if isinstance(field, models.DateField) and not isinstance(field, models.DateTimeField): + return bool(PERSONAL_FIELD_NAME.search(field.name)) + if not isinstance(field, TEXT_LIKE_FIELDS): + return False + return ( + isinstance(field, models.JSONField) + or is_linked_to_person(model) + or bool(PERSONAL_FIELD_NAME.search(field.name)) + ) + + +def managed_models() -> Iterable[type[models.Model]]: + for model in apps.get_models(): + if model._meta.managed and not model._meta.proxy: + yield model + + +def find_unclassified_fields() -> list[str]: + covered: dict[str, frozenset[str]] = {} + for rule in RULES: + covered[rule.model_label] = covered.get(rule.model_label, frozenset()) | rule.fields + + unclassified = [] + for model in managed_models(): + label = model._meta.label + rule_fields = covered.get(label, frozenset()) + if rule_fields == ALL_FIELDS: + continue + reviewed_fields = rule_fields | NOT_PERSONAL.get(label, frozenset()) + unclassified.extend( + f"{label}.{field.name}" + for field in model._meta.concrete_fields + if field.name not in reviewed_fields and is_suspicious(model, field) + ) + + return unclassified + + +def find_stale_classifications() -> list[str]: + """Rules and `NOT_PERSONAL` entries that name a model or field that no longer exists.""" + classified = [(rule.model_label, rule.fields - ALL_FIELDS) for rule in RULES] + list(NOT_PERSONAL.items()) + + stale = [] + for label, field_names in classified: + try: + model = apps.get_model(label) + except LookupError: + stale.append(label) + continue + existing = {field.name for field in model._meta.concrete_fields} + stale.extend(f"{label}.{name}" for name in sorted(field_names - existing)) + + return stale diff --git a/kompassi/core/test_pseudonymization.py b/kompassi/core/test_pseudonymization.py new file mode 100644 index 000000000..d1fcb55bb --- /dev/null +++ b/kompassi/core/test_pseudonymization.py @@ -0,0 +1,111 @@ +import pytest +from django.db import connection + +from kompassi.core.models import Event, Person +from kompassi.core.pseudonymization import ( + REDACTED, + find_orphan_tables, + find_stale_classifications, + find_unclassified_fields, + pseudonymize, + pseudonymize_form_data, +) +from kompassi.forms.models.form import Form +from kompassi.forms.models.response import Response +from kompassi.forms.models.survey import Survey + + +def test_every_suspicious_field_is_classified(): + """ + A field that may hold personal data needs a rule in RULES or an entry in NOT_PERSONAL. + `python manage.py pseudonymize_db --check` lists the fields that need one. + """ + assert find_unclassified_fields() == [] + + +def test_no_classification_names_a_missing_field(): + assert find_stale_classifications() == [] + + +def test_pseudonymize_form_data(): + fields = [ + dict(slug="name", type="SingleLineText"), + dict(slug="empty", type="SingleLineText"), + dict(slug="shirt", type="SingleSelect"), + dict(slug="days", type="MultiSelect"), + dict(slug="birthday", type="DateField"), + dict(slug="photo", type="FileUpload"), + dict(slug="hetu", type="SingleLineText", encryptTo=["someone"]), + ] + form_data = { + "name": "Markku Mahtinen", + "empty": "", + "shirt": "xl", + "days.friday": "on", + "birthday": "1984-01-01", + "photo": ["https://example.com/photo.jpg"], + "hetu": "eyJhbGciOi...", + # answer to a field that has since been removed from the form + "phone": "+358501234567", + } + + assert pseudonymize_form_data(fields, form_data) == { + "name": REDACTED, + "empty": "", + "shirt": "xl", + "days.friday": "on", + } + + +def test_pseudonymize_form_data_redaction_wins(): + """A key that matches both a kept and a redacted field is redacted whatever the field order.""" + kept = dict(slug="days", type="MultiSelect") + redacted = dict(slug="days.friday", type="SingleLineText") + form_data = {"days.friday": "Markku Mahtinen"} + + assert pseudonymize_form_data([redacted, kept], form_data) == {"days.friday": REDACTED} + assert pseudonymize_form_data([kept, redacted], form_data) == {"days.friday": REDACTED} + + +@pytest.mark.django_db +def test_pseudonymize(): + """ + Runs every rule against a database that has a little data in it, so that each rule's SQL + gets executed at least once. + """ + person, _ = Person.get_or_create_dummy() + event, _ = Event.get_or_create_dummy() + survey = Survey.objects.create(event=event, slug="pseudonymize-survey") + form = Form.objects.create( + event=event, + survey=survey, + language="en", + fields=[dict(slug="name", type="SingleLineText", isKeyField=True)], + ) + survey.refresh_cached_key_fields(form) + response = Response.objects.create(form=form, form_data={"name": "Markku"}, ip_address="192.0.2.1") + response.refresh_cached_fields() + with connection.cursor() as cursor: + # left behind by an app that has since been removed + cursor.execute("CREATE TABLE sms_smsmessageout (id serial PRIMARY KEY, message text)") + + list(pseudonymize()) + + assert find_orphan_tables() == [] + + person.refresh_from_db() + assert person.first_name == f"Person{person.pk}" + assert person.email == f"person{person.pk}@example.com" + assert person.birth_date.month == person.birth_date.day == 1 + + user = person.user + assert user + user.refresh_from_db() + assert user.username == f"user{user.pk}" + assert user.email == person.email + assert not user.has_usable_password() + + response.refresh_from_db() + assert response.form_data == {"name": REDACTED} + assert response.cached_key_fields == {"name": REDACTED} + assert response.ip_address == "" diff --git a/scripts/load-dump.sh b/scripts/load-dump.sh new file mode 100755 index 000000000..9075be354 --- /dev/null +++ b/scripts/load-dump.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Replaces the docker compose development database with a dump from scripts/pseudonymized-dump.sh: +# +# scripts/load-dump.sh kompassi-production-20260929.pgdump +# +# Pseudonymization leaves no usable passwords or OAuth2 client secrets, so afterwards this +# recreates the dev superuser mahti/mahti and the OAuth2 client of the local V2 frontend. +set -eu + +if [ $# -ne 1 ]; then + echo "usage: $0 DUMP_FILE" >&2 + exit 2 +fi +dump="$1" + +cd "$(dirname "$0")/.." + +log() { + echo "==> $*" >&2 +} + +log "Stopping the services that hold database connections" +docker compose up -d router postgres redis +docker compose stop backend worker uvicorn + +log "Recreating the database" +until docker compose exec -T postgres pg_isready -q -U kompassi; do + sleep 1 +done +docker compose exec -T postgres dropdb -U kompassi --maintenance-db=postgres --if-exists --force kompassi +docker compose exec -T postgres createdb -U kompassi kompassi + +log "Restoring $dump" +# Not with --jobs: a parallel restore creates the triggers of a partitioned table while its +# partitions are still loading, and the triggers fail on the rows being copied. +docker compose exec -T postgres pg_restore -U kompassi -d kompassi --no-owner --no-acl --exit-on-error <"$dump" + +log "Migrating and recreating development credentials" +docker compose exec -T redis redis-cli -n 1 flushdb >/dev/null +docker compose run --rm --no-deps backend sh -c "python manage.py migrate && python manage.py setup_api_v2" + +log "Done. Start the rest with: docker compose up" diff --git a/scripts/pseudonymized-dump.sh b/scripts/pseudonymized-dump.sh new file mode 100755 index 000000000..e714d2dca --- /dev/null +++ b/scripts/pseudonymized-dump.sh @@ -0,0 +1,107 @@ +#!/bin/sh +# Writes a pseudonymized pg_dump (custom format) of a deployed Kompassi database to +# NAMESPACE-YYYYMMDD.pgdump in the current directory: +# +# scripts/pseudonymized-dump.sh # kompassi-production-YYYYMMDD.pgdump +# scripts/pseudonymized-dump.sh kompassi-staging # kompassi-staging-YYYYMMDD.pgdump +# +# The personal data never leaves the cluster. A short-lived pod copies the database into a +# Postgres of its own, runs `manage.py pseudonymize_db` there, and only the result is streamed +# out. The pod is built from the pod template of the `kompassi` Deployment, so it runs the +# deployed image with the deployed configuration: the pseudonymization rules always match the +# schema they run against. Load the dump with scripts/load-dump.sh. +set -eu + +namespace="${1:-kompassi-production}" +postgres_image=postgres:18 +database=kompassi_pseudo +socket=/var/run/postgresql +pod="kompassi-pseudonymize-$(date +%Y%m%d%H%M%S)" +output="$namespace-$(date +%Y%m%d).pgdump" +# Written under another name until complete, so an interrupted run never leaves a truncated +# dump that looks finished. +partial="$output.partial" + +log() { + echo "==> $*" >&2 +} + +deployment="$(kubectl -n "$namespace" get deployment kompassi -o json)" + +# The postgres container runs as the image's own postgres user because initdb needs the uid to +# have a passwd entry. It only listens on a socket shared with the pseudonymize container. +printf '%s' "$deployment" | jq \ + --arg pod "$pod" --arg postgres_image "$postgres_image" --arg database "$database" --arg socket "$socket" ' + .spec.template.spec as $spec + | ($spec.containers[] | select(.name == "master")) as $backend + | [$backend.env[] | select(.name | startswith("POSTGRES_"))] as $source + | { + apiVersion: "v1", + kind: "Pod", + metadata: {name: $pod, labels: {stack: "kompassi", component: "pseudonymize"}}, + spec: ($spec | del(.initContainers, .affinity, .terminationGracePeriodSeconds) + { + restartPolicy: "Never", + activeDeadlineSeconds: 7200, + volumes: ($spec.volumes + [ + {name: "postgres-data", emptyDir: {}}, + {name: "postgres-socket", emptyDir: {}} + ]), + containers: [ + { + name: "postgres", + image: $postgres_image, + command: ["sh", "-c", "initdb --auth=trust --username=postgres --encoding=UTF8 --locale-provider=icu --icu-locale=fi-FI -D /var/lib/postgresql/data && exec postgres -D /var/lib/postgresql/data -c listen_addresses= -c unix_socket_directories=\($socket)"], + env: [$source[] | .name |= sub("^POSTGRES_"; "SOURCE_")], + securityContext: ($backend.securityContext + {runAsUser: 999, runAsGroup: 999}), + volumeMounts: [ + {name: "postgres-data", mountPath: "/var/lib/postgresql"}, + {name: "postgres-socket", mountPath: $socket}, + {name: "kompassi-temp", mountPath: "/tmp"} + ] + }, + ($backend | del(.args, .ports, .startupProbe, .readinessProbe, .livenessProbe, .lifecycle) + { + name: "pseudonymize", + command: ["sleep", "infinity"], + env: ([.env[] | select(.name | startswith("POSTGRES_") | not)] + [ + {name: "POSTGRES_HOSTNAME", value: $socket}, + {name: "POSTGRES_DATABASE", value: $database}, + {name: "POSTGRES_USERNAME", value: "postgres"}, + {name: "POSTGRES_PASSWORD", value: ""}, + {name: "POSTGRES_SSLMODE", value: "disable"} + ]), + volumeMounts: (.volumeMounts + [{name: "postgres-socket", mountPath: $socket}]) + }) + ] + }) + } +' | kubectl -n "$namespace" apply -f - >/dev/null + +trap 'rm -f "$partial"; kubectl -n "$namespace" delete pod "$pod" --wait=false >/dev/null' EXIT + +log "Waiting for pod $namespace/$pod" +kubectl -n "$namespace" wait --for=condition=Ready "pod/$pod" --timeout=300s >/dev/null +# kubectl reports each failed attempt on stderr while initdb is still running. +until kubectl -n "$namespace" exec "$pod" -c postgres -- pg_isready -q -h "$socket" 2>/dev/null; do + sleep 2 +done + +log "Copying the $namespace database into the pod" +kubectl -n "$namespace" exec "$pod" -c postgres -- sh -c ' + set -eu + createdb -h "$1" "$2" + # Through a file rather than a pipe, so that a failing pg_dump stops the script. + PGPASSWORD="$SOURCE_PASSWORD" pg_dump -Fc --no-owner --no-acl -f "$3" \ + "host=$SOURCE_HOSTNAME dbname=$SOURCE_DATABASE user=$SOURCE_USERNAME sslmode=$SOURCE_SSLMODE" + pg_restore -h "$1" -d "$2" --no-owner --no-acl --exit-on-error "$3" + rm "$3" +' sh "$socket" "$database" /var/lib/postgresql/source.pgdump >&2 + +log "Pseudonymizing" +kubectl -n "$namespace" exec "$pod" -c pseudonymize -- python manage.py pseudonymize_db --really >&2 + +log "Streaming the pseudonymized dump" +kubectl -n "$namespace" exec "$pod" -c postgres -- pg_dump -Fc --no-owner --no-acl -h "$socket" "$database" \ + >"$partial" +mv "$partial" "$output" + +log "Wrote $output"