diff --git a/Dockerfile b/Dockerfile index 40df834..5a8e0e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG FROM=docker.io/alpine:3.23 +ARG FROM=docker.io/alpine:3.24 FROM ${FROM} AS builder RUN apk add --no-cache \ @@ -12,12 +12,12 @@ RUN apk add --no-cache \ jpeg-dev \ libevent-dev \ libffi-dev \ + libpq-dev \ libxslt-dev \ xmlsec-dev \ make \ musl-dev \ openldap-dev \ - postgresql-dev \ py3-pip \ python3-dev \ && python3 -m venv /opt/peering-manager/venv \ @@ -63,16 +63,17 @@ RUN apk add --no-cache \ libffi \ libjpeg-turbo \ libldap \ + libpq \ libsasl \ libxslt \ openssl \ - postgresql-client \ - postgresql-libs \ + postgresql18-client \ py3-pip \ python3 \ tini \ - unit \ - unit-python3 \ + tzdata \ + freeunit \ + freeunit-python3 \ util-linux COPY --from=builder /opt/peering-manager/venv /opt/peering-manager/venv @@ -88,6 +89,7 @@ COPY docker/configuration.docker.py /opt/peering-manager/peering_manager/configu COPY docker/ldap_config.docker.py /opt/peering-manager/peering_manager/ldap_config.py COPY docker/docker-entrypoint.sh /opt/peering-manager/docker-entrypoint.sh COPY docker/run-command.sh /opt/peering-manager/run-command.sh +COPY docker/healthcheck.sh /opt/peering-manager/healthcheck.sh COPY docker/launch-peering-manager.sh /opt/peering-manager/launch-peering-manager.sh COPY configuration/ /etc/peering-manager/config/ COPY docker/nginx-unit.json /etc/unit/ @@ -97,7 +99,7 @@ WORKDIR /opt/peering-manager # Must set permissions for '/opt/peering-manager/static' directory # to g+w so that `./manage.py collectstatic` can be executed during # container startup. -RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \ +RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ /opt/unit/static-root/ \ && chown -R unit:root /opt/unit/ \ && chmod -R g+w /opt/unit/ \ && cd /opt/peering-manager/ \ diff --git a/README.md b/README.md index 729c227..f123d05 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,36 @@ To run the Peering Manager application stack with Docker Compose, copy the `docker-compose.override.yml.example` file into `docker-compose.override.yml` and override definitions from this file. +### Scheduled Tasks + +Peering Manager 1.11 and later run housekeeping and PeeringDB synchronisation +from the `rqworker` service. Manage the schedule of each task from +**Admin > Scheduled Tasks** in the web interface. + +The `housekeeping` and `peeringdb-sync` services stay in `docker-compose.yml` +for users who prefer an external scheduler. They do not start by default. +First disable the matching tasks in **Admin > Scheduled Tasks**, then start +them: + +```shell +docker compose --profile external-scheduler up --detach +``` + +Do not run a task twice. Two runs of the same task waste resources, and a +PeeringDB synchronisation can take about 90 minutes. + +### Base Path + +Set `BASE_PATH` in `env/peering-manager.env` to serve Peering Manager from a +directory, for example `https://example.com/peering/`: + +```shell +BASE_PATH=peering/ +``` + +The container applies the value to the URLs, to the static files and to its +health check. + ## About This work is based on the great diff --git a/build.sh b/build.sh index 1e2ec7c..feb2df4 100755 --- a/build.sh +++ b/build.sh @@ -216,7 +216,7 @@ fi # Determining the value for DOCKER_FROM ### if [ -z "${DOCKER_FROM}" ]; then - DOCKER_FROM="docker.io/alpine:3.23" + DOCKER_FROM="docker.io/alpine:3.24" fi ### diff --git a/configuration/configuration.py b/configuration/configuration.py index 9f52cdf..742ca42 100644 --- a/configuration/configuration.py +++ b/configuration/configuration.py @@ -1,7 +1,9 @@ import json import re +from collections.abc import Callable from os import environ -from typing import Any, Callable +from pathlib import Path +from typing import Any # For reference see: https://docs.peering-manager.net/configuration/ @@ -9,12 +11,10 @@ # Read secret from file def _read_secret(secret_name: str, default: str | None = None) -> str | None: try: - f = open(f"/run/secrets/{secret_name}", encoding="utf-8") - except EnvironmentError: - return default - else: - with f: + with Path(f"/run/secrets/{secret_name}").open(encoding="utf-8") as f: return f.readline().strip() + except OSError: + return default # If the `map_fn` isn't defined, then the value that is read from the environment (or @@ -28,7 +28,7 @@ def _read_secret(secret_name: str, default: str | None = None) -> str | None: def _environ_get_and_map( variable_name: str, default: str | None = None, - map_fn: Callable[[str], Any | None] = None, + map_fn: Callable[[str], Any | None] | None = None, ) -> Any | None: env_value = environ.get(variable_name, default) @@ -158,6 +158,10 @@ def _AS_STRUCT(value: str) -> dict[Any, Any] | list[Any]: if "ADMINS" in environ: ADMINS = _environ_get_and_map("ADMINS", "[]", _AS_STRUCT) +# Serve Peering Manager from a directory. For example, set BASE_PATH to "peering/" +# to serve it at https://example.com/peering/. Empty means the root of the domain. +BASE_PATH = environ.get("BASE_PATH", "") + # Maximum number of days to retain logged changes. Set to 0 to retain changes # indefinitely. (Default: 90) if "CHANGELOG_RETENTION" in environ: @@ -172,6 +176,23 @@ def _AS_STRUCT(value: str) -> dict[Any, Any] | list[Any]: elif "JOBRESULT_RETENTION" in environ: JOB_RETENTION = _environ_get_and_map("JOBRESULT_RETENTION", None, _AS_INT) +# Number of seconds to keep the details of a BGP session, an IX-API answer or a +# prefix list in the cache. (Defaults: 900, 900 and 3600) +if "CACHE_BGP_DETAIL_TIMEOUT" in environ: + CACHE_BGP_DETAIL_TIMEOUT = _environ_get_and_map( + "CACHE_BGP_DETAIL_TIMEOUT", None, _AS_INT + ) +if "CACHE_IXAPI_TIMEOUT" in environ: + CACHE_IXAPI_TIMEOUT = _environ_get_and_map("CACHE_IXAPI_TIMEOUT", None, _AS_INT) +if "CACHE_PREFIX_LIST_TIMEOUT" in environ: + CACHE_PREFIX_LIST_TIMEOUT = _environ_get_and_map( + "CACHE_PREFIX_LIST_TIMEOUT", None, _AS_INT + ) + +# Number of seconds to wait for an answer from an IX-API endpoint. (Default: 30) +if "IXAPI_TIMEOUT" in environ: + IXAPI_TIMEOUT = _environ_get_and_map("IXAPI_TIMEOUT", None, _AS_INT) + # API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set # to True, all origins will be allowed. Otherwise, define a list of allowed origins # using either CORS_ORIGIN_WHITELIST or CORS_ORIGIN_REGEX_WHITELIST. For more @@ -258,6 +279,13 @@ def _AS_STRUCT(value: str) -> dict[Any, Any] | list[Any]: if "MAX_PAGE_SIZE" in environ: MAX_PAGE_SIZE = _environ_get_and_map("MAX_PAGE_SIZE", None, _AS_INT) +# Default preferences applied to a user who did not set their own, as a JSON +# object. For example: DEFAULT_USER_PREFERENCES={"pagination": {"per_page": 100}} +if "DEFAULT_USER_PREFERENCES" in environ: + DEFAULT_USER_PREFERENCES = _environ_get_and_map( + "DEFAULT_USER_PREFERENCES", None, _AS_STRUCT + ) + # Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' METRICS_ENABLED = _environ_get_and_map("METRICS_ENABLED", "False", _AS_BOOL) @@ -318,6 +346,9 @@ def _AS_STRUCT(value: str) -> dict[Any, Any] | list[Any]: # The name to use for the csrf token cookie. CSRF_COOKIE_NAME = environ.get("CSRF_COOKIE_NAME", "csrftoken") +# Send the csrf token cookie only over HTTPS. (Default: False) +CSRF_COOKIE_SECURE = _environ_get_and_map("CSRF_COOKIE_SECURE", "False", _AS_BOOL) + # Cross-Site-Request-Forgery-Attack settings. If Peering Manager is sitting behind a # reverse proxy, you might need to set the CSRF_TRUSTED_ORIGINS flag. Django 4.0 # requires to specify the URL Scheme in this setting. An example environment variable @@ -328,6 +359,21 @@ def _AS_STRUCT(value: str) -> dict[Any, Any] | list[Any]: # The name to use for the session cookie. SESSION_COOKIE_NAME = environ.get("SESSION_COOKIE_NAME", "sessionid") +# Send the session cookie only over HTTPS. (Default: False) +SESSION_COOKIE_SECURE = _environ_get_and_map("SESSION_COOKIE_SECURE", "False", _AS_BOOL) + +# Header and value that tell Peering Manager a reverse proxy handled the request +# over HTTPS, as two space separated words. Empty ignores such a header. +# (Default: "HTTP_X_FORWARDED_PROTO https") +if "SECURE_PROXY_SSL_HEADER" in environ: + SECURE_PROXY_SSL_HEADER = ( + tuple(_environ_get_and_map("SECURE_PROXY_SSL_HEADER", None, _AS_LIST)) or None + ) + +# Use the X-Forwarded-Host header to build absolute URLs. (Default: True) +if "USE_X_FORWARDED_HOST" in environ: + USE_X_FORWARDED_HOST = _environ_get_and_map("USE_X_FORWARDED_HOST", None, _AS_BOOL) + # By default, Peering Manager will store session data in the database. Alternatively, # a file path can be specified here to use local file storage instead. (This can be # useful for enabling authentication on a standby instance with read-only database @@ -338,6 +384,22 @@ def _AS_STRUCT(value: str) -> dict[Any, Any] | list[Any]: # Time zone (default: UTC) TIME_ZONE = environ.get("TIME_ZONE", "UTC") +# Formats used to display a date and a time. See the Django documentation for the +# accepted format specifiers: +# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date +if "DATE_FORMAT" in environ: + DATE_FORMAT = environ.get("DATE_FORMAT", "jS F, Y") +if "DATETIME_FORMAT" in environ: + DATETIME_FORMAT = environ.get("DATETIME_FORMAT", "jS F, Y G:i") +if "SHORT_DATE_FORMAT" in environ: + SHORT_DATE_FORMAT = environ.get("SHORT_DATE_FORMAT", "Y-m-d") +if "SHORT_DATETIME_FORMAT" in environ: + SHORT_DATETIME_FORMAT = environ.get("SHORT_DATETIME_FORMAT", "Y-m-d H:i") +if "SHORT_TIME_FORMAT" in environ: + SHORT_TIME_FORMAT = environ.get("SHORT_TIME_FORMAT", "H:i:s") +if "TIME_FORMAT" in environ: + TIME_FORMAT = environ.get("TIME_FORMAT", "G:i") + # Text to include on the login page above the login form. HTML is allowed. if "BANNER_LOGIN" in environ: BANNER_LOGIN = environ.get("BANNER_LOGIN", None) @@ -360,13 +422,11 @@ def _AS_STRUCT(value: str) -> dict[Any, Any] | list[Any]: ) if "NAPALM_TIMEOUT" in environ: NAPALM_TIMEOUT = _environ_get_and_map("NAPALM_TIMEOUT", "10", _AS_INT) -NAPALM_ARGS = dict( - [ - (var[len("NAPALM_ARG_") :].lower(), environ.get(var)) - for var in environ.keys() - if var.startswith("NAPALM_ARG_") - ] -) +NAPALM_ARGS = { + var[len("NAPALM_ARG_") :].lower(): environ.get(var) + for var in environ + if var.startswith("NAPALM_ARG_") +} # The path to the bgpq3 or bgpq4 binary if "BGPQ3_PATH" in environ: @@ -444,6 +504,20 @@ def _AS_STRUCT(value: str) -> dict[Any, Any] | list[Any]: "VALIDATE_BGP_COMMUNITY_VALUE", "True", _AS_BOOL ) +# Status given to the sessions that Peering Manager creates when an operator +# accepts a peering request. (Default: "requested") +if "PEERING_REQUEST_SESSION_STATUS" in environ: + PEERING_REQUEST_SESSION_STATUS = environ.get( + "PEERING_REQUEST_SESSION_STATUS", "requested" + ) + +# Block the creation of a session that is already part of a pending peering +# request. (Default: False) +if "PEERING_REQUEST_BLOCKS_SESSION_CREATION" in environ: + PEERING_REQUEST_BLOCKS_SESSION_CREATION = _environ_get_and_map( + "PEERING_REQUEST_BLOCKS_SESSION_CREATION", "False", _AS_BOOL + ) + # When merging configuration contexts, Peering Manager needs to know what # should happen to nested dictionaries/hashes and to list. These two options # can be changed to reproduce the wanted behaviour. They are similar to diff --git a/docker-compose.override.yml.example b/docker-compose.override.yml.example index d77ae77..a1a1844 100644 --- a/docker-compose.override.yml.example +++ b/docker-compose.override.yml.example @@ -12,8 +12,11 @@ services: # should be higher than on subsequent starts. For the first start after # major version upgrades of Peering Manager the start_period might also # need to be set higher. - # Default value in docker-compose.yml is 60s - # start_period: 90s + # Default value in docker-compose.yml is 240s + # start_period: 300s # environment: + # Serve Peering Manager from a directory, for example + # https://example.com/peering/ + # BASE_PATH: "peering/" # SKIP_SUPERUSER: "false" # SUPERUSER_PASSWORD: "" diff --git a/docker-compose.yml b/docker-compose.yml index c7e0ddc..b565a16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,8 @@ services: env_file: env/peering-manager.env user: "unit:root" healthcheck: - test: curl -f http://localhost:8080/login/ || exit 1 - start_period: 90s + test: /opt/peering-manager/healthcheck.sh + start_period: 240s # Allow migrations to run, some can be expensive timeout: 3s interval: 15s volumes: @@ -30,8 +30,13 @@ services: start_period: 20s timeout: 3s interval: 15s + # As of 1.11 the rqworker runs these two tasks itself, on a schedule managed + # in Admin > Scheduled Tasks. The services below serve an external scheduler + # instead: disable the tasks there, then start this profile. housekeeping: <<: *peering-manager + profiles: + - external-scheduler environment: COMMAND_INTERVAL: 86400 depends_on: @@ -47,6 +52,8 @@ services: interval: 15s peeringdb-sync: <<: *peering-manager + profiles: + - external-scheduler environment: COMMAND_INTERVAL: 86400 depends_on: diff --git a/docker/healthcheck.sh b/docker/healthcheck.sh new file mode 100755 index 0000000..a8e915b --- /dev/null +++ b/docker/healthcheck.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Checks that Peering Manager answers, whatever base path it is served under + +BASE_PATH_FILE="/opt/unit/base-path" + +if [ -r "${BASE_PATH_FILE}" ]; then + # Written by the launch script from the Django settings + BASE_PATH="$(cat "${BASE_PATH_FILE}")" +else + # Fall back to the variable, normalised the way Django normalises it + BASE_PATH="$(echo "${BASE_PATH-}" | sed -e 's|^/*||' -e 's|/*$||')" + if [ -n "${BASE_PATH}" ]; then + BASE_PATH="${BASE_PATH}/" + fi +fi + +exec curl --fail --silent --output /dev/null "http://localhost:8080/${BASE_PATH}login/" diff --git a/docker/launch-peering-manager.sh b/docker/launch-peering-manager.sh index c38d612..b543e23 100755 --- a/docker/launch-peering-manager.sh +++ b/docker/launch-peering-manager.sh @@ -1,7 +1,64 @@ #!/bin/bash UNIT_CONFIG="${UNIT_CONFIG-/etc/unit/nginx-unit.json}" +UNIT_RENDERED_CONFIG="/opt/unit/nginx-unit.json" UNIT_SOCKET="/opt/unit/unit.sock" +STATIC_ROOT="/opt/unit/static-root" +BASE_PATH_FILE="/opt/unit/base-path" + +# A user can set the base path with BASE_PATH but also in any configuration file, +# so ask Django for the value that the application really uses. +resolve_base_path() { + local answer + + answer="$( + PYTHONPATH=/opt/peering-manager \ + DJANGO_SETTINGS_MODULE=peering_manager.settings \ + /opt/peering-manager/venv/bin/python -c \ + 'from django.conf import settings; print(f"__PM_BASE_PATH__:{settings.BASE_PATH}:")' \ + 2>/dev/null | grep '^__PM_BASE_PATH__:' | tail -1 + )" + + if [ -n "${answer}" ]; then + answer="${answer#__PM_BASE_PATH__:}" + BASE_PATH="${answer%:}" + return 0 + fi + + echo "⚠️ Could not read the base path from the configuration; using BASE_PATH" + + # Normalise the variable the way Django normalises it + BASE_PATH="$(echo "${BASE_PATH-}" | sed -e 's|^/*||' -e 's|/*$||')" + if [ -n "${BASE_PATH}" ]; then + BASE_PATH="${BASE_PATH}/" + fi +} + +publish_base_path() { + # The health check reads it to build the URL it asks for + if ! echo "${BASE_PATH}" >"${BASE_PATH_FILE}"; then + echo "⚠️ Could not write the base path to ${BASE_PATH_FILE}" + return 1 + fi +} + +prepare_static_files() { + # Unit cannot strip the base path from a URI, so publish the static directory + # under the base path with a symbolic link + if ! (rm -rf "${STATIC_ROOT}" && + mkdir -p "${STATIC_ROOT}/${BASE_PATH}" && + ln -sfn /opt/peering-manager/static "${STATIC_ROOT}/${BASE_PATH}static"); then + echo "⚠️ Could not publish the static files in ${STATIC_ROOT}" + return 1 + fi +} + +render_configuration() { + if ! sed -e "s|__BASE_PATH__|${BASE_PATH}|g" "${UNIT_CONFIG}" >"${UNIT_RENDERED_CONFIG}"; then + echo "⚠️ Could not write the Unit configuration to ${UNIT_RENDERED_CONFIG}" + return 1 + fi +} load_configuration() { MAX_WAIT=10 @@ -30,7 +87,7 @@ load_configuration() { --output /dev/null \ --write-out '%{http_code}' \ --request PUT \ - --data-binary "@${UNIT_CONFIG}" \ + --data-binary "@${UNIT_RENDERED_CONFIG}" \ --unix-socket $UNIT_SOCKET \ http://localhost/config ) @@ -43,6 +100,11 @@ load_configuration() { echo "✅ Unit configuration loaded successfully" } +resolve_base_path +publish_base_path +prepare_static_files +render_configuration + load_configuration & exec unitd \ diff --git a/docker/nginx-unit.json b/docker/nginx-unit.json index 5c4bee4..9e4b896 100644 --- a/docker/nginx-unit.json +++ b/docker/nginx-unit.json @@ -17,10 +17,10 @@ "main": [ { "match": { - "uri": "/static/*" + "uri": "/__BASE_PATH__static/*" }, "action": { - "share": "/opt/peering-manager${uri}" + "share": "/opt/unit/static-root${uri}" } }, {