From 221620846364ad034a924b60360c0fa157dd368d Mon Sep 17 00:00:00 2001 From: Hans Ott Date: Fri, 4 Sep 2026 15:06:27 +0200 Subject: [PATCH] Move install to separate step and give more leeway to benchmarks The install step is executed per retry of the benchmark (up to 3 times) More leeway because of GitHub actions runners being slow and unreliable for testing performance. --- .github/workflows/benchmark.yml | 10 ++++++++-- benchmarks/flask-mysql-benchmarks.js | 16 ++++++++-------- benchmarks/wrk_benchmark/flask_mysql.py | 2 +- benchmarks/wrk_benchmark/flask_mysql_uwsgi.py | 4 ++-- .../wrk_benchmark/starlette_postgres_uvicorn.py | 2 +- sample-apps/common.mk | 4 +--- sample-apps/flask-mysql-uwsgi/Makefile | 10 ++++++---- sample-apps/flask-mysql/Makefile | 10 ++++++---- sample-apps/starlette-postgres-uvicorn/Makefile | 10 ++++++---- 9 files changed, 39 insertions(+), 29 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 338ee8122..4f7efc820 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -63,9 +63,12 @@ jobs: run: | python -m pip install --upgrade pip make install && make build + - name: Install flask-mysql + working-directory: ./sample-apps/flask-mysql + run: make install - name: Start flask-mysql working-directory: ./sample-apps/flask-mysql - run: nohup make runBenchmark & nohup make runZenDisabled & + run: nohup make startBenchmark & nohup make startZenDisabled & - name: Check app health (max 3 attempts) uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 @@ -113,9 +116,12 @@ jobs: run: | python -m pip install --upgrade pip make install && make build + - name: Install app + working-directory: ./sample-apps/${{ matrix.app }} + run: make install - name: Start app working-directory: ./sample-apps/${{ matrix.app }} - run: nohup make runBenchmark & nohup make runZenDisabled & + run: nohup make startBenchmark & nohup make startZenDisabled & - name: Check app health (max 3 attempts) uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 diff --git a/benchmarks/flask-mysql-benchmarks.js b/benchmarks/flask-mysql-benchmarks.js index a35d61121..b9e120cc2 100644 --- a/benchmarks/flask-mysql-benchmarks.js +++ b/benchmarks/flask-mysql-benchmarks.js @@ -10,42 +10,42 @@ export const options = { vus: 1, // Number of virtual users thresholds: { test_40mb_payload: [{ - threshold: "avg<15", // This is a higher threshold due to the data being processed + threshold: "avg<25", // This is a higher threshold due to the data being processed abortOnFail: true, delayAbortEval: '10s', }], test_multiple_queries: [{ - threshold: "avg<6", + threshold: "avg<10", abortOnFail: true, delayAbortEval: '10s', }], test_multiple_queries_with_big_body: [{ - threshold: "avg<6", + threshold: "avg<10", abortOnFail: true, delayAbortEval: '10s', }], test_create_with_big_body: [{ - threshold: "avg<6", + threshold: "avg<10", abortOnFail: true, delayAbortEval: '10s', }], test_normal_route: [{ - threshold: "avg<6", + threshold: "avg<10", abortOnFail: true, delayAbortEval: '10s', }], test_id_route: [{ - threshold: "avg<6", + threshold: "avg<10", abortOnFail: true, delayAbortEval: '10s', }], test_open_file: [{ - threshold: "avg<6", + threshold: "avg<10", abortOnFail: true, delayAbortEval: '10s', }], test_execute_shell: [{ - threshold: "avg<10", + threshold: "avg<15", abortOnFail: true, delayAbortEval: '10s', }], diff --git a/benchmarks/wrk_benchmark/flask_mysql.py b/benchmarks/wrk_benchmark/flask_mysql.py index d58aebf3e..2fea48d85 100644 --- a/benchmarks/wrk_benchmark/flask_mysql.py +++ b/benchmarks/wrk_benchmark/flask_mysql.py @@ -6,5 +6,5 @@ "http://localhost:8086/create", "http://localhost:8087/create", "", - percentage_limit=40 + percentage_limit=50 ) diff --git a/benchmarks/wrk_benchmark/flask_mysql_uwsgi.py b/benchmarks/wrk_benchmark/flask_mysql_uwsgi.py index e728b7596..c276e326c 100644 --- a/benchmarks/wrk_benchmark/flask_mysql_uwsgi.py +++ b/benchmarks/wrk_benchmark/flask_mysql_uwsgi.py @@ -6,7 +6,7 @@ "http://localhost:8088/benchmark", "http://localhost:8089/benchmark", "a non empty route which makes a simulated request to a database", - percentage_limit=40 + percentage_limit=50 ) @@ -14,5 +14,5 @@ "http://localhost:8088/benchmark_io", "http://localhost:8089/benchmark_io", "a route that makes multiple I/O calls", - percentage_limit=35 + percentage_limit=45 ) diff --git a/benchmarks/wrk_benchmark/starlette_postgres_uvicorn.py b/benchmarks/wrk_benchmark/starlette_postgres_uvicorn.py index 9b3c347a7..798d7b142 100644 --- a/benchmarks/wrk_benchmark/starlette_postgres_uvicorn.py +++ b/benchmarks/wrk_benchmark/starlette_postgres_uvicorn.py @@ -6,5 +6,5 @@ "http://localhost:8102/benchmark", "http://localhost:8103/benchmark", "a non empty route which makes a simulated request to a database", - percentage_limit=40 + percentage_limit=50 ) diff --git a/sample-apps/common.mk b/sample-apps/common.mk index 389950bb2..06322f92b 100644 --- a/sample-apps/common.mk +++ b/sample-apps/common.mk @@ -32,9 +32,7 @@ AIKIDO_ENV_DISABLED = \ # Common target definitions .PHONY: install install: - # ignore poetry install failures, these can be due to locking issues when - # 2 poetry installs are run at the same time. - poetry install || true; + poetry install .PHONY: health-check health-check: diff --git a/sample-apps/flask-mysql-uwsgi/Makefile b/sample-apps/flask-mysql-uwsgi/Makefile index c89280b27..2a0561dd0 100644 --- a/sample-apps/flask-mysql-uwsgi/Makefile +++ b/sample-apps/flask-mysql-uwsgi/Makefile @@ -9,14 +9,16 @@ run: install $(AIKIDO_ENV_COMMON) \ poetry run uwsgi --ini uwsgi.ini -.PHONY: runBenchmark -runBenchmark: install +.PHONY: runBenchmark startBenchmark +runBenchmark: install startBenchmark +startBenchmark: @echo "Running sample app flask-mysql-uwsgi with Zen on port $(PORT)" $(AIKIDO_ENV_BENCHMARK) \ poetry run uwsgi --single-interpreter --ini uwsgi.ini -.PHONY: runZenDisabled -runZenDisabled: install +.PHONY: runZenDisabled startZenDisabled +runZenDisabled: install startZenDisabled +startZenDisabled: @echo "Running sample app flask-mysql-uwsgi without Zen on port $(PORT_DISABLED)" $(AIKIDO_ENV_DISABLED) \ poetry run uwsgi --single-interpreter --ini uwsgi2.ini diff --git a/sample-apps/flask-mysql/Makefile b/sample-apps/flask-mysql/Makefile index b68ae0c86..6ba524b86 100644 --- a/sample-apps/flask-mysql/Makefile +++ b/sample-apps/flask-mysql/Makefile @@ -9,14 +9,16 @@ run: install $(AIKIDO_ENV_COMMON) \ poetry run flask --app app.py run --host=0.0.0.0 --port=$(PORT) --no-reload -.PHONY: runBenchmark -runBenchmark: install +.PHONY: runBenchmark startBenchmark +runBenchmark: install startBenchmark +startBenchmark: @echo "Running sample app flask-mysql with Zen (benchmark mode) on port $(PORT)" $(AIKIDO_ENV_BENCHMARK) \ poetry run flask --app app.py run --host=0.0.0.0 --port=$(PORT) --no-reload -.PHONY: runZenDisabled -runZenDisabled: install +.PHONY: runZenDisabled startZenDisabled +runZenDisabled: install startZenDisabled +startZenDisabled: @echo "Running sample app flask-mysql without Zen on port $(PORT_DISABLED)" $(AIKIDO_ENV_DISABLED) \ poetry run flask --app app.py run --host=0.0.0.0 --port=$(PORT_DISABLED) --no-reload diff --git a/sample-apps/starlette-postgres-uvicorn/Makefile b/sample-apps/starlette-postgres-uvicorn/Makefile index f0f1354eb..d0cdfdbc9 100644 --- a/sample-apps/starlette-postgres-uvicorn/Makefile +++ b/sample-apps/starlette-postgres-uvicorn/Makefile @@ -8,12 +8,14 @@ run: install @echo "Running sample app starlette-postgres-uvicorn with Zen on port $(PORT)" $(AIKIDO_ENV_COMMON) poetry run uvicorn app:app --host 0.0.0.0 --port $(PORT) --workers 4 -.PHONY: runBenchmark -runBenchmark: install +.PHONY: runBenchmark startBenchmark +runBenchmark: install startBenchmark +startBenchmark: @echo "Running sample app starlette-postgres-uvicorn with Zen (benchmark mode) on port $(PORT)" $(AIKIDO_ENV_BENCHMARK) poetry run uvicorn app:app --host 0.0.0.0 --port $(PORT) --workers 4 -.PHONY: runZenDisabled -runZenDisabled: install +.PHONY: runZenDisabled startZenDisabled +runZenDisabled: install startZenDisabled +startZenDisabled: @echo "Running sample app starlette-postgres-uvicorn without Zen on port $(PORT_DISABLED)" $(AIKIDO_ENV_DISABLED) poetry run uvicorn app:app --host 0.0.0.0 --port $(PORT_DISABLED) --workers 4