diff --git a/.github/workflows/load-test.yml b/.github/workflows/load-test.yml new file mode 100644 index 00000000..6f595891 --- /dev/null +++ b/.github/workflows/load-test.yml @@ -0,0 +1,151 @@ +# Manual-trigger workflow to run Gatling load tests against overpass.deflock.org. +# +# Trigger from the Actions tab → "Load Test" → "Run workflow" → pick a scenario. +# The HTML report is uploaded as a downloadable artifact (retained 30 days). +# +# When "all" is selected, all 4 simulations run in parallel on separate runners, +# hitting the server simultaneously for distributed load. A summary job then +# collects all reports and posts a PR comment with download links. + +name: Load Test + +on: + workflow_dispatch: + inputs: + scenario: + description: 'Test scenario to run' + required: true + default: 'baseline' + type: choice + options: + - baseline + - concurrent + - stress + - burst + - all + +concurrency: + group: load-test + cancel-in-progress: true + +jobs: + # Build the matrix dynamically based on the selected scenario. + # This avoids the `matrix` context parsing issue in job-level `if`. + resolve-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: | + if [ "${{ inputs.scenario }}" = "all" ]; then + echo 'matrix={"include":[{"name":"baseline","class":"deflock.OverpassSimulation"},{"name":"concurrent","class":"deflock.ConcurrentSimulation"},{"name":"stress","class":"deflock.StressSimulation"},{"name":"burst","class":"deflock.BurstSimulation"}]}' >> "$GITHUB_OUTPUT" + elif [ "${{ inputs.scenario }}" = "baseline" ]; then + echo 'matrix={"include":[{"name":"baseline","class":"deflock.OverpassSimulation"}]}' >> "$GITHUB_OUTPUT" + elif [ "${{ inputs.scenario }}" = "concurrent" ]; then + echo 'matrix={"include":[{"name":"concurrent","class":"deflock.ConcurrentSimulation"}]}' >> "$GITHUB_OUTPUT" + elif [ "${{ inputs.scenario }}" = "stress" ]; then + echo 'matrix={"include":[{"name":"stress","class":"deflock.StressSimulation"}]}' >> "$GITHUB_OUTPUT" + elif [ "${{ inputs.scenario }}" = "burst" ]; then + echo 'matrix={"include":[{"name":"burst","class":"deflock.BurstSimulation"}]}' >> "$GITHUB_OUTPUT" + fi + + load-test: + needs: resolve-matrix + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.resolve-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v5 + + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + + # Caches Gradle wrapper and dependencies between runs + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Run Gatling simulation (${{ matrix.name }}) + working-directory: load-tests + run: ./gradlew gatlingRun --simulation ${{ matrix.class }} --non-interactive + + - name: Upload Gatling report + if: always() + uses: actions/upload-artifact@v4 + with: + name: gatling-report-${{ matrix.name }} + path: load-tests/build/reports/gatling/ + retention-days: 30 + + # Post a summary comment on the PR (if triggered from a PR branch) with + # links to download each report artifact. + summary: + needs: load-test + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Find associated PR + id: find-pr + run: | + # Search for a PR with this branch as the head ref. Check the + # current repo first, then the parent (upstream) repo if this is a fork. + PR=$(gh pr list --repo "${{ github.repository }}" --head "${{ github.ref_name }}" --json number --jq '.[0].number' 2>/dev/null || echo "") + if [ -z "$PR" ]; then + PARENT=$(gh api "repos/${{ github.repository }}" --jq '.parent.full_name // empty' 2>/dev/null || echo "") + if [ -n "$PARENT" ]; then + PR=$(gh pr list --repo "$PARENT" --head "${{ github.ref_name }}" --json number --jq '.[0].number' 2>/dev/null || echo "") + if [ -n "$PR" ]; then + echo "pr_repo=$PARENT" >> "$GITHUB_OUTPUT" + fi + fi + else + echo "pr_repo=${{ github.repository }}" >> "$GITHUB_OUTPUT" + fi + echo "pr_number=$PR" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Post PR comment with report links + if: steps.find-pr.outputs.pr_number != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + SCENARIO="${{ inputs.scenario }}" + RUN_ID="${{ github.run_id }}" + + # Build artifact links from the matrix + ARTIFACTS="" + if [ "$SCENARIO" = "all" ] || [ "$SCENARIO" = "baseline" ]; then + ARTIFACTS="$ARTIFACTS\n| Baseline | [Download]($RUN_URL/artifacts) |" + fi + if [ "$SCENARIO" = "all" ] || [ "$SCENARIO" = "concurrent" ]; then + ARTIFACTS="$ARTIFACTS\n| Concurrent | [Download]($RUN_URL/artifacts) |" + fi + if [ "$SCENARIO" = "all" ] || [ "$SCENARIO" = "stress" ]; then + ARTIFACTS="$ARTIFACTS\n| Stress | [Download]($RUN_URL/artifacts) |" + fi + if [ "$SCENARIO" = "all" ] || [ "$SCENARIO" = "burst" ]; then + ARTIFACTS="$ARTIFACTS\n| Burst | [Download]($RUN_URL/artifacts) |" + fi + + BODY=$(cat < Download the artifact ZIP → extract → open \`index.html\` for interactive charts. + EOF + ) + + gh pr comment "${{ steps.find-pr.outputs.pr_number }}" --repo "${{ steps.find-pr.outputs.pr_repo }}" --body "$BODY" diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3b894d35..1656dcca 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -26,6 +26,46 @@ jobs: - name: Test run: flutter test + load-test-compile: + name: Load Test Compile & Unit Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 2 + + - name: Check for load-test changes + id: changes + run: | + if git diff --name-only HEAD~1 HEAD | grep -q '^load-tests/'; then + echo "changed=true" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event_name }}" = "pull_request" ]; then + BASE=${{ github.event.pull_request.base.sha }} + if git diff --name-only "$BASE" HEAD | grep -q '^load-tests/'; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Set up JDK 21 + if: steps.changes.outputs.changed == 'true' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + + - name: Setup Gradle + if: steps.changes.outputs.changed == 'true' + uses: gradle/actions/setup-gradle@v4 + + - name: Compile & test + if: steps.changes.outputs.changed == 'true' + working-directory: load-tests + run: ./gradlew compileGatlingScala test + build-debug-apk: name: Build Debug APK runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index db2e32aa..5417a608 100644 --- a/.gitignore +++ b/.gitignore @@ -103,4 +103,4 @@ build_keys.conf linux/ macos/ web/ -windows/ \ No newline at end of file +windows/ diff --git a/load-tests/.devcontainer/Dockerfile b/load-tests/.devcontainer/Dockerfile new file mode 100644 index 00000000..a7ca5004 --- /dev/null +++ b/load-tests/.devcontainer/Dockerfile @@ -0,0 +1,24 @@ +# Dev container image for running Gatling load tests. +# +# Based on Microsoft's Java 21 dev container, which includes: +# - JDK 21 (Eclipse Temurin) +# - Gradle (via the wrapper in the project) +# - Standard dev tools (git, curl, etc.) +# +# We add Coursier (the Scala package manager) for Scala tooling support +# in VS Code via the Metals extension. + +FROM mcr.microsoft.com/devcontainers/java:21 + +# Install Coursier for Scala tooling (used by the Metals VS Code extension). +# Note: the Scala compiler itself is managed by Gradle via the Gatling plugin, +# so we only need Coursier for IDE support. +# +# Pinned to a specific version with checksum verification to prevent +# supply chain attacks via mutable "latest" release URLs. +ARG COURSIER_VERSION=v2.1.24 +ARG COURSIER_SHA256=1517a0b6c4b9608dc45da8f34bc8290707ed50104ee92662f57808d2c012be54 +RUN curl -fL "https://github.com/coursier/coursier/releases/download/${COURSIER_VERSION}/cs-x86_64-pc-linux.gz" \ + | gzip -d > /usr/local/bin/cs \ + && echo "${COURSIER_SHA256} /usr/local/bin/cs" | sha256sum -c - \ + && chmod +x /usr/local/bin/cs diff --git a/load-tests/.devcontainer/devcontainer.json b/load-tests/.devcontainer/devcontainer.json new file mode 100644 index 00000000..f9f5d2f4 --- /dev/null +++ b/load-tests/.devcontainer/devcontainer.json @@ -0,0 +1,28 @@ +// Dev container for the Gatling load test project. +// +// This provides a ready-to-go JVM + Scala environment without needing +// to install anything locally. Open the load-tests/ folder in VS Code +// and select "Dev Containers: Reopen in Container". +// +// Includes: +// - JDK 21 (Temurin) +// - Scala via Coursier +// - VS Code extensions: Scala Metals (IDE support) + Gradle (build tasks) +{ + "name": "Deflock Load Tests", + "build": { + "dockerfile": "Dockerfile" + }, + "workspaceFolder": "/workspaces/deflock-app/load-tests", + "customizations": { + "vscode": { + "extensions": [ + "scalameta.metals", + "vscjava.vscode-gradle" + ] + } + }, + // Pre-download all Gradle + Gatling dependencies so the first + // `./gradlew gatlingRun` is fast. + "postCreateCommand": "./gradlew dependencies" +} diff --git a/load-tests/.gitignore b/load-tests/.gitignore new file mode 100644 index 00000000..686141c9 --- /dev/null +++ b/load-tests/.gitignore @@ -0,0 +1,5 @@ +# Gradle build outputs (includes Gatling HTML reports in build/reports/gatling/) +build/ + +# Gradle cache +.gradle/ diff --git a/load-tests/CLAUDE.md b/load-tests/CLAUDE.md new file mode 100644 index 00000000..e24e293e --- /dev/null +++ b/load-tests/CLAUDE.md @@ -0,0 +1,64 @@ +# Load Tests + +Gatling load tests for `overpass.deflock.org`. Scala 2.13, Gradle build, JDK 21. + +## Architecture + +Three source sets to avoid circular dependencies: + +- **`src/shared/`** — Pure Scala logic (no Gatling dependency). `OverpassQuery` (query builder, tag filters, timeouts) and `TestData` (cities, viewports, feeders). +- **`src/gatling/`** — Gatling simulations. Depends on `shared`. Contains `OverpassRequests` (HTTP request def) and four simulations. +- **`src/test/`** — ScalaTest unit tests. Depends on `shared`. Tests pure logic without Gatling. + +The `shared` source set exists because the Gatling Gradle plugin creates a circular dependency if `test` depends on `gatling` output directly. + +## Key files + +| File | Purpose | +|---|---| +| `src/shared/scala/deflock/OverpassQuery.scala` | Query builder, tag filters (must match app), timeouts | +| `src/shared/scala/deflock/TestData.scala` | Cities, viewports, feeders (`randomFeeder`, `weightedZoomFeeder`) | +| `src/gatling/scala/deflock/OverpassRequests.scala` | Gatling HTTP request definition, `feederForZoom` | +| `src/gatling/scala/deflock/OverpassSimulation.scala` | Baseline: 1 user, all cities x all zooms, deterministic | +| `src/gatling/scala/deflock/ConcurrentSimulation.scala` | Ramp to 50 users, find degradation point | +| `src/gatling/scala/deflock/StressSimulation.scala` | Spike to 500 users, exceed server capacity | +| `src/gatling/scala/deflock/BurstSimulation.scala` | Realistic app sessions in waves | +| `build.gradle.kts` | Gradle config with `shared` source set, ScalaTest deps | + +## Commands + +```bash +./gradlew gatlingRun # baseline +./gradlew gatlingRun --simulation deflock.ConcurrentSimulation # concurrent +./gradlew gatlingRun --simulation deflock.StressSimulation # stress +./gradlew gatlingRun --simulation deflock.BurstSimulation # burst +./gradlew test # unit tests +./gradlew compileGatlingScala # compile check +``` + +Do NOT use `gatlingRun-deflock.ClassName` syntax — it doesn't work with the Gatling Gradle plugin v3.15.0. Use `--simulation` flag instead. + +## Tag filter parity + +`OverpassQuery.tagFilters` must exactly match the app's `NodeProfile.getDefaults()` in `lib/models/node_profile.dart`. There are 11 built-in profiles. Empty tag values (e.g., `camera:mount: ''`) are filtered out, matching what `OverpassService._buildQuery()` does in `lib/services/overpass_service.dart`. + +When profiles change in the app, update `tagFilters` to match. + +## Conventions + +- Baseline simulation must be deterministic (no randomization) for reproducible results. +- Concurrent/stress/burst simulations use randomized feeders for realistic traffic. +- `ThreadLocalRandom` (not `scala.util.Random`) for feeders used by concurrent simulations. +- Gatling session keys are constants in `OverpassQuery` (`CityName`, `ZoomLevel`, `QueryBody`). +- User-Agent headers identify load test traffic: `DeFlock/LoadTest-{Scenario}`. +- Client timeout = server timeout + 5s so we receive server-side timeout responses. + +## GitHub Actions + +The `Load Test` workflow (`.github/workflows/load-test.yml`) has a scenario picker dropdown. Reports are uploaded as artifacts. A summary job posts a comment on the PR with download links. + +Trigger via Actions tab or API: +```bash +gh api repos/{owner}/{repo}/actions/workflows/{id}/dispatches \ + -f ref=feat/load-tests -f 'inputs[scenario]=baseline' +``` diff --git a/load-tests/README.md b/load-tests/README.md new file mode 100644 index 00000000..9dfd8700 --- /dev/null +++ b/load-tests/README.md @@ -0,0 +1,177 @@ +# Deflock Load Tests + +Gatling load tests for validating [`overpass.deflock.org`](https://overpass.deflock.org) performance before rolling it out as the primary Overpass API endpoint for all Deflock app users. + +## What is this? + +The Deflock app fetches surveillance camera data from the [Overpass API](https://wiki.openstreetmap.org/wiki/Overpass_API) every time a user pans or zooms the map. We've deployed our own Overpass instance at `overpass.deflock.org` to reduce dependence on the public endpoint. These load tests validate that our instance can handle realistic traffic patterns before we switch users over to it. + +The tests use [Gatling](https://gatling.io), an open-source load testing framework. Gatling simulates virtual users sending HTTP requests and produces detailed HTML reports with latency percentiles, error rates, and throughput metrics. + +## Quick start + +### Prerequisites + +- **JDK 21+** — install via [SDKMAN](https://sdkman.io) (`sdk install java 21-tem`) or your package manager +- **No other tools needed** — Gradle and Scala are handled automatically by the included wrapper and build config + +Or use the included [dev container](#dev-container) to skip local setup entirely. + +### Run the tests + +```bash +cd load-tests +./gradlew gatlingRun # baseline (default) +./gradlew gatlingRun --simulation deflock.ConcurrentSimulation # concurrent users +./gradlew gatlingRun --simulation deflock.StressSimulation # stress test +./gradlew gatlingRun --simulation deflock.BurstSimulation # burst traffic +``` + +When finished, Gatling prints a `file://` URL to the HTML report — open it in your browser. Reports land in `build/reports/gatling//index.html`. + +### Run via GitHub Actions + +1. Go to the **Actions** tab in GitHub +2. Select the **"Load Test"** workflow +3. Click **"Run workflow"** and pick a scenario from the dropdown: + - **baseline** — single-user zoom progression (~2 minutes) + - **concurrent** — ramp to 50 users over 4 minutes + - **stress** — spike to 500 users over 5 minutes + - **burst** — realistic wave pattern over ~3 minutes + - **all** — run all 4 in parallel on separate runners +4. When complete, download the **gatling-report-{name}** artifact (retained for 30 days) + +## Scenarios + +### Baseline (`OverpassSimulation`) + +A single virtual user walks through zoom levels z15→z10, querying every city at each level (6 zooms × 6 cities = 36 deterministic requests). Measures how response time scales with viewport size. + +- **Duration:** ~2 minutes +- **Assertions:** p99 < 30s, errors < 5% + +### Concurrent (`ConcurrentSimulation`) + +Ramps from 1 to 50 users over 2 minutes. Each user loops forever with random city/zoom picks and 500ms pauses until the 4-minute max duration. Designed to find the degradation inflection point — where does p95 start climbing? + +- **Duration:** 4 minutes (max) +- **Assertions:** p95 < 45s, errors < 20% +- **Look for:** inflection point in "Response time percentiles over time" chart + +### Stress (`StressSimulation`) + +Three phases totaling 500 users: warmup ramp (100 over 30s), spike (200 at once), sustained ramp (200 over 30s). Uses 100ms pauses and shared connections. Designed to exceed the server's ~512 Overpass compute slots. + +- **Duration:** 5 minutes (max) +- **Assertions:** none meaningful (just `requestsPerSec > 0`) +- **Expected failures:** 429s (slot exhaustion), 502/503s (nginx), timeouts +- **Look for:** when errors start, throughput plateau/collapse, bimodal latency + +### Burst (`BurstSimulation`) + +Models real app usage: each user does 10-20 requests (a map browsing session) then exits. Users arrive in waves (20→50→100→80) with gaps between waves. Uses weighted zoom feeder (80% z13-z15) matching real user behavior. + +- **Duration:** ~3 minutes +- **Assertions:** p95 < 30s, errors < 10% +- **Look for:** wave pattern in active users chart, recovery between bursts + +### Test data + +Six US cities were chosen for high surveillance camera density in their downtown areas: + +| City | Center coordinates | Landmark | +|---|---|---| +| Denver | 39.75, -105.00 | 16th St Mall / Union Station | +| Los Angeles | 34.05, -118.25 | Pershing Square, DTLA | +| San Francisco | 37.79, -122.40 | Financial District | +| New York | 40.75, -73.98 | Midtown / 42nd & 6th Ave | +| Boston | 42.36, -71.06 | Downtown Crossing | +| Chicago | 41.88, -87.63 | State & Madison, The Loop | + +### Zoom levels and viewport sizes + +Each zoom level corresponds to a different viewport size on a typical mobile phone screen (~400x800px, portrait): + +| Zoom | Area covered | Lat x Lng span | +|---|---|---| +| 15 | A few city blocks (~1.5 x 3 km) | 0.026 x 0.017 deg | +| 14 | A neighborhood (~3 x 6 km) | 0.053 x 0.034 deg | +| 13 | A district (~6 x 12 km) | 0.105 x 0.069 deg | +| 12 | A mid-size city (~12 x 23 km) | 0.210 x 0.140 deg | +| 11 | A large city (~23 x 47 km) | 0.420 x 0.270 deg | +| 10 | A metro region (~47 x 93 km) | 0.840 x 0.550 deg | + +## Interpreting the report + +The Gatling HTML report includes several views. Here's what to look for: + +### Key metrics + +- **p50 (median) latency** — what a typical user experiences +- **p95 latency** — should be under 10s for a good user experience +- **p99 latency** — should be under 30s (the assertion threshold) +- **Error rate** — should be 0% under single-user load + +### Report sections + +- **Response time distribution** — histogram showing how many requests fell into each latency bucket +- **Response time percentiles over time** — trend lines for p50/p75/p95/p99 throughout the test +- **Requests per second** — throughput over the test duration +- **Individual request details** — click any request name (e.g., "Overpass z15 - Denver") to see its specific metrics + +### What "good" looks like + +From our baseline runs, typical single-user performance is: + +| Zoom | Expected latency | +|---|---| +| z15 (blocks) | ~400-600ms | +| z13-z14 (neighborhood) | ~600-1000ms | +| z10-z11 (city/metro) | ~1000-1600ms | + +## Project structure + +``` +load-tests/ +├── .devcontainer/ # VS Code dev container (JDK 21 + Scala) +│ ├── devcontainer.json +│ └── Dockerfile +├── build.gradle.kts # Build config (Gatling + Scala + test plugins) +├── settings.gradle.kts # Gradle project name +├── gradlew / gradlew.bat # Gradle wrapper (no global install needed) +├── gradle/wrapper/ # Gradle wrapper jar + config +├── src/ +│ ├── shared/scala/deflock/ # Pure logic (no Gatling dependency) +│ │ ├── TestData.scala # City data, viewports, feeders, buildFeedEntry +│ │ └── OverpassQuery.scala# Query builder, timeouts, tag filters, constants +│ ├── gatling/ +│ │ ├── scala/deflock/ # Gatling simulations (depend on shared) +│ │ │ ├── OverpassSimulation.scala # Baseline: single-user zoom progression +│ │ │ ├── ConcurrentSimulation.scala # Ramp to 50 users +│ │ │ ├── StressSimulation.scala # Spike to 500 users +│ │ │ ├── BurstSimulation.scala # Realistic wave pattern +│ │ │ └── OverpassRequests.scala # Gatling HTTP request def + feederForZoom +│ │ └── resources/ +│ │ ├── gatling.conf # Gatling charting config +│ │ └── logback-test.xml +│ └── test/scala/deflock/ # Unit tests for shared logic (ScalaTest) +│ ├── TestDataSpec.scala +│ └── OverpassQuerySpec.scala +└── build/reports/gatling/ # Generated HTML reports (.gitignored) +``` + +The `shared` source set contains pure Scala logic with no Gatling dependency. Both the `gatling` source set (simulations) and the `test` source set (unit tests) depend on it. This avoids a circular dependency that would occur if tests depended directly on the Gatling source set. + +Run unit tests with `./gradlew test`. + +## Dev container + +If you don't want to install JDK locally, the included dev container provides a ready-to-go environment: + +1. Open the `load-tests/` folder in VS Code +2. Install the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) +3. Press `Ctrl+Shift+P` → "Dev Containers: Reopen in Container" +4. Wait for the container to build (first time takes a few minutes) +5. Open a terminal and run `./gradlew gatlingRun` + +The container includes JDK 21, Scala (via [Coursier](https://get-coursier.io)), and VS Code extensions for Scala (Metals) and Gradle. diff --git a/load-tests/build.gradle.kts b/load-tests/build.gradle.kts new file mode 100644 index 00000000..bfd4ea7a --- /dev/null +++ b/load-tests/build.gradle.kts @@ -0,0 +1,51 @@ +// Gatling load test build configuration. +// +// Gatling (https://gatling.io) is a load testing framework that simulates +// virtual users sending HTTP requests and produces HTML performance reports. +// +// The `scala` plugin compiles our Scala simulation files. +// The `io.gatling.gradle` plugin adds the `gatlingRun` task and manages +// Gatling + Scala library dependencies automatically. +// +// Run simulations: ./gradlew gatlingRun +// Run unit tests: ./gradlew test +// Reports: build/reports/gatling/ + +plugins { + scala + id("io.gatling.gradle") version "3.15.0" +} + +repositories { + mavenCentral() +} + +// --- Source sets --- +// "shared" contains pure Scala logic (TestData, OverpassQuery) with no Gatling +// dependency. Both the "gatling" source set (simulations) and "test" source set +// (unit tests) depend on it. This avoids the circular dependency that would +// occur if tests depended directly on the gatling source set (since the Gatling +// plugin makes gatling extend test). + +sourceSets { + create("shared") +} + +dependencies { + // shared source set needs Scala stdlib (provided transitively by Gatling, + // but shared compiles independently) + "sharedImplementation"("org.scala-lang:scala-library:2.13.16") + + // Gatling simulations use shared logic + "gatlingImplementation"(sourceSets["shared"].output) + + // Unit tests use shared logic + ScalaTest + testImplementation(sourceSets["shared"].output) + testImplementation("org.scalatest:scalatest_2.13:3.2.19") + testImplementation("org.scalatestplus:junit-5-11_2.13:3.2.19.0") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.11.4") +} + +tasks.withType { + useJUnitPlatform() +} diff --git a/load-tests/gradle/wrapper/gradle-wrapper.jar b/load-tests/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..d997cfc6 Binary files /dev/null and b/load-tests/gradle/wrapper/gradle-wrapper.jar differ diff --git a/load-tests/gradle/wrapper/gradle-wrapper.properties b/load-tests/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..dbc3ce4a --- /dev/null +++ b/load-tests/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.0-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/load-tests/gradlew b/load-tests/gradlew new file mode 100755 index 00000000..d06d3135 --- /dev/null +++ b/load-tests/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/b5fe9efed6cae7b9f2fbdb2d380fb69af16bb752/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='-Dfile.encoding=UTF-8 "-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/load-tests/gradlew.bat b/load-tests/gradlew.bat new file mode 100644 index 00000000..bd8a8c05 --- /dev/null +++ b/load-tests/gradlew.bat @@ -0,0 +1,93 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS=-Dfile.encoding=UTF-8 "-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/load-tests/settings.gradle.kts b/load-tests/settings.gradle.kts new file mode 100644 index 00000000..7bb784cb --- /dev/null +++ b/load-tests/settings.gradle.kts @@ -0,0 +1,2 @@ +// Gradle project name — only affects build output naming, not functionality. +rootProject.name = "deflock-load-tests" diff --git a/load-tests/src/gatling/resources/gatling.conf b/load-tests/src/gatling/resources/gatling.conf new file mode 100644 index 00000000..a35bec22 --- /dev/null +++ b/load-tests/src/gatling/resources/gatling.conf @@ -0,0 +1,19 @@ +// Gatling configuration. +// See https://docs.gatling.io/reference/script/core/configuration/ for all options. +// +// The Gradle plugin handles most settings (like report output directory). +// This file only overrides charting thresholds for the HTML report. + +gatling { + charting { + // These thresholds define the color bands in the response time + // distribution chart in the HTML report: + // Green: < lowerBound (under 1 second — great) + // Yellow: lowerBound to higherBound (1-5 seconds — acceptable) + // Red: > higherBound (over 5 seconds — needs attention) + indicators { + lowerBound = 1000 + higherBound = 5000 + } + } +} diff --git a/load-tests/src/gatling/resources/logback-test.xml b/load-tests/src/gatling/resources/logback-test.xml new file mode 100644 index 00000000..4eade1fe --- /dev/null +++ b/load-tests/src/gatling/resources/logback-test.xml @@ -0,0 +1,23 @@ + + + + + + %d{HH:mm:ss.SSS} [%-5level] %logger{36} - %msg%n + + + + + + + diff --git a/load-tests/src/gatling/scala/deflock/BurstSimulation.scala b/load-tests/src/gatling/scala/deflock/BurstSimulation.scala new file mode 100644 index 00000000..30b7b577 --- /dev/null +++ b/load-tests/src/gatling/scala/deflock/BurstSimulation.scala @@ -0,0 +1,75 @@ +package deflock + +import io.gatling.core.Predef._ +import io.gatling.http.Predef._ + +import scala.concurrent.duration._ +import java.util.concurrent.ThreadLocalRandom + +/** + * Realistic burst simulation modeling actual Deflock app usage patterns. + * + * Each virtual user represents one app session: the user opens the app, + * pans/zooms the map 10-20 times (each interaction triggers an Overpass + * query), then closes the app. Pauses between requests are 200-800ms, + * matching the time it takes to pan/zoom on a phone. + * + * Users arrive in waves, simulating the bursty nature of real traffic + * (e.g., morning commutes, lunch breaks): + * Wave 1: 20 users over 15s (light morning traffic) + * Gap: 10s + * Wave 2: 50 users over 15s (mid-morning pickup) + * Gap: 10s + * Wave 3: 100 users over 20s (lunch rush) + * Gap: 10s + * Wave 4: 80 users over 30s (sustained afternoon) + * + * Uses the weighted zoom feeder (z13-z15 heavy) since most real users + * are zoomed in to neighborhood level. + * + * == What to look for in the report == + * + * - Response time should stay relatively stable during waves 1-2, then + * may degrade during wave 3 (the peak). + * - The "Active users over time" chart should show the wave pattern. + * - Compare error rates between waves to see if the server recovers + * between bursts. + * + * == Running == + * {{{ + * cd load-tests + * ./gradlew gatlingRun --simulation deflock.BurstSimulation + * }}} + */ +class BurstSimulation extends Simulation { + + val httpProtocol = http + .baseUrl("https://overpass.deflock.org") + .userAgentHeader("DeFlock/LoadTest-Burst (+https://deflock.org)") + .acceptHeader("application/json") + + // Each user does 10-20 requests (a realistic app session), then exits. + val burstScenario = scenario("Burst app sessions") + .repeat(session => 10 + ThreadLocalRandom.current().nextInt(11)) { + feed(TestData.weightedZoomFeeder) + .exec(OverpassRequests.overpassRequest) + .pause(200.milliseconds, 800.milliseconds) + } + + setUp( + burstScenario.inject( + rampUsers(20).during(15.seconds), // Wave 1: light traffic + nothingFor(10.seconds), + rampUsers(50).during(15.seconds), // Wave 2: mid-morning + nothingFor(10.seconds), + rampUsers(100).during(20.seconds), // Wave 3: lunch rush + nothingFor(10.seconds), + rampUsers(80).during(30.seconds) // Wave 4: sustained afternoon + ) + ).protocols(httpProtocol) + .maxDuration(10.minutes) + .assertions( + global.responseTime.percentile(95).lt(30000), + global.failedRequests.percent.lt(10.0) + ) +} diff --git a/load-tests/src/gatling/scala/deflock/ConcurrentSimulation.scala b/load-tests/src/gatling/scala/deflock/ConcurrentSimulation.scala new file mode 100644 index 00000000..dcc0c1dc --- /dev/null +++ b/load-tests/src/gatling/scala/deflock/ConcurrentSimulation.scala @@ -0,0 +1,56 @@ +package deflock + +import io.gatling.core.Predef._ +import io.gatling.http.Predef._ + +import scala.concurrent.duration._ + +/** + * Concurrent user simulation to find the degradation point. + * + * Ramps from 1 to 50 virtual users over 2 minutes. Each user loops + * forever (pick a random city/zoom, send the query, pause 500ms, repeat) + * until maxDuration (4 minutes total) is reached. + * + * The server has ~512 Overpass compute slots behind nginx. This simulation + * should reveal where latency starts climbing (slot contention) without + * pushing into outright failure territory. + * + * == What to look for in the report == + * + * - The "Response time percentiles over time" chart should show a clear + * inflection point where p95 starts climbing — that's the concurrency + * level where the server begins queuing requests. + * - The "Active users over time" chart maps directly to user count. + * - Compare p50 at 10 users vs 40 users to see degradation magnitude. + * + * == Running == + * {{{ + * cd load-tests + * ./gradlew gatlingRun --simulation deflock.ConcurrentSimulation + * }}} + */ +class ConcurrentSimulation extends Simulation { + + val httpProtocol = http + .baseUrl("https://overpass.deflock.org") + .userAgentHeader("DeFlock/LoadTest-Concurrent (+https://deflock.org)") + .acceptHeader("application/json") + + val concurrentScenario = scenario("Concurrent users") + .forever { + feed(TestData.randomFeeder) + .exec(OverpassRequests.overpassRequest) + .pause(500.milliseconds) + } + + setUp( + concurrentScenario.inject(rampUsers(50).during(2.minutes)) + ).protocols(httpProtocol) + .maxDuration(4.minutes) + .assertions( + // Lenient thresholds — we're exploring capacity, not enforcing SLA + global.responseTime.percentile(95).lt(45000), + global.failedRequests.percent.lt(20.0) + ) +} diff --git a/load-tests/src/gatling/scala/deflock/OverpassRequests.scala b/load-tests/src/gatling/scala/deflock/OverpassRequests.scala new file mode 100644 index 00000000..a2b0ec2c --- /dev/null +++ b/load-tests/src/gatling/scala/deflock/OverpassRequests.scala @@ -0,0 +1,54 @@ +package deflock + +import io.gatling.core.Predef._ +import io.gatling.http.Predef._ + +/** + * Gatling-specific Overpass API request definitions and feeders. + * + * Pure logic (query building, constants, timeouts) lives in OverpassQuery + * (shared source set). This object adds the Gatling HTTP DSL wrappers and + * the feederForZoom method that uses Gatling's .random extension. + * + * The request format mirrors the Deflock app (POST to /api/interpreter + * with form-encoded Overpass QL) using the same per-profile tag filters. + * See OverpassQuery.tagFilters for details. + */ +object OverpassRequests { + + /** + * Create a Gatling feeder that picks a random city for a given zoom level. + * + * A "feeder" in Gatling is a data source that injects variables into the + * virtual user's session before each request. This one pre-computes the + * Overpass query body so it's built once at startup, not on every request. + * + * @param viewport The zoom level and its corresponding viewport dimensions + * @return A Gatling feeder that randomly selects a city and provides session + * variables: cityName, zoomLevel, and the pre-built queryBody + */ + def feederForZoom(viewport: ZoomViewport) = + TestData.cities.map(TestData.buildFeedEntry(_, viewport)).toIndexedSeq.random + + /** + * The HTTP request definition that Gatling will execute. + * + * Uses Gatling's #{...} Expression Language syntax to inject session + * variables at request time. These variables are populated by the feeders + * in TestData — see feederForZoom() and the infinite feeders. + * + * The request name (e.g., "Overpass z15 - Denver") appears in the Gatling + * HTML report, making it easy to compare performance across zoom levels + * and cities. + * + * Checks: + * - HTTP 200 status (Overpass returns 200 even for empty results) + * - Response body contains an "elements" array (valid Overpass JSON) + */ + val overpassRequest = http("Overpass z#{zoomLevel} - #{cityName}") + .post("/api/interpreter") + .formParam("data", "#{queryBody}") + .requestTimeout(OverpassQuery.clientTimeout) + .check(status.is(200)) + .check(jsonPath("$.elements").exists) +} diff --git a/load-tests/src/gatling/scala/deflock/OverpassSimulation.scala b/load-tests/src/gatling/scala/deflock/OverpassSimulation.scala new file mode 100644 index 00000000..851837ee --- /dev/null +++ b/load-tests/src/gatling/scala/deflock/OverpassSimulation.scala @@ -0,0 +1,75 @@ +package deflock + +import io.gatling.core.Predef._ +import io.gatling.http.Predef._ + +import scala.concurrent.duration._ + +/** + * Gatling simulation for load-testing the Deflock Overpass API endpoint. + * + * This simulation validates the performance of overpass.deflock.org before + * it becomes the primary endpoint for all Deflock app users. It sends + * Overpass queries using the same per-profile tag filters as the app + * (see OverpassQuery.tagFilters and lib/models/node_profile.dart). + * + * == How it works == + * + * A single virtual user walks through zoom levels from tightest (z15, a few + * city blocks) to widest (z10, a metro region). At each zoom level, it + * queries every city deterministically — no randomization — so results are + * reproducible and city-to-city variance is visible in the report. + * + * This progression reveals how response time scales with viewport size — + * larger viewports return more surveillance nodes, producing bigger responses. + * + * == Running == + * + * {{{ + * cd load-tests + * ./gradlew gatlingRun + * }}} + * + * The HTML report will be in build/reports/gatling/ — open index.html. + * + */ +class OverpassSimulation extends Simulation { + + // Target our self-hosted Overpass instance (not the public OSMF one). + // The User-Agent identifies load test traffic in server logs. + val httpProtocol = http + .baseUrl("https://overpass.deflock.org") + .userAgentHeader("DeFlock/LoadTest (+https://deflock.org)") + .acceptHeader("application/json") + + // Walk through zoom levels from tightest (z15) to widest (z10). + // At each zoom level, query every city deterministically so results + // are reproducible and per-city variance is visible in the report. + // + // The nested flatMap produces a chain of (zoom × city) steps. For + // 6 zoom levels × 6 cities = 36 total requests with 500ms pauses. + val baselineScenario = scenario("Single-user zoom progression") + .exec( + TestData.zoomViewports.flatMap { viewport => + TestData.cities.map { city => + exec(_.setAll(TestData.buildFeedEntry(city, viewport))) + .exec(OverpassRequests.overpassRequest) + .pause(500.milliseconds) + } + }.reduce(_.exec(_)) + ) + + // --- Test setup --- + // atOnceUsers(1): inject exactly 1 virtual user immediately (no ramp-up). + // This is a baseline test — we want clean, isolated measurements before + // adding concurrency in future scenarios. + setUp( + baselineScenario.inject(atOnceUsers(1)) + ).protocols(httpProtocol) + .assertions( + // p99 response time under 30 seconds (generous for Overpass) + global.responseTime.percentile(99).lt(30000), + // Less than 5% of requests should fail + global.failedRequests.percent.lt(5.0) + ) +} diff --git a/load-tests/src/gatling/scala/deflock/StressSimulation.scala b/load-tests/src/gatling/scala/deflock/StressSimulation.scala new file mode 100644 index 00000000..43db2560 --- /dev/null +++ b/load-tests/src/gatling/scala/deflock/StressSimulation.scala @@ -0,0 +1,72 @@ +package deflock + +import io.gatling.core.Predef._ +import io.gatling.http.Predef._ + +import scala.concurrent.duration._ + +/** + * Stress simulation designed to exceed the server's 512 Overpass compute slots. + * + * Three phases totaling 500 virtual users: + * 1. Warmup: ramp to 100 users over 30s + * 2. Spike: inject 200 users at once + * 3. Sustained: ramp another 200 users over 30s + * + * Each user fires requests with only 100ms pauses — deliberately aggressive + * to saturate nginx connections and exhaust Overpass slots. + * + * `shareConnections` is enabled so that virtual users share the HTTP + * connection pool. Without this, 500 users would try to open 500 separate + * TCP connections to the server, and the test would fail on the client side + * (socket exhaustion) before stressing the server. + * + * == Expected failure modes (in order) == + * + * 1. Overpass slot exhaustion → 429 or "rate_limited" responses + * 2. nginx max connections → 502/503 gateway errors + * 3. Disk I/O saturation → timeouts on wide-zoom queries + * + * == What to look for in the report == + * + * - Error rate timeline: when do errors start, and what type? + * - Response time distribution: is there a bimodal pattern (fast cache hits + * vs slow/failed queries)? + * - Requests/sec: does throughput plateau or collapse? + * + * == Running == + * {{{ + * cd load-tests + * ./gradlew gatlingRun --simulation deflock.StressSimulation + * }}} + */ +class StressSimulation extends Simulation { + + val httpProtocol = http + .baseUrl("https://overpass.deflock.org") + .userAgentHeader("DeFlock/LoadTest-Stress (+https://deflock.org)") + .acceptHeader("application/json") + .shareConnections + + val stressScenario = scenario("Stress test") + .forever { + feed(TestData.randomFeeder) + .exec(OverpassRequests.overpassRequest) + .pause(100.milliseconds) + } + + setUp( + stressScenario.inject( + rampUsers(100).during(30.seconds), // Phase 1: warmup + nothingFor(1.second), // Brief gap so phases are visible in report + atOnceUsers(200), // Phase 2: spike + rampUsers(200).during(30.seconds) // Phase 3: sustained pressure + ) + ).protocols(httpProtocol) + .maxDuration(5.minutes) + .assertions( + // No meaningful SLA — this test is designed to break things. + // Just verify the test itself ran and produced data. + global.requestsPerSec.gt(0.0) + ) +} diff --git a/load-tests/src/shared/scala/deflock/OverpassQuery.scala b/load-tests/src/shared/scala/deflock/OverpassQuery.scala new file mode 100644 index 00000000..52470e48 --- /dev/null +++ b/load-tests/src/shared/scala/deflock/OverpassQuery.scala @@ -0,0 +1,91 @@ +package deflock + +import scala.concurrent.duration._ + +/** + * Pure Overpass API query logic — no Gatling dependency. + * + * Contains the query builder, tag filters, timeouts, and feeder session + * key constants. The Gatling HTTP request definition that uses these lives + * in OverpassRequests (in the gatling source set). + */ +object OverpassQuery { + + // --- Timeouts --- + // The Overpass QL query tells the server to abort after this many seconds. + // This matches kOverpassQueryTimeout in the app (lib/dev_config.dart). + val serverTimeoutSeconds = 45 + + // The HTTP client timeout is slightly longer than the server timeout so that + // we always receive the server's own timeout error response (a 200 with a + // "remark" field) rather than the client aborting the connection first. + val clientTimeout: FiniteDuration = (serverTimeoutSeconds + 5).seconds + + // --- Overpass tag filters --- + // One entry per built-in NodeProfile in the app (lib/models/node_profile.dart). + // Each string is the concatenated tag filters for that profile, with empty + // values (e.g., camera:mount='') filtered out — exactly as the app's + // OverpassService._buildQuery() does. Keep this list in sync with + // NodeProfile.getDefaults(). + val tagFilters: Seq[String] = Seq( + // generic-alpr + """["man_made"="surveillance"]["surveillance:type"="ALPR"]""", + // flock + """["man_made"="surveillance"]["surveillance"="public"]["surveillance:type"="ALPR"]["surveillance:zone"="traffic"]["camera:type"="fixed"]["manufacturer"="Flock Safety"]["manufacturer:wikidata"="Q108485435"]""", + // motorola + """["man_made"="surveillance"]["surveillance"="public"]["surveillance:type"="ALPR"]["surveillance:zone"="traffic"]["camera:type"="fixed"]["manufacturer"="Motorola Solutions"]["manufacturer:wikidata"="Q634815"]""", + // genetec + """["man_made"="surveillance"]["surveillance"="public"]["surveillance:type"="ALPR"]["surveillance:zone"="traffic"]["camera:type"="fixed"]["manufacturer"="Genetec"]["manufacturer:wikidata"="Q30295174"]""", + // leonardo + """["man_made"="surveillance"]["surveillance"="public"]["surveillance:type"="ALPR"]["surveillance:zone"="traffic"]["camera:type"="fixed"]["manufacturer"="Leonardo"]["manufacturer:wikidata"="Q910379"]""", + // neology + """["man_made"="surveillance"]["surveillance"="public"]["surveillance:type"="ALPR"]["surveillance:zone"="traffic"]["camera:type"="fixed"]["manufacturer"="Neology, Inc."]""", + // rekor + """["man_made"="surveillance"]["surveillance"="public"]["surveillance:type"="ALPR"]["surveillance:zone"="traffic"]["camera:type"="fixed"]["manufacturer"="Rekor"]""", + // axis + """["man_made"="surveillance"]["surveillance"="public"]["surveillance:type"="ALPR"]["surveillance:zone"="traffic"]["camera:type"="fixed"]["manufacturer"="Axis Communications"]["manufacturer:wikidata"="Q2347731"]""", + // generic-gunshot + """["man_made"="surveillance"]["surveillance:type"="gunshot_detector"]""", + // shotspotter + """["man_made"="surveillance"]["surveillance"="public"]["surveillance:type"="gunshot_detector"]["surveillance:brand"="ShotSpotter"]["surveillance:brand:wikidata"="Q107740188"]""", + // flock-raven + """["man_made"="surveillance"]["surveillance"="public"]["surveillance:type"="gunshot_detector"]["brand"="Flock Safety"]["brand:wikidata"="Q108485435"]""" + ) + + // --- Feeder session keys --- + // These constants are the variable names injected into each virtual user's + // session by the feeders in TestData. Using constants here (instead of raw + // strings) prevents typos that would silently break at runtime. + val CityName = "cityName" + val ZoomLevel = "zoomLevel" + val QueryBody = "queryBody" + + /** + * Build an Overpass QL query string for the given bounding box. + * + * The query structure mirrors the app's OverpassService._buildQuery(): + * union node clauses in a bbox, then fetch parent ways/relations. Uses + * the same per-profile tag filters as the app (see tagFilters above). + * + * Overpass bbox format is (south, west, north, east) — note this is + * different from many mapping libraries that use (west, south, east, north). + * + * @return A complete Overpass QL query string ready to POST + */ + def buildQuery(south: Double, west: Double, north: Double, east: Double): String = { + val nodeClauses = tagFilters.map { tags => + s" node$tags($south,$west,$north,$east);" + }.mkString("\n") + + s"""[out:json][timeout:$serverTimeoutSeconds]; + |( + |$nodeClauses + |); + |out body; + |( + | way(bn); + | rel(bn); + |); + |out skel;""".stripMargin + } +} diff --git a/load-tests/src/shared/scala/deflock/TestData.scala b/load-tests/src/shared/scala/deflock/TestData.scala new file mode 100644 index 00000000..9b63fa85 --- /dev/null +++ b/load-tests/src/shared/scala/deflock/TestData.scala @@ -0,0 +1,133 @@ +package deflock + +import java.util.concurrent.ThreadLocalRandom + +/** + * Center coordinates for a city's downtown area. + * + * These are the starting points for building map viewport bounding boxes. + * Each coordinate was verified against map data and chosen for its high + * density of surveillance infrastructure (cameras, ALPR, etc.), which + * produces realistic Overpass API response sizes. + * + * @param name Human-readable city name (appears in Gatling report labels) + * @param lat Latitude of the downtown center point + * @param lng Longitude of the downtown center point + */ +case class CityCenter(name: String, lat: Double, lng: Double) + +/** + * The dimensions of a map viewport at a given zoom level. + * + * These represent what a user sees on their phone screen at each zoom level. + * Larger viewports (lower zoom) fetch more data from the Overpass API, so + * we use these to measure how response time scales with area. + * + * @param zoom OSM/Slippy map zoom level (10 = metro region, 15 = a few blocks) + * @param latSpan Height of the viewport in degrees of latitude + * @param lngSpan Width of the viewport in degrees of longitude + */ +case class ZoomViewport(zoom: Int, latSpan: Double, lngSpan: Double) + +object TestData { + + /** + * US cities with verified downtown coordinates targeting high-surveillance areas. + * + * Each city was chosen because its downtown has significant camera density, + * producing realistic query results. The coordinates point to specific + * well-known locations in each city's central business district. + */ + val cities: Seq[CityCenter] = Seq( + CityCenter("Denver", 39.7478, -104.9995), // 16th St Mall / Union Station + CityCenter("Los Angeles", 34.0483, -118.2530), // Pershing Square, DTLA + CityCenter("San Francisco", 37.7946, -122.3999), // Financial District / Market & Montgomery + CityCenter("New York", 40.7549, -73.9840), // Midtown / 42nd & 6th Ave + CityCenter("Boston", 42.3567, -71.0588), // Downtown Crossing + CityCenter("Chicago", 41.8783, -87.6258) // State & Madison, The Loop + ) + + /** + * Map viewport sizes for zoom levels 10 through 15. + * + * Calculated for a ~400x800px mobile screen (portrait orientation) at ~40 deg N + * latitude using standard OSM/Slippy map tile math (Mercator projection, + * 256px tiles). Each zoom level doubles the tile count, halving the viewport span. + * + * | Zoom | Approx area covered | Example | + * |------|-----------------------|------------------------------| + * | 15 | ~1.5 x 3 km | A few city blocks | + * | 14 | ~3 x 6 km | A neighborhood | + * | 13 | ~6 x 12 km | A district | + * | 12 | ~12 x 23 km | A mid-size city | + * | 11 | ~23 x 47 km | A large city extent | + * | 10 | ~47 x 93 km | A metro region | + * + * Ordered from tightest to widest so the simulation can walk through them + * and show the performance impact of increasing viewport size. + */ + val zoomViewports: Seq[ZoomViewport] = Seq( + ZoomViewport(15, 0.026, 0.017), + ZoomViewport(14, 0.053, 0.034), + ZoomViewport(13, 0.105, 0.069), + ZoomViewport(12, 0.210, 0.140), + ZoomViewport(11, 0.420, 0.270), + ZoomViewport(10, 0.840, 0.550) + ) + + /** + * Build the Gatling session variables for a city/viewport combination. + * + * Centers the viewport on the city's downtown coordinates and pre-builds + * the Overpass query body. All feeders delegate to this method. + */ + def buildFeedEntry(city: CityCenter, viewport: ZoomViewport): Map[String, Any] = { + val south = city.lat - viewport.latSpan / 2 + val north = city.lat + viewport.latSpan / 2 + val west = city.lng - viewport.lngSpan / 2 + val east = city.lng + viewport.lngSpan / 2 + + Map( + OverpassQuery.CityName -> city.name, + OverpassQuery.ZoomLevel -> viewport.zoom, + OverpassQuery.QueryBody -> OverpassQuery.buildQuery(south, west, north, east) + ) + } + + /** + * Infinite feeder that picks a random city and random zoom level each call. + * + * Unlike feederForZoom (which fixes a zoom level), this is useful for + * concurrent and stress simulations where zoom progression doesn't matter — + * we just want a stream of realistic, varied requests. + * + * Produces the same session keys (cityName, zoomLevel, queryBody) so it + * works with the existing overpassRequest definition. + */ + val randomFeeder: Iterator[Map[String, Any]] = Iterator.continually { + val rng = ThreadLocalRandom.current() + buildFeedEntry(cities(rng.nextInt(cities.length)), zoomViewports(rng.nextInt(zoomViewports.length))) + } + + /** + * Infinite feeder weighted toward zoom levels 13-15 (~80% of picks). + * + * Models real app usage: most users are zoomed in to neighborhood/district + * level (z13-z15). Only ~20% of requests come from zoomed-out views + * (z10-z12) where users are browsing before zooming in. + * + * Weight distribution (matches zoomViewports order, tightest to widest): + * z15: 30%, z14: 25%, z13: 25%, z12: 10%, z11: 5%, z10: 5% + */ + val weightedZoomFeeder: Iterator[Map[String, Any]] = { + // Weights parallel zoomViewports order (z15 first, z10 last). + val weights = Seq(6, 5, 5, 2, 1, 1) + val weightedPool: IndexedSeq[ZoomViewport] = + zoomViewports.zip(weights).flatMap { case (vp, count) => Seq.fill(count)(vp) }.toIndexedSeq + + Iterator.continually { + val rng = ThreadLocalRandom.current() + buildFeedEntry(cities(rng.nextInt(cities.length)), weightedPool(rng.nextInt(weightedPool.length))) + } + } +} diff --git a/load-tests/src/test/scala/deflock/OverpassQuerySpec.scala b/load-tests/src/test/scala/deflock/OverpassQuerySpec.scala new file mode 100644 index 00000000..e9af1356 --- /dev/null +++ b/load-tests/src/test/scala/deflock/OverpassQuerySpec.scala @@ -0,0 +1,56 @@ +package deflock + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class OverpassQuerySpec extends AnyFlatSpec with Matchers { + + "buildQuery" should "include the server timeout" in { + val query = OverpassQuery.buildQuery(39.0, -105.0, 40.0, -104.0) + query should include (s"[timeout:${OverpassQuery.serverTimeoutSeconds}]") + } + + it should "request JSON output" in { + val query = OverpassQuery.buildQuery(39.0, -105.0, 40.0, -104.0) + query should include ("[out:json]") + } + + it should "include the bounding box in Overpass order (south,west,north,east)" in { + val query = OverpassQuery.buildQuery(39.0, -105.0, 40.0, -104.0) + query should include ("(39.0,-105.0,40.0,-104.0)") + } + + it should "include all configured tag filters" in { + val query = OverpassQuery.buildQuery(39.0, -105.0, 40.0, -104.0) + OverpassQuery.tagFilters.foreach { filter => + query should include (filter) + } + } + + it should "include parent way and relation lookups" in { + val query = OverpassQuery.buildQuery(39.0, -105.0, 40.0, -104.0) + query should include ("way(bn)") + query should include ("rel(bn)") + } + + it should "end with out skel for parent geometry" in { + val query = OverpassQuery.buildQuery(39.0, -105.0, 40.0, -104.0) + query should endWith ("out skel;") + } + + "tagFilters" should "have one entry per app profile (11 built-in profiles)" in { + OverpassQuery.tagFilters should have size 11 + } + + it should "all start with man_made=surveillance" in { + every(OverpassQuery.tagFilters) should include ("""["man_made"="surveillance"]""") + } + + "serverTimeoutSeconds" should "be positive" in { + OverpassQuery.serverTimeoutSeconds should be > 0 + } + + "clientTimeout" should "be longer than server timeout" in { + OverpassQuery.clientTimeout.toSeconds should be > OverpassQuery.serverTimeoutSeconds.toLong + } +} diff --git a/load-tests/src/test/scala/deflock/TestDataSpec.scala b/load-tests/src/test/scala/deflock/TestDataSpec.scala new file mode 100644 index 00000000..8138bd60 --- /dev/null +++ b/load-tests/src/test/scala/deflock/TestDataSpec.scala @@ -0,0 +1,127 @@ +package deflock + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TestDataSpec extends AnyFlatSpec with Matchers { + + // --- Test data constants --- + + "cities" should "contain 6 entries" in { + TestData.cities should have length 6 + } + + it should "have unique names" in { + val names = TestData.cities.map(_.name) + names.distinct should have length names.length.toLong + } + + it should "have valid coordinates (US mainland)" in { + TestData.cities.foreach { city => + city.lat should (be >= 24.0 and be <= 50.0) + city.lng should (be >= -125.0 and be <= -66.0) + } + } + + "zoomViewports" should "cover zoom levels 10-15" in { + TestData.zoomViewports.map(_.zoom).sorted shouldBe Seq(10, 11, 12, 13, 14, 15) + } + + it should "have spans that grow as zoom decreases (wider view)" in { + val sortedByZoom = TestData.zoomViewports.sortBy(_.zoom) + sortedByZoom.sliding(2).foreach { case Seq(wider, tighter) => + wider.latSpan should be > tighter.latSpan + wider.lngSpan should be > tighter.lngSpan + } + } + + it should "have roughly 2x span increase per zoom level" in { + val sortedByZoom = TestData.zoomViewports.sortBy(_.zoom) + sortedByZoom.sliding(2).foreach { case Seq(wider, tighter) => + val latRatio = wider.latSpan / tighter.latSpan + latRatio should (be >= 1.8 and be <= 2.2) + } + } + + // --- buildFeedEntry --- + + "buildFeedEntry" should "produce all required session keys" in { + val city = CityCenter("Test", 40.0, -74.0) + val viewport = ZoomViewport(15, 0.026, 0.017) + val entry = TestData.buildFeedEntry(city, viewport) + + entry should contain key OverpassQuery.CityName + entry should contain key OverpassQuery.ZoomLevel + entry should contain key OverpassQuery.QueryBody + } + + it should "set cityName and zoomLevel from inputs" in { + val city = CityCenter("Denver", 39.7478, -104.9995) + val viewport = ZoomViewport(13, 0.105, 0.069) + val entry = TestData.buildFeedEntry(city, viewport) + + entry(OverpassQuery.CityName) shouldBe "Denver" + entry(OverpassQuery.ZoomLevel) shouldBe 13 + } + + it should "center the bounding box on the city coordinates" in { + val city = CityCenter("Test", 40.0, -74.0) + val viewport = ZoomViewport(15, 0.026, 0.017) + val entry = TestData.buildFeedEntry(city, viewport) + val query = entry(OverpassQuery.QueryBody).asInstanceOf[String] + + // south = 40.0 - 0.013 = 39.987, north = 40.0 + 0.013 = 40.013 + // west = -74.0 - 0.0085 = -74.0085, east = -74.0 + 0.0085 = -73.9915 + query should include ("39.987") + query should include ("40.013") + query should include ("-74.0085") + query should include ("-73.9915") + } + + // --- randomFeeder --- + + "randomFeeder" should "produce valid entries on each call" in { + val entries = (1 to 20).map(_ => TestData.randomFeeder.next()) + + entries.foreach { entry => + entry should contain key OverpassQuery.CityName + entry should contain key OverpassQuery.ZoomLevel + entry should contain key OverpassQuery.QueryBody + TestData.cities.map(_.name) should contain (entry(OverpassQuery.CityName)) + TestData.zoomViewports.map(_.zoom) should contain (entry(OverpassQuery.ZoomLevel)) + } + } + + it should "produce varied results (not always the same)" in { + val entries = (1 to 50).map(_ => TestData.randomFeeder.next()) + val uniqueCities = entries.map(_(OverpassQuery.CityName)).distinct + val uniqueZooms = entries.map(_(OverpassQuery.ZoomLevel)).distinct + + uniqueCities.length should be > 1 + uniqueZooms.length should be > 1 + } + + // --- weightedZoomFeeder --- + + "weightedZoomFeeder" should "produce valid entries" in { + val entries = (1 to 20).map(_ => TestData.weightedZoomFeeder.next()) + + entries.foreach { entry => + entry should contain key OverpassQuery.CityName + entry should contain key OverpassQuery.ZoomLevel + entry should contain key OverpassQuery.QueryBody + } + } + + it should "favor zoom levels 13-15 over 10-12" in { + val entries = (1 to 1000).map(_ => TestData.weightedZoomFeeder.next()) + val zooms = entries.map(_(OverpassQuery.ZoomLevel).asInstanceOf[Int]) + + val closeZooms = zooms.count(z => z >= 13 && z <= 15) + val farZooms = zooms.count(z => z >= 10 && z <= 12) + + // 80% should be z13-z15, 20% z10-z12. Allow some variance. + closeZooms.toDouble / entries.length should be > 0.65 + farZooms.toDouble / entries.length should be < 0.35 + } +}