diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..69cd23d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,103 @@ +name: CI + +on: + push: + branches: + - master + - "ai/**" + tags: + - "*" + pull_request: + release: + types: + - published + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + package-smoke: + name: Package smoke (${{ matrix.os }}, Python ${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + python-version: + - "3.11" + - "3.13" + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install build tools + run: python -m pip install --upgrade pip build + + - name: Install package + run: python -m pip install -e . + + - name: Compile sources + run: python -m compileall -q setup.py index.py tools twms + + - name: Import WSGI application + run: python -c "import importlib.metadata; import twms; import twms.daemon; assert twms.__version__ == '0.07z'; assert importlib.metadata.version('twms') == '0.7+z'; print(type(twms.daemon.application).__name__)" + + - name: Run legacy smoke tests + run: python -m unittest discover -s tests + + - name: Build source and wheel distributions + run: python -m build + + windows-exe: + name: Windows executable artifact + runs-on: windows-latest + needs: package-smoke + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install build tools + run: python -m pip install --upgrade pip pyinstaller + + - name: Install package + run: python -m pip install -e . + + - name: Write PyInstaller entry points + shell: pwsh + run: | + @' + from twms.server import main + + if __name__ == "__main__": + main() + '@ | Set-Content -Encoding UTF8 twms-stdlib-entry.py + + @' + from twms.daemon import main + + if __name__ == "__main__": + main() + '@ | Set-Content -Encoding UTF8 twms-webpy-entry.py + + - name: Build executables + run: | + python -m PyInstaller --onefile --name twms twms-stdlib-entry.py + python -m PyInstaller --onefile --name twms-webpy twms-webpy-entry.py + + - name: Upload executable artifact + uses: actions/upload-artifact@v4 + with: + name: twms-windows-exe-${{ github.sha }} + path: dist/*.exe diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c713d46 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +*.py[cod] + +build/ +dist/ +*.egg-info/ + +*.spec diff --git a/.hgtags b/.hgtags deleted file mode 100644 index c0e3b5e..0000000 --- a/.hgtags +++ /dev/null @@ -1,5 +0,0 @@ -0a1cc6bba99284ea984710c18cf9bc116d791928 0.01q -bfab1f075a7f33cfd760956b94f3de3fbcbefdf2 0.02w -962296dc4deb9b4821e21d0dd0e17e1fde2ef2d1 0.03e -e40a1db64469baab230c1b37d0069bf15d2abf90 0.04r -413c35c82b67468f77e35663dc37040c681ccaa2 0.05t diff --git a/COPYING b/COPYING index f41b044..1d14a85 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,5 @@ -The authors, Darafei Praliaskouski (Komzpa) and Andrew Shadura, +The authors, Darafei Praliaskouski (Komzpa), Andrew Shadura, and +Eugene Dvoretsky (Radioxoma), explicitly disclaim copyright in all jurisdictions which recognise such a disclaimer. In such jurisdictions, this software is released into the Public Domain. @@ -7,6 +8,7 @@ In jurisdictions which do not recognise Public Domain property, this software is Copyright © 2009—2013 Darafei Praliaskouski Copyright © 2010—2013, 2016 Andrew Shadura + Copyright © 2020—2021, 2023—2024 Eugene Dvoretsky and is released under the terms of the ISC License (see below). diff --git a/README.md b/README.md index 942386c..01f5fa8 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,258 @@ -About -===== +# twms -twms is a script that connects World of Tiles and World of WMS. -The name ‘twms’ stands for twms web map server. +twms is a tiny web map service that connects the world of tiles and +the world of WMS. -The primary purpose of twms is to export your map tiles to the -WMS-enabled applications. +The primary purpose of twms is to export raster tile sets to +WMS-enabled GIS applications. It can also act as a small tile proxy: +fetching remote tiles, keeping a filesystem cache, and serving that +cache back through WMS, WMTS, TileJSON, or direct slippy-map tile URLs. -twms can export a set of raster tiles as a WMS service -so GIS applications that support WMS protocol can access -this tile set. Also, twms can act as a proxy and perform -WMS requests to external services and serve the tile cache +twms intentionally stays small. The default server uses the Python +standard library, while the older `web.py` WSGI/standalone path is +still available for deployments that already use it. -TODO -==== +## Install and run - - Make fetchers work with proxy - - Full reprojection support - - Imagery realignment +Install from a checkout: -Conventions -=========== +```sh +python -m pip install -e . +``` - - Inside tWMS, only EPSG:4326 latlon should be used for transmitting coordinates. +Run the default stdlib server: + +```sh +twms 8080 +``` + +or: + +```sh +python -m twms 8080 +``` + +The previous `web.py` server and WSGI adapter are still installed as: + +```sh +twms-webpy +``` + +That path is kept for older deployments, including Windows/JOSM proxy +bundles that depended on `web.py`. + +On GitHub, CI builds Windows executable artifacts for both entry points: + +- `twms.exe` +- `twms-webpy.exe` + +Optional launcher templates are shipped under `share/twms/contrib`: + +- `twms.bat` starts `python -m twms` minimized for small Windows/JOSM proxy + deployments. +- `twms.desktop` is a simple terminal desktop-entry template for Linux + desktops or downstream packages. + +## Configuration + +twms loads Python configuration from: + +1. `/etc/twms/twms.conf` +2. the packaged `twms/twms.conf` +3. `twms.conf` in the current script directory + +Important settings: + +- `tiles_cache`: root directory for the filesystem tile cache +- `gpx_cache`: cache for downloaded OSM GPX traces +- `service_url`: externally visible base URL used in generated capabilities +- `upstream_timeout`: default upstream HTTP timeout in seconds; the example + config uses 30 seconds so threaded servers do not wait forever on a stalled + tile source +- `upstream_retries` / `upstream_retry_delay`: optional retry budget for + transient upstream network errors. The default is one attempt, preserving the + historical no-retry behavior; layers can opt in with their own values. +- `default_layers`: layer list used when a request does not name layers +- `default_format`: output image MIME type, usually `image/jpeg` +- `layers`: configured imagery layers and their fetchers + +Layer dictionaries usually define: + +- `name`: human readable title +- `prefix`: cache subdirectory +- `ext`: tile extension such as `jpg` or `png` +- `proj`: tile pyramid projection, commonly `EPSG:3857` or `EPSG:3395` +- `remote_url`: upstream tile/WMS URL template; legacy tile `%s/%s/%s` + templates still work, named tile placeholders `{z}`, `{x}`, `{y}`, `{-y}`, + and `{q}` are accepted for readable Slippy/TMS/Bing URLs, and WMS upstream + templates may use `{bbox}`, `{width}`, `{height}`, and `{proj}` +- `headers`: optional upstream HTTP request headers, such as `Referer`, + `User-Agent`, or authentication cookies required by a particular source +- `fetch`: fetcher function, normally `fetchers.Tile`; readable aliases + `"tms"` / `"tile"` and `"wms"` are also accepted +- `timeout`: optional per-layer upstream HTTP timeout in seconds; set to + `None` only if an old deployment deliberately wants the historical unbounded + wait +- `upstream_retries` / `upstream_retry_delay`: optional per-layer retry + override for temporary network failures. HTTP errors such as 404 are still + handled by the cache/TNE rules instead of being retried as generic transport + failures. +- `min_zoom` / `max_zoom`: optional zoom limits +- `cache_ttl`: optional fresh-cache lifetime in seconds +- `cache_layout`: optional cache path layout; the default is TWMS' + historical grouped layout, while `zxy` stores slippy/MOBAC-style + `////.` tiles +- `dead_tile`: optional dead-tile marker, either a legacy file path or a + `{ "size": ..., "md5": {...} }` dictionary; dictionaries may also set + `http_status` for an upstream status code that should be cached as `.tne` + +TWMS intentionally does not read browser cookie stores automatically. If a +private deployment needs a short-lived cookie, copy it into the layer `headers` +or load it from your own local config code so the server does not gain a +browser-profile dependency. + +## Client URLs + +Assuming the server runs at `http://127.0.0.1:8080/`: + +- overview page: + `http://127.0.0.1:8080/` +- WMS 1.1.1/1.3.0 endpoint: + `http://127.0.0.1:8080/` +- WMS capabilities: + `http://127.0.0.1:8080/?service=WMS&request=GetCapabilities&version=1.3.0` +- WMTS capabilities: + `http://127.0.0.1:8080/wmts/1.0.0/WMTSCapabilities.xml` +- TileJSON for a layer: + `http://127.0.0.1:8080/tilejson/osm.json` +- direct tile URL: + `http://127.0.0.1:8080/osm/{z}/{x}/{y}.png` +- WMTS REST tile URL: + `http://127.0.0.1:8080/wmts/osm/{z}/{x}/{y}.png` + +The legacy `GetTile` request is still supported because it is useful +when clients need filters or other TWMS-specific request parameters in +the tile URL: + +```text +http://127.0.0.1:8080/?request=GetTile&layers=osm&z={z}&x={x}&y={y}&format=png +``` + +## QGIS + +For WMS, create a WMS/WMTS connection pointing at: + +```text +http://127.0.0.1:8080/ +``` + +QGIS may request WMS 1.3.0 capabilities with uppercase parameter names +such as `SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0`; twms +accepts those. WMS 1.3.0 also accepts the `crs` parameter and advertises +`CRS:84` for lon/lat bounds, while the old WMS 1.1.1 capabilities keep +their existing `SRS` listings. + +For WMTS, use: + +```text +http://127.0.0.1:8080/wmts/1.0.0/WMTSCapabilities.xml +``` + +## JOSM + +For a normal local proxy, add a TMS imagery entry such as: + +```text +tms:http://127.0.0.1:8080/osm/{zoom}/{x}/{y}.png +``` + +JOSM can also consume the generated imagery list: + +```text +http://127.0.0.1:8080/josm/imagery.xml +``` + +The generated list includes configured layer bounds, zoom limits, overlays, +attribution URLs, and known no-tile MD5 checksums when those are present. + +For TWMS-specific parameters, use the WMS-style `GetTile` URL instead: + +```text +tms:http://127.0.0.1:8080/?request=GetTile&layers=osm&z={zoom}&x={x}&y={y}&format=png +``` + +JOSM can also point directly at a compatible local slippy-map cache with +`file://` if no proxy or reprojection is needed. Configure that layer with +`cache_layout: "zxy"` so TWMS uses the same `//` path: + +```text +tms:file:///home/user/SAS.Planet/cache_ma/osm/{zoom}/{x}/{y}.png +``` + +On Windows the same idea uses a Windows path: + +```text +tms:file:///C:/SAS.Planet/cache_ma/osm/{zoom}/{x}/{y}.png +``` + +## Shared tile caches + +By default, twms keeps the historical filesystem cache layout under +`tiles_cache`: + +```text +//z//x//y. +``` + +Fresh cached tiles are served without network access. When `cache_ttl` +expires, twms tries to refresh the tile; if the remote fetch fails, the +stale cached tile can still be used. Missing/dead tiles can be recorded +as `.tne` files so repeated requests do not hammer upstream services. + +Layers that need to share a slippy-map/MOBAC-style cache, including +SAS.Planet and similar offline tile workflows, can opt in with +`cache_layout: "zxy"`: + +```text +////. +``` + +## Optional dependencies + +The base install keeps dependencies small: + +- `Pillow` +- `web.py` for the legacy WSGI/standalone server path + +Optional extras: + +```sh +python -m pip install -e '.[proj]' +python -m pip install -e '.[cairo]' +``` + +`twms[proj]` enables pyproj-backed transformations for configured +non-core projections. Without it, twms still has built-in pure-Python +support for the common EPSG:4326/EPSG:3857/EPSG:3395 path. + +`twms[cairo]` enables Cairo-backed vector rendering where the old +rendering path uses it. + +## Credits + +twms was originally written by Darafei Praliaskouski (Komzpa). +Andrew Shadura maintains Debian packaging and wrote the Debian manpage. +Eugene Dvoretsky (Radioxoma) contributed modernization work around +packaging, serving, caching, projections, documentation, and tile +protocols. + +## TODO + +- Make fetchers work with proxy. +- Full reprojection support. +- Imagery realignment. + +## Conventions + +- Inside twms, EPSG:4326 lon/lat should be used for transmitting + coordinates. diff --git a/contrib/twms.bat b/contrib/twms.bat new file mode 100644 index 0000000..c555bec --- /dev/null +++ b/contrib/twms.bat @@ -0,0 +1,3 @@ +@echo off +rem Start TWMS minimized for small Windows/JOSM proxy deployments. +start "twms" /MIN python -m twms %* diff --git a/contrib/twms.desktop b/contrib/twms.desktop new file mode 100644 index 0000000..cb6f868 --- /dev/null +++ b/contrib/twms.desktop @@ -0,0 +1,11 @@ +[Desktop Entry] +Version=1.0 +Type=Application +Name=TWMS +GenericName=Tiny WMS/TMS server +Comment=Start the tiny WMS/TMS tile proxy +Categories=Geoscience;Geography;Network; +Exec=twms +Terminal=true +StartupNotify=false +Icon=utilities-terminal diff --git a/index.py b/index.py index d3321c0..f2b79d6 100644 --- a/index.py +++ b/index.py @@ -10,8 +10,9 @@ if __name__ != "__main__": try: - from mod_python import apache, util import datetime + + from mod_python import apache, util except ImportError: pass diff --git a/irs_nxt.jpg b/irs_nxt.jpg deleted file mode 100644 index e69de29..0000000 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..301a9c6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..57f9133 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,77 @@ +[metadata] +name = twms +version = attr: twms.version.__packaging_version__ +author = Darafei Praliaskouski +author_email = me@komzpa.net +url = https://github.com/komzpa/twms +description = tiny web map service +long_description = file: README.md +license = Public Domain or ISC +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Web Environment + Intended Audience :: Developers + Intended Audience :: End Users/Desktop + License :: Public Domain + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 + Topic :: Internet :: WWW/HTTP + Topic :: Scientific/Engineering :: GIS + +[options] +python_requires = >= 3.11 +packages = find: +install_requires = + Pillow + web.py +include_package_data = True + +[options.extras_require] +proj = pyproj +cairo = pycairo + +[options.entry_points] +console_scripts = + twms = twms.server:main + twms-webpy = twms.daemon:main + +[flake8] +doctests = yes +max-line-length = 130 +exclude = .git,build,__pycache__,setup.py +ignore = E121,E123,E126,E133,E226,E241,E242,E704,E501,E301,E261,E127,E128,W391,W503,W504 +# https://pep8.readthedocs.org/en/latest/intro.html#error-codes +# These are ignored by default: +# - E121: continuation line under-indented for hanging indent +# - E123: closing bracket does not match indentation of opening bracket's line +# - E126: continuation line over-indented for hanging indent +# - E133: closing bracket does not match visual indentation +# - E226: missing whitespace around arithmetic operator +# - E241: multiple spaces after ',' +# - E242: tab after ',' +# - E704: multiple statements on one line (def) +# These were added because PEP-8 allows exceptions, but pep8 doesn't: +# - E501: line too long +# - E301: expected 1 blank line, found 0 +# - E261: at least two spaces before inline comment +# - E127: continuation line over-indented for visual indent +# - E128: continuation line under-indented for visual indent +# - W391: blank line at end of file +# These were added because PEP-8 is wrong sometimes +# - W503: line break before binary operator +# - W504: line break after binary operator + +[isort] +# from X import ( +# a, +# b, +# ) +multi_line_output = 3 +include_trailing_comma = true +lines_after_imports = 2 +line_length = 130 +reverse_relative = true +default_section = THIRDPARTY diff --git a/setup.py b/setup.py index 476becd..d22973a 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,9 @@ import os import platform from glob import glob as abs_glob -from setuptools import setup, find_packages + +from setuptools import find_packages, setup + __platform__ = platform.system() is_windows = __platform__ in ['Windows'] @@ -42,6 +44,7 @@ def config_files(): # monkey patch setuptools to use distutils owner/group functionality from setuptools.command import sdist + sdist_org = sdist.sdist @@ -55,42 +58,11 @@ def initialize_options(self): setup( name = __name__, - version = "0.06y", - author = 'Darafei Praliaskoiski', - author_email = 'me@komzpa.net', - url = 'https://github.com/komzpa/twms', - description = 'tiny web map service', - long_description = read('README.md'), - license = 'Public Domain or ISC', - packages = find_packages(), - install_requires = ['Pillow', 'web.py'], - extras_require = { - 'proj': ['pyproj'], - 'cairo': ['pycairo'], - }, - classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Web Environment', - 'Intended Audience :: Developers', - 'Intended Audience :: End Users/Desktop', - 'License :: Public Domain', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Scientific/Engineering :: GIS', - ], - include_package_data = True, data_files = [ (os.path.join('share', 'doc', __name__), ['COPYING']), (os.path.join('share', 'doc', __name__), glob('*.md')), (os.path.join('share', __name__), glob('*.jpg')), - (os.path.join('share', __name__, 'tools'), glob(os.path.join('tools', '*.py'))) + (os.path.join('share', __name__, 'tools'), glob(os.path.join('tools', '*.py'))), + (os.path.join('share', __name__, 'contrib'), glob(os.path.join('contrib', '*'))), ] + man_files('*.1') + config_files(), - entry_points = { - 'console_scripts': [ - 'twms = twms.daemon:main' - ] - } ) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py new file mode 100644 index 0000000..415b1e2 --- /dev/null +++ b/tests/test_legacy_smoke.py @@ -0,0 +1,1910 @@ +import datetime +import importlib +import importlib.metadata +import hashlib +import json +import math +import os +from collections import OrderedDict +from io import BytesIO +import tempfile +import threading +import unittest +import urllib.error +import urllib.request +import warnings +from http.server import ThreadingHTTPServer +import xml.etree.ElementTree as ET +from unittest import mock + +from PIL import Image + +import twms +import twms.canvas +import twms.config_loader +import twms.daemon +import twms.fetchers +import twms.filter +import twms.projections +import twms.server +import twms.twms + + +class LegacySmokeTest(unittest.TestCase): + def image_bytes(self, color, image_format="PNG"): + buffer = BytesIO() + image = Image.new("RGBA", (256, 256), color) + if image_format == "JPEG": + image = image.convert("RGB") + image.save(buffer, image_format) + return buffer.getvalue() + + def cache_path(self, cache_root, layer, z, x, y): + return os.path.join( + cache_root, + layer["prefix"], + "z%s" % z, + "%s" % (x // 1024), + "x%s" % x, + "%s" % (y // 1024), + "y%s.%s" % (y, twms.fetchers._layer_extension(layer)), + ) + + def zxy_cache_path(self, cache_root, layer, z, x, y): + return os.path.join( + cache_root, + layer["prefix"], + "%s" % z, + "%s" % x, + "%s.%s" % (y, twms.fetchers._layer_extension(layer)), + ) + + def test_public_version_keeps_keyboard_suffix(self): + self.assertEqual(twms.__version__, "0.07z") + self.assertEqual(importlib.metadata.version("twms"), "0.7+z") + + def test_layer_metadata_normalizes_ext_and_mimetype(self): + module = type("Config", (), {})() + module.default_format = "image/png" + module.layers = { + "mimetype-only": {"mimetype": "image/png"}, + "ext-only": {"ext": "jpg"}, + "default-format": {}, + } + + twms.config_loader.normalize_layer_metadata(module) + + self.assertEqual(module.layers["mimetype-only"]["ext"], "png") + self.assertEqual(module.layers["ext-only"]["mimetype"], "image/jpeg") + self.assertEqual(module.layers["default-format"]["mimetype"], "image/png") + self.assertEqual(module.layers["default-format"]["ext"], "png") + + def test_layer_metadata_supports_layer_defaults(self): + module = type("Config", (), {})() + module.layer_defaults = { + "mimetype": "image/png", + "proj": "EPSG:3857", + "cached": False, + } + module.layers = { + "defaulted": {"name": "Defaulted", "prefix": "defaulted"}, + "override": {"name": "Override", "prefix": "override", "ext": "jpg"}, + } + + twms.config_loader.normalize_layer_metadata(module) + + defaulted = module.layers["defaulted"] + override = module.layers["override"] + self.assertNotIn("ext", defaulted) + self.assertEqual(defaulted["proj"], "EPSG:3857") + self.assertEqual(defaulted.get("mimetype"), "image/png") + self.assertEqual(defaulted.get("ext"), "png") + self.assertEqual(defaulted.get("cached"), False) + self.assertEqual(override["proj"], "EPSG:3857") + self.assertEqual(override.get("mimetype"), "image/jpeg") + self.assertEqual(override.get("ext"), "jpg") + + def test_layer_metadata_accepts_string_fetch_aliases(self): + module = type("Config", (), {})() + module.layers = { + "tiles": {"fetch": "tms"}, + "wms": {"fetch": "wms"}, + } + + twms.config_loader.normalize_layer_metadata(module) + + self.assertIs(module.layers["tiles"]["fetch"], twms.fetchers.Tile) + self.assertIs(module.layers["wms"]["fetch"], twms.fetchers.WMS) + + def test_layer_metadata_accepts_default_string_fetch_alias(self): + module = type("Config", (), {})() + module.layer_defaults = {"fetch": "wms"} + module.layers = { + "defaulted": {"name": "Defaulted", "prefix": "defaulted"}, + } + + twms.config_loader.normalize_layer_metadata(module) + + self.assertIs(module.layers["defaulted"]["fetch"], twms.fetchers.WMS) + + def test_layer_metadata_rejects_unknown_string_fetch_alias(self): + module = type("Config", (), {})() + module.layers = {"bad": {"fetch": "factory-factory"}} + + with self.assertRaisesRegex(ValueError, "factory-factory"): + twms.config_loader.normalize_layer_metadata(module) + + def test_legacy_modules_import_as_package_modules(self): + modules = [ + "twms.bbox", + "twms.capabilities", + "twms.correctify", + "twms.drawing", + "twms.filter", + "twms.gpxparse", + "twms.image_compat", + "twms.josm", + "twms.overview", + "twms.projections", + "twms.reproject", + "twms.sketch", + "twms.wmts", + ] + for module in modules: + with self.subTest(module=module): + importlib.import_module(module) + + def test_wms_capabilities_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetCapabilities", + "version": "1.1.1", + "ref": "http://example.test/wms", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/vnd.ogc.wms_xml") + self.assertIn("image/webp", body) + self.assertNotIn("CRS:84", body) + + def test_wms_130_capabilities_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "SERVICE": "WMS", + "REQUEST": "GetCapabilities", + "VERSION": "1.3.0", + "ref": "http://example.test/wms", + } + ) + + root = ET.fromstring(body) + namespaces = {"wms": "http://www.opengis.net/wms"} + osm = root.find( + "./wms:Capability/wms:Layer/wms:Layer[wms:Name='osm']", + namespaces, + ) + crs_values = [element.text for element in osm.findall("wms:CRS", namespaces)] + bbox = osm.find("wms:BoundingBox", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(root.tag, "{http://www.opengis.net/wms}WMS_Capabilities") + self.assertEqual(root.attrib["version"], "1.3.0") + self.assertIn("CRS:84", crs_values) + self.assertIn("EPSG:3857", crs_values) + self.assertEqual(bbox.attrib["CRS"], "EPSG:3857") + self.assertLess(float(bbox.attrib["minx"]), -20000000) + self.assertGreater(float(bbox.attrib["maxx"]), 20000000) + + def test_wms_getmap_accepts_crs_parameter(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "transparent", + "format": "image/png", + "width": "32", + "height": "32", + "crs": "CRS:84", + "bbox": "-1,-1,1,1", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (32, 32)) + + def test_wms_getmap_accepts_webp_format(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "transparent", + "format": "image/webp", + "width": "32", + "height": "32", + "srs": "EPSG:3857", + "bbox": "-1,-1,1,1", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/webp") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.format, "WEBP") + self.assertEqual(image.size, (32, 32)) + + def test_overview_smoke(self): + status, content_type, body = twms.twms.twms_main({"ref": "http://example.test/"}) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/html") + self.assertIn("", body) + self.assertIn("Yandex Satellite", body) + + def test_overview_accepts_bounds_alias_and_provider_link(self): + old_config_layers = twms.twms.config.layers + old_overview_layers = twms.twms.overview.layers + layers = { + "bounded": { + "name": "Bounded", + "prefix": "bounded", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (1.0, 2.0, 3.0, 4.0), + "provider_url": "http://provider.example/", + } + } + twms.twms.config.layers = layers + twms.twms.overview.layers = layers + try: + status, content_type, body = twms.twms.twms_main( + {"ref": "http://example.test/"} + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/html") + self.assertIn("bbox=1.0,2.0,3.0,4.0", body) + self.assertIn( + '' + "Bounded", + body, + ) + finally: + twms.twms.config.layers = old_config_layers + twms.twms.overview.layers = old_overview_layers + + def test_overview_accepts_mimetype_only_layer(self): + old_config_layers = twms.twms.config.layers + old_overview_layers = twms.twms.overview.layers + layers = { + "typed": { + "name": "Typed", + "prefix": "typed", + "mimetype": "image/png", + "proj": "EPSG:3857", + } + } + twms.twms.config.layers = layers + twms.twms.overview.layers = layers + try: + status, content_type, body = twms.twms.twms_main( + {"ref": "http://example.test/"} + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/html") + self.assertIn("http://example.test/typed/!/!/!.png", body) + finally: + twms.twms.config.layers = old_config_layers + twms.twms.overview.layers = old_overview_layers + + def test_overview_accepts_layer_defaults(self): + old_config_layers = twms.twms.config.layers + old_overview_layers = twms.twms.overview.layers + module = type("Config", (), {})() + module.layer_defaults = {"mimetype": "image/png", "proj": "EPSG:3857"} + module.layers = {"defaulted": {"name": "Defaulted", "prefix": "defaulted"}} + twms.config_loader.normalize_layer_metadata(module) + twms.twms.config.layers = module.layers + twms.twms.overview.layers = module.layers + try: + status, content_type, body = twms.twms.twms_main( + {"ref": "http://example.test/"} + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/html") + self.assertIn("http://example.test/defaulted/!/!/!.png", body) + finally: + twms.twms.config.layers = old_config_layers + twms.twms.overview.layers = old_overview_layers + + def test_gettile_transparent_layer_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "0", + "x": "0", + "y": "0", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (256, 256)) + self.assertEqual(image.mode, "RGBA") + + def test_legacy_getcorrections_without_rectify_file(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetCorrections", + "layers": "osm", + "points": "27.6,53.2", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/plain") + self.assertEqual(body, "27.6,53.2;\n") + + def test_legacy_filter_smoke(self): + image = Image.new("RGBA", (2, 1), (10, 20, 30, 255)) + + filtered = twms.filter.raster(image, ("swaprb", "brightness:2")) + + self.assertEqual(filtered.getpixel((0, 0)), (60, 40, 20, 255)) + + def test_legacy_wkt_drawing_without_color_parameter(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "transparent", + "format": "image/png", + "width": "32", + "height": "32", + "bbox": "-1,-1,1,1", + "wkt": "LINESTRING(-1 -1,1 1)", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (32, 32)) + self.assertTrue( + any(image.getchannel("A").tobytes()), + "WKT overlay should draw visible pixels", + ) + + def test_legacy_canvas_blank_tile_smoke(self): + canvas = twms.canvas.WmsCanvas(tile_size=(32, 32)) + + canvas.FetchTile(0, 0) + + self.assertEqual(canvas.tiles[(0, 0)]["im"].size, (32, 32)) + self.assertEqual(canvas.tiles[(0, 0)]["im"].mode, "RGBA") + + def test_legacy_canvas_uses_default_upstream_timeout(self): + canvas = twms.canvas.WmsCanvas( + wms_url="http://example.test/wms?", + proj="EPSG:3857", + ) + + with mock.patch("twms.canvas.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes((1, 2, 3, 255)) + canvas.FetchTile(0, 0) + + urlopen.assert_called_once_with(mock.ANY, timeout=30) + + def test_legacy_canvas_can_preserve_unbounded_upstream_wait(self): + canvas = twms.canvas.WmsCanvas( + wms_url="http://example.test/wms?", + proj="EPSG:3857", + timeout=None, + ) + + with mock.patch("twms.canvas.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes((1, 2, 3, 255)) + canvas.FetchTile(0, 0) + + urlopen.assert_called_once_with(mock.ANY, timeout=None) + + def test_getimg_resize_works_with_current_pillow(self): + tile = Image.new("RGBA", (256, 256), (1, 2, 3, 255)) + tile.is_ok = True + layer = { + "name": "Resize smoke", + "prefix": "resize", + "proj": "EPSG:3857", + "cached": False, + "max_zoom": 1, + } + + with mock.patch("twms.twms.tile_image", return_value=tile): + image = twms.twms.getimg( + (-1.0, -1.0, 1.0, 1.0), + "EPSG:3857", + (32, 32), + layer, + datetime.datetime.now(), + (), + ) + + self.assertEqual(image.size, (32, 32)) + self.assertEqual(image.getpixel((0, 0)), (1, 2, 3, 255)) + + def test_legacy_empty_color_overlay_is_transparent(self): + base = Image.new("RGBA", (2, 1), (10, 20, 30, 255)) + overlay = Image.new("RGBA", (2, 1), (255, 255, 255, 255)) + overlay.putpixel((1, 0), (255, 0, 0, 255)) + base.is_ok = True + overlay.is_ok = True + layers = { + "base": {"empty_color": "#000000"}, + "overlay": {"empty_color": "#ffffff"}, + } + + with mock.patch.object(twms.twms.config, "layers", layers): + with mock.patch.object(twms.twms, "getimg", side_effect=[base, overlay]): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "base,overlay", + "format": "image/png", + "bbox": "0,0,1,1", + "width": "2", + "height": "1", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + image = image.convert("RGBA") + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + self.assertEqual(image.getpixel((1, 0)), (132, 10, 15, 255)) + + def test_legacy_empty_color_delta_applies_per_channel(self): + base = Image.new("RGBA", (1, 1), (10, 20, 30, 255)) + overlay = Image.new("RGBA", (1, 1), (255, 254, 253, 255)) + base.is_ok = True + overlay.is_ok = True + layers = { + "base": {"empty_color": "#000000"}, + "overlay": { + "empty_color": "#ffffff", + "empty_color_delta": 2, + }, + } + + with mock.patch.object(twms.twms.config, "layers", layers): + with mock.patch.object(twms.twms, "getimg", side_effect=[base, overlay]): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "base,overlay", + "format": "image/png", + "bbox": "0,0,1,1", + "width": "1", + "height": "1", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.convert("RGBA").getpixel((0, 0)), (10, 20, 30, 255)) + + def test_tilejson_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTileJSON", + "layers": "osm", + "ref": "http://example.test/", + } + ) + + doc = json.loads(body) + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/json") + self.assertEqual(doc["tilejson"], "3.0.0") + self.assertEqual(doc["name"], "OpenStreetMap mapnik") + self.assertEqual(doc["scheme"], "xyz") + self.assertEqual(doc["tiles"], ["http://example.test/osm/{z}/{x}/{y}.png"]) + self.assertEqual(doc["bounds"], [-180.0, -85.0511287798, 180.0, 85.0511287798]) + self.assertEqual(doc["minzoom"], 0) + self.assertEqual(doc["maxzoom"], 18) + + def test_tilejson_accepts_layer_bounds_alias(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "bounded": { + "name": "Bounded", + "prefix": "bounded", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (1.0, 2.0, 3.0, 4.0), + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTileJSON", + "layers": "bounded", + "ref": "http://example.test/", + } + ) + + doc = json.loads(body) + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/json") + self.assertEqual(doc["bounds"], [1.0, 2.0, 3.0, 4.0]) + self.assertEqual(doc["center"], [2.0, 3.0, 0]) + finally: + twms.twms.config.layers = old_layers + + def test_tilejson_accepts_mimetype_only_layer(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "typed": { + "name": "Typed", + "prefix": "typed", + "mimetype": "image/png", + "proj": "EPSG:3857", + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTileJSON", + "layers": "typed", + "ref": "http://example.test/", + } + ) + + doc = json.loads(body) + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/json") + self.assertEqual(doc["tiles"], ["http://example.test/typed/{z}/{x}/{y}.png"]) + finally: + twms.twms.config.layers = old_layers + + def test_tilejson_accepts_layer_defaults(self): + old_layers = twms.twms.config.layers + module = type("Config", (), {})() + module.layer_defaults = {"mimetype": "image/png", "proj": "EPSG:3857"} + module.layers = {"defaulted": {"name": "Defaulted", "prefix": "defaulted"}} + twms.config_loader.normalize_layer_metadata(module) + twms.twms.config.layers = module.layers + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTileJSON", + "layers": "defaulted", + "ref": "http://example.test/", + } + ) + + doc = json.loads(body) + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/json") + self.assertEqual( + doc["tiles"], ["http://example.test/defaulted/{z}/{x}/{y}.png"] + ) + finally: + twms.twms.config.layers = old_layers + + def test_josm_imagery_xml_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetJOSMImagery", + "ref": "http://example.test/", + } + ) + + namespaces = {"josm": "http://josm.openstreetmap.de/maps-1.0"} + root = ET.fromstring(body) + osm = root.find("./josm:entry[josm:id='twms-osm']", namespaces) + landsat = root.find("./josm:entry[josm:id='twms-landsat']", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(root.tag, "{http://josm.openstreetmap.de/maps-1.0}imagery") + self.assertEqual(osm.find("josm:default", namespaces).text, "true") + self.assertEqual(osm.find("josm:name", namespaces).text, "OpenStreetMap mapnik") + self.assertEqual(osm.find("josm:type", namespaces).text, "tms") + self.assertEqual( + osm.find("josm:url", namespaces).text, + "http://example.test/osm/{zoom}/{x}/{y}.png", + ) + self.assertEqual( + osm.find("josm:description", namespaces).text, + "OpenStreetMap mapnik", + ) + self.assertEqual(osm.find("josm:valid-georeference", namespaces).text, "true") + self.assertEqual(landsat.find("josm:max-zoom", namespaces).text, "11") + + def test_josm_imagery_xml_accepts_mimetype_only_layer(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "typed": { + "name": "Typed", + "prefix": "typed", + "mimetype": "image/png", + "proj": "EPSG:3857", + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetJOSMImagery", + "ref": "http://example.test/", + } + ) + + namespaces = {"josm": "http://josm.openstreetmap.de/maps-1.0"} + root = ET.fromstring(body) + entry = root.find("./josm:entry[josm:id='twms-typed']", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual( + entry.find("josm:url", namespaces).text, + "http://example.test/typed/{zoom}/{x}/{y}.png", + ) + finally: + twms.twms.config.layers = old_layers + + def test_josm_imagery_xml_layer_metadata(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "metadata": { + "name": "Metadata", + "prefix": "metadata", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (1.0, 2.0, 3.0, 4.0), + "overlay": True, + "provider_url": "http://provider.example/", + "dead_tile": { + "md5": { + "11111111111111111111111111111111", + "22222222222222222222222222222222", + }, + }, + "min_zoom": 3, + "max_zoom": 7, + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetJOSMImagery", + "ref": "http://example.test/", + } + ) + + namespaces = {"josm": "http://josm.openstreetmap.de/maps-1.0"} + root = ET.fromstring(body) + entry = root.find("./josm:entry[josm:id='twms-metadata']", namespaces) + bounds = entry.find("josm:bounds", namespaces) + checksums = entry.findall("josm:no-tile-checksum", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(entry.attrib["overlay"], "true") + self.assertEqual( + entry.find("josm:attribution-url", namespaces).text, + "http://provider.example/", + ) + self.assertEqual( + bounds.attrib, + { + "min-lon": "1.0", + "min-lat": "2.0", + "max-lon": "3.0", + "max-lat": "4.0", + }, + ) + self.assertEqual( + [(item.attrib["type"], item.attrib["value"]) for item in checksums], + [ + ("MD5", "11111111111111111111111111111111"), + ("MD5", "22222222222222222222222222222222"), + ], + ) + self.assertEqual(entry.find("josm:min-zoom", namespaces).text, "3") + self.assertEqual(entry.find("josm:max-zoom", namespaces).text, "6") + finally: + twms.twms.config.layers = old_layers + + def test_wmts_capabilities_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "service": "WMTS", + "request": "GetCapabilities", + "ref": "http://example.test/", + } + ) + + root = ET.fromstring(body) + namespaces = { + "wmts": "http://www.opengis.net/wmts/1.0", + "ows": "http://www.opengis.net/ows/1.1", + } + layer_ids = [ + element.text + for element in root.findall( + "./wmts:Contents/wmts:Layer/ows:Identifier", + namespaces, + ) + ] + resource = root.find( + "./wmts:Contents/wmts:Layer[ows:Identifier='osm']/wmts:ResourceURL", + namespaces, + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(root.tag, "{http://www.opengis.net/wmts/1.0}Capabilities") + self.assertIn("osm", layer_ids) + self.assertEqual( + resource.attrib["template"], + "http://example.test/wmts/osm/{TileMatrix}/{TileCol}/{TileRow}.png", + ) + + def test_wmts_capabilities_accepts_layer_bounds_alias(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "bounded": { + "name": "Bounded", + "prefix": "bounded", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (1.0, 2.0, 3.0, 4.0), + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "service": "WMTS", + "request": "GetCapabilities", + "ref": "http://example.test/", + } + ) + + root = ET.fromstring(body) + namespaces = { + "wmts": "http://www.opengis.net/wmts/1.0", + "ows": "http://www.opengis.net/ows/1.1", + } + bounds = root.find( + "./wmts:Contents/wmts:Layer[ows:Identifier='bounded']/ows:WGS84BoundingBox", + namespaces, + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(bounds.find("ows:LowerCorner", namespaces).text, "1.0 2.0") + self.assertEqual(bounds.find("ows:UpperCorner", namespaces).text, "3.0 4.0") + finally: + twms.twms.config.layers = old_layers + + def test_wmts_capabilities_accepts_mimetype_only_layer(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "typed": { + "name": "Typed", + "prefix": "typed", + "mimetype": "image/png", + "proj": "EPSG:3857", + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "service": "WMTS", + "request": "GetCapabilities", + "ref": "http://example.test/", + } + ) + + root = ET.fromstring(body) + namespaces = { + "wmts": "http://www.opengis.net/wmts/1.0", + "ows": "http://www.opengis.net/ows/1.1", + } + layer = root.find( + "./wmts:Contents/wmts:Layer[ows:Identifier='typed']", + namespaces, + ) + resource = layer.find("wmts:ResourceURL", namespaces) + format_node = layer.find("wmts:Format", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(resource.attrib["format"], "image/png") + self.assertEqual( + resource.attrib["template"], + "http://example.test/wmts/typed/{TileMatrix}/{TileCol}/{TileRow}.png", + ) + self.assertEqual(format_node.text, "image/png") + finally: + twms.twms.config.layers = old_layers + + def test_wmts_kvp_gettile_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "service": "WMTS", + "request": "GetTile", + "layer": "transparent", + "format": "image/png", + "tilematrix": "0", + "tilecol": "0", + "tilerow": "0", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (256, 256)) + self.assertEqual(image.mode, "RGBA") + + def test_webmercator_projection_clamps_poles(self): + maxbounds = 6378137 * math.pi + + projected = twms.projections.from4326( + (-180.0, -90.0, 180.0, 90.0), + "EPSG:3857", + ) + + self.assertEqual(projected[0], -maxbounds) + self.assertEqual(projected[1], -maxbounds) + self.assertEqual(projected[2], maxbounds) + self.assertEqual(projected[3], maxbounds) + + def test_optional_pyproj_projection_uses_modern_transformer(self): + if not hasattr(twms.projections.pyproj, "Transformer"): + self.skipTest("pyproj extra is not installed") + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + point = twms.projections.from4326((27.6, 53.2), "EPSG:32635") + + self.assertTrue(540000 < point[0] < 541000) + self.assertTrue(5894000 < point[1] < 5895000) + self.assertFalse( + [warning for warning in caught if warning.category is FutureWarning], + ) + + def test_non_core_projection_reports_missing_pyproj_extra(self): + if hasattr(twms.projections.pyproj, "Transformer"): + self.skipTest("pyproj extra is installed") + + with self.assertRaises(NotImplementedError) as raised: + twms.projections.from4326((27.6, 53.2), "EPSG:32635") + + self.assertIn("twms[proj]", str(raised.exception)) + + def test_tile_cache_uses_fresh_file_without_network(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "ttl", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_ttl": 3600, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_not_called() + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_legacy_tile_image_reuses_historical_cache_path(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.twms.config.tiles_cache + twms.twms.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "legacy-hit", + "proj": "EPSG:3857", + "ext": "png", + "cached": True, + "scalable": False, + "empty_color": "#000000", + "fetch": mock.Mock(return_value=None), + } + try: + path = self.cache_path(cache_root, layer, 3, 3, 3) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + + image = twms.twms.tile_image( + layer, + 3, + 3, + 3, + datetime.datetime.now(), + real=True, + ) + + layer["fetch"].assert_not_called() + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + finally: + twms.twms.config.tiles_cache = old_cache + + def test_legacy_ram_cache_is_lru_bounded(self): + old_cached_objs = twms.twms.cached_objs + old_limit = twms.twms.config.max_ram_cached_tiles + twms.twms.cached_objs = OrderedDict() + twms.twms.config.max_ram_cached_tiles = 2 + layer = {"prefix": "ram"} + first = Image.new("RGBA", (1, 1), (1, 1, 1, 255)) + second = Image.new("RGBA", (1, 1), (2, 2, 2, 255)) + third = Image.new("RGBA", (1, 1), (3, 3, 3, 255)) + try: + first_key = twms.twms._ram_cache_key(layer, 1, 1, 1) + second_key = twms.twms._ram_cache_key(layer, 1, 2, 2) + third_key = twms.twms._ram_cache_key(layer, 1, 3, 3) + twms.twms._ram_cache_put(first_key, first) + twms.twms._ram_cache_put(second_key, second) + + self.assertIs(twms.twms._ram_cache_get(first_key), first) + + twms.twms._ram_cache_put(third_key, third) + + self.assertIn(first_key, twms.twms.cached_objs) + self.assertNotIn(second_key, twms.twms.cached_objs) + self.assertIn(third_key, twms.twms.cached_objs) + finally: + twms.twms.cached_objs = old_cached_objs + twms.twms.config.max_ram_cached_tiles = old_limit + + def test_legacy_gettile_fast_cache_reads_binary_tile(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.twms.config.tiles_cache + old_layers = twms.twms.config.layers + twms.twms.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "legacy-fast-hit", + "proj": "EPSG:3857", + "ext": "png", + } + twms.twms.config.layers = {"legacy-fast-hit": layer} + try: + path = self.cache_path(cache_root, layer, 3, 3, 4) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "legacy-fast-hit", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + finally: + twms.twms.config.tiles_cache = old_cache + twms.twms.config.layers = old_layers + + def test_legacy_response_cache_reads_binary_tile(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = getattr(twms.twms.config, "cache_tile_responses", None) + had_cache = hasattr(twms.twms.config, "cache_tile_responses") + twms.twms.config.cache_tile_responses = { + ("EPSG:3857", ("transparent",), (), 256, 256, (), "PNG"): ( + cache_root, + "png", + ), + } + try: + path = os.path.join(cache_root, "2", "3", "4.png") + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + finally: + if had_cache: + twms.twms.config.cache_tile_responses = old_cache + else: + del twms.twms.config.cache_tile_responses + + def test_legacy_response_cache_accepts_mime_format_key(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = getattr(twms.twms.config, "cache_tile_responses", None) + had_cache = hasattr(twms.twms.config, "cache_tile_responses") + twms.twms.config.cache_tile_responses = { + ("EPSG:3857", ("transparent",), (), 256, 256, (), "image/png"): ( + cache_root, + "png", + ), + } + try: + path = os.path.join(cache_root, "2", "3", "4.png") + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((12, 34, 56, 255))) + + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.getpixel((0, 0)), (12, 34, 56, 255)) + finally: + if had_cache: + twms.twms.config.cache_tile_responses = old_cache + else: + del twms.twms.config.cache_tile_responses + + def test_legacy_response_cache_writes_binary_tile(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = getattr(twms.twms.config, "cache_tile_responses", None) + had_cache = hasattr(twms.twms.config, "cache_tile_responses") + twms.twms.config.cache_tile_responses = { + ("EPSG:3857", ("transparent",), (), 256, 256, (), "PNG"): ( + cache_root, + "png", + ), + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + path = os.path.join(cache_root, "2", "3", "4.png") + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with open(path, "rb") as cached_tile: + self.assertEqual(cached_tile.read(), body) + with Image.open(path) as image: + self.assertEqual(image.size, (256, 256)) + finally: + if had_cache: + twms.twms.config.cache_tile_responses = old_cache + else: + del twms.twms.config.cache_tile_responses + + def test_legacy_response_cache_writes_mime_format_key(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = getattr(twms.twms.config, "cache_tile_responses", None) + had_cache = hasattr(twms.twms.config, "cache_tile_responses") + twms.twms.config.cache_tile_responses = { + ("EPSG:3857", ("transparent",), (), 256, 256, (), "image/png"): ( + cache_root, + "png", + ), + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + path = os.path.join(cache_root, "2", "3", "4.png") + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with open(path, "rb") as cached_tile: + self.assertEqual(cached_tile.read(), body) + with Image.open(path) as image: + self.assertEqual(image.size, (256, 256)) + finally: + if had_cache: + twms.twms.config.cache_tile_responses = old_cache + else: + del twms.twms.config.cache_tile_responses + + def test_tile_cache_can_use_zxy_layout(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "zxy", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_layout": "zxy", + } + try: + path = self.zxy_cache_path(cache_root, layer, 2, 3, 4) + legacy_path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (10, 20, 30, 255) + ) + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + self.assertTrue(os.path.exists(path)) + self.assertFalse(os.path.exists(legacy_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_cache_can_reuse_fresh_zxy_file_without_network(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "zxy-hit", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_layout": "zxy", + "cache_ttl": 3600, + } + try: + path = self.zxy_cache_path(cache_root, layer, 2, 3, 4) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as tile_file: + tile_file.write(self.image_bytes((70, 80, 90, 255))) + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.getpixel((0, 0)), (70, 80, 90, 255)) + urlopen.assert_not_called() + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_cache_refetches_expired_file_and_keeps_stale_on_error(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "ttl", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_ttl": 1, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + old_time = 946684800 + os.utime(path, (old_time, old_time)) + + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (40, 50, 60, 255) + ) + image = twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_called_once() + self.assertEqual(image.getpixel((0, 0)), (40, 50, 60, 255)) + + os.utime(path, (old_time, old_time)) + with mock.patch("twms.fetchers.urlopen", side_effect=OSError): + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.getpixel((0, 0)), (40, 50, 60, 255)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_fetch_retries_transient_upstream_error(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "retry", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "upstream_retries": 2, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + response = mock.Mock() + response.read.return_value = self.image_bytes((11, 22, 33, 255)) + with mock.patch( + "twms.fetchers.urlopen", + side_effect=[OSError("temporary"), response], + ) as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(urlopen.call_count, 2) + self.assertEqual(image.getpixel((0, 0)), (11, 22, 33, 255)) + self.assertTrue(os.path.exists(path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_fetch_does_not_retry_http_tne_status(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "retry-http-tne", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "upstream_retries": 3, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + error = urllib.error.HTTPError( + url="http://example.test/2/3/4.png", + code=404, + msg="Not Found", + hdrs={}, + fp=None, + ) + with mock.patch("twms.fetchers.urlopen", side_effect=error) as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_called_once() + self.assertFalse(image) + self.assertTrue(os.path.exists(tne_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_cache_tne_suppresses_fetch_until_ttl_expires(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "ttl", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_ttl": 3600, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + os.makedirs(os.path.dirname(tne_path)) + open(tne_path, "wb").close() + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_not_called() + self.assertIsNone(image) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_dead_tile_dict_is_recorded_as_tne(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + body = self.image_bytes((255, 0, 0, 255)) + layer = { + "prefix": "dead", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "dead_tile": { + "size": len(body), + "md5": {hashlib.md5(body).hexdigest()}, + }, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = body + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertFalse(image) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(tne_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_invalid_downloaded_tile_is_not_cached(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "invalid", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = b"not an image" + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertFalse(image) + self.assertFalse(os.path.exists(path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_cache_converts_download_to_layer_extension(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "format", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.jpg", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (20, 30, 40), image_format="JPEG" + ) + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.format, "JPEG") + with Image.open(path) as cached_image: + self.assertEqual(cached_image.format, "PNG") + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_cache_accepts_mimetype_only_layer(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "mimetype", + "mimetype": "image/png", + "remote_url": "http://example.test/%s/%s/%s.jpg", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (20, 30, 40), image_format="JPEG" + ) + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.format, "JPEG") + self.assertTrue(path.endswith(".png")) + with Image.open(path) as cached_image: + self.assertEqual(cached_image.format, "PNG") + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_http_404_tile_is_recorded_as_tne(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "http-tne", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + error = urllib.error.HTTPError( + "http://example.test/2/3/4.png", + 404, + "Not Found", + hdrs={}, + fp=None, + ) + with mock.patch("twms.fetchers.urlopen", side_effect=error): + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertFalse(image) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(tne_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_wms_fetcher_caches_downloaded_image(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "wms-cache", + "ext": "png", + "remote_url": "http://example.test/wms?", + "proj": "EPSG:3857", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + image = twms.fetchers.WMS(2, 3, 4, layer) + + self.assertEqual(image.getpixel((0, 0)), (1, 2, 3, 255)) + self.assertTrue(os.path.exists(path)) + urlopen.assert_called_once_with(mock.ANY, timeout=30) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_wms_fetcher_keeps_legacy_url_append(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "wms-legacy-url", + "ext": "png", + "remote_url": "http://example.test/wms?", + "proj": "EPSG:3857", + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.WMS(2, 3, 4, layer) + + url = urlopen.call_args.args[0] + self.assertIsInstance(url, str) + self.assertEqual(urlopen.call_args.kwargs["timeout"], 30) + self.assertTrue(url.startswith("http://example.test/wms?bbox=")) + self.assertIn("&width=384&height=384&srs=EPSG:3857", url) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_wms_fetcher_formats_named_url_placeholders(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "wms-template-url", + "ext": "png", + "remote_url": ( + "http://example.test/wms?SERVICE=WMS&REQUEST=GetMap" + "&WIDTH={width}&HEIGHT={height}&CRS={proj}&BBOX={bbox}" + ), + "proj": "EPSG:3857", + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.WMS(2, 3, 4, layer) + + url = urlopen.call_args.args[0] + self.assertIsInstance(url, str) + self.assertEqual(urlopen.call_args.kwargs["timeout"], 30) + self.assertIn("WIDTH=384&HEIGHT=384&CRS=EPSG:3857&BBOX=", url) + self.assertNotIn("srs=EPSG:3857", url) + self.assertEqual(url.count("BBOX="), 1) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_fetcher_respects_min_zoom(self): + layer = { + "prefix": "minzoom", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "min_zoom": 3, + } + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertIsNone(image) + urlopen.assert_not_called() + + def test_legacy_tile_image_accepts_layer_bounds_alias(self): + layer = { + "prefix": "bounded", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (170.0, -80.0, 171.0, -79.0), + "scalable": False, + "fetch": twms.fetchers.Tile, + "remote_url": "http://example.test/%s/%s/%s.png", + } + + with mock.patch("twms.fetchers.fetch") as fetch: + image = twms.twms.tile_image( + layer, + 2, + 0, + 0, + datetime.datetime.now(), + ) + + self.assertIsNone(image) + fetch.assert_not_called() + + def test_wms_fetcher_respects_min_zoom(self): + layer = { + "prefix": "wms-minzoom", + "ext": "png", + "remote_url": "http://example.test/wms?", + "proj": "EPSG:3857", + "min_zoom": 3, + } + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.WMS(2, 3, 4, layer) + + self.assertIsNone(image) + urlopen.assert_not_called() + + def test_tile_fetcher_keeps_legacy_exclusive_max_zoom(self): + layer = { + "prefix": "maxzoom", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "max_zoom": 2, + } + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertIsNone(image) + urlopen.assert_not_called() + + def test_tile_fetcher_sends_configured_headers(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "headers", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "headers": { + "Referer": "https://example.test/map/", + "User-Agent": "twms-test", + }, + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(2, 3, 4, layer) + + request = urlopen.call_args.args[0] + headers = { + key.lower(): value for key, value in request.header_items() + } + self.assertEqual(urlopen.call_args.kwargs["timeout"], 30) + self.assertEqual(request.full_url, "http://example.test/2/3/4.png") + self.assertEqual(headers["referer"], "https://example.test/map/") + self.assertEqual(headers["user-agent"], "twms-test") + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_wms_fetcher_sends_configured_headers(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "wms-headers", + "ext": "png", + "remote_url": "http://example.test/wms?", + "proj": "EPSG:3857", + "headers": { + "Referer": "https://example.test/wms-client/", + }, + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.WMS(2, 3, 4, layer) + + request = urlopen.call_args.args[0] + headers = { + key.lower(): value for key, value in request.header_items() + } + self.assertEqual(urlopen.call_args.kwargs["timeout"], 30) + self.assertTrue(request.full_url.startswith("http://example.test/wms?")) + self.assertIn("bbox=", request.full_url) + self.assertEqual(headers["referer"], "https://example.test/wms-client/") + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_fetcher_uses_configured_layer_timeout(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "timeout", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "timeout": 7, + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_called_once_with( + "http://example.test/2/3/4.png", + timeout=7, + ) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_configured_http_status_tile_is_recorded_as_tne(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "http-tne", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "dead_tile": {"http_status": 410}, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + error = urllib.error.HTTPError( + "http://example.test/2/3/4.png", + 410, + "Gone", + hdrs={}, + fp=None, + ) + with mock.patch("twms.fetchers.urlopen", side_effect=error): + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertFalse(image) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(tne_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_legacy_percent_tile_template_still_uses_transform_tuple(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "url", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "transform_tile_number": lambda z, x, y: (x, y, z - 1), + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_called_once_with( + "http://example.test/3/4/1.png", + timeout=30, + ) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_named_tile_template_placeholders(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "url", + "ext": "png", + "remote_url": "http://example.test/{z}/{x}/{y}/{-y}/{q}.png", + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(4, 9, 5, layer) + + urlopen.assert_called_once_with( + "http://example.test/4/9/5/10/1203.png", + timeout=30, + ) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_named_tile_template_uses_transform_tuple(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "url", + "ext": "png", + "remote_url": "http://example.test/z{z}/x{x}/y{y}.png", + "transform_tile_number": lambda z, x, y: (z - 1, x + 1, y + 2), + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(4, 5, 6, layer) + + urlopen.assert_called_once_with( + "http://example.test/z3/x6/y8.png", + timeout=30, + ) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_wsgi_application_imports(self): + self.assertTrue(callable(twms.daemon.application)) + + def test_stdlib_server_startup_banner_lists_client_urls(self): + banner = twms.server.startup_banner("", 8080) + + self.assertIn("TWMS server 0.07z listening on 0.0.0.0:8080", banner) + self.assertIn("Overview: http://127.0.0.1:8080/", banner) + self.assertIn( + "WMS: http://127.0.0.1:8080/wms?SERVICE=WMS&REQUEST=GetCapabilities", + banner, + ) + self.assertIn( + "WMTS: http://127.0.0.1:8080/wmts/1.0.0/WMTSCapabilities.xml", + banner, + ) + self.assertIn("JOSM imagery: http://127.0.0.1:8080/josm/maps.xml", banner) + self.assertNotIn("Cookie", banner) + self.assertNotIn("headers", banner.lower()) + + def test_stdlib_server_serves_wms_and_gettile(self): + httpd = ThreadingHTTPServer(("127.0.0.1", 0), twms.server.TWMSRequestHandler) + thread = threading.Thread(target=httpd.serve_forever) + thread.daemon = True + thread.start() + base = "http://127.0.0.1:%s" % httpd.server_address[1] + try: + with urllib.request.urlopen( + base + "/?request=GetCapabilities&version=1.1.1" + ) as response: + body = response.read().decode("utf-8") + self.assertEqual(response.status, 200) + self.assertIn("twms/0.07z", response.headers["Server"]) + self.assertIn("application/vnd.ogc.wms_xml", response.headers["Content-Type"]) + self.assertIn(" """ ) - pset = set(projections.projs.keys()) - pset = pset.union(set(projections.proj_alias.keys())) - for proj in pset: + for proj in _legacy_srs_ids(): req += "%s" % proj req += """ @@ -192,6 +318,7 @@ def get(version, ref): image/jpeg image/gif image/bmp + image/webp @@ -216,9 +343,7 @@ def get(version, ref): World Map""" ) - pset = set(projections.projs.keys()) - pset = pset.union(set(projections.proj_alias.keys())) - for proj in pset: + for proj in _legacy_srs_ids(): req += "%s" % proj req += """ diff --git a/twms/config_loader.py b/twms/config_loader.py new file mode 100644 index 0000000..3c5770c --- /dev/null +++ b/twms/config_loader.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- + +import importlib.machinery +import importlib.util +import mimetypes +import os +import sys + + +class LayerConfig(dict): + def __init__(self, defaults, values): + super().__init__(values) + self.defaults = defaults + + def __missing__(self, key): + if key in self.defaults: + return self.defaults[key] + raise KeyError(key) + + def get(self, key, default=None): + if key in self: + return super().get(key) + return self.defaults.get(key, default) + + +def _extension_from_mimetype(mimetype): + extension = mimetypes.guess_extension(mimetype or "") + if not extension: + return None + return extension.strip(".").lower().replace("jpeg", "jpg") + + +def _mimetype_from_extension(extension): + extension = (extension or "").lower().strip(".").replace("jpeg", "jpg") + if extension == "jpg": + return "image/jpeg" + if extension: + return mimetypes.types_map.get("." + extension, "image/" + extension) + return None + + +def _normalize_format_metadata(layer, default_mimetype=None): + if "mimetype" in layer and "ext" not in layer: + extension = _extension_from_mimetype(layer["mimetype"]) + if extension: + layer["ext"] = extension + if "ext" in layer and "mimetype" not in layer: + mimetype = _mimetype_from_extension(layer["ext"]) + if mimetype: + layer["mimetype"] = mimetype + if "ext" not in layer and "mimetype" not in layer and default_mimetype: + layer["mimetype"] = default_mimetype + extension = _extension_from_mimetype(default_mimetype) + if extension: + layer["ext"] = extension + + +def _normalize_fetch_metadata(layer): + fetch = layer.get("fetch") + if not isinstance(fetch, str): + return + + from twms import fetchers + + fetchers_by_name = { + "tile": fetchers.Tile, + "tms": fetchers.Tile, + "wms": fetchers.WMS, + } + try: + layer["fetch"] = fetchers_by_name[fetch.lower()] + except KeyError: + raise ValueError("Unknown fetcher alias: %s" % fetch) + + +def normalize_layer_metadata(module): + default_mimetype = getattr(module, "default_format", None) + layer_defaults = getattr(module, "layer_defaults", None) + if isinstance(layer_defaults, dict): + _normalize_format_metadata(layer_defaults, default_mimetype) + _normalize_fetch_metadata(layer_defaults) + for name, layer in list(getattr(module, "layers", {}).items()): + _normalize_format_metadata(layer, default_mimetype) + _normalize_fetch_metadata(layer) + if isinstance(layer_defaults, dict): + module.layers[name] = LayerConfig(layer_defaults, layer) + + +def load_config(path): + loader = importlib.machinery.SourceFileLoader("twms.config", path) + spec = importlib.util.spec_from_loader("twms.config", loader) + module = importlib.util.module_from_spec(spec) + sys.modules["twms.config"] = module + sys.modules["config"] = module + loader.exec_module(module) + normalize_layer_metadata(module) + return module + + +def load_default_config(): + if "config" in sys.modules: + return sys.modules["config"] + + package_dir = os.path.dirname(__file__) + paths = [ + "/etc/twms/twms.conf", + os.path.join(package_dir, "twms.conf"), + os.path.join(os.path.realpath(sys.path[0]), "twms.conf"), + ] + for path in paths: + if os.path.exists(path): + return load_config(path) + return load_config(paths[-1]) diff --git a/twms/correctify.py b/twms/correctify.py index 6d17736..1f8254c 100644 --- a/twms/correctify.py +++ b/twms/correctify.py @@ -5,12 +5,14 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -import projections import os + import config +import projections -distance = lambda z, x, y, g: ((z - y) ** 2 + (x - g) ** 2) ** (0.5) +def distance(z, x, y, g): + return ((z - y) ** 2 + (x - g) ** 2) ** (0.5) def has_corrections(layer): diff --git a/twms/daemon.py b/twms/daemon.py index 97e61b5..15e06fa 100755 --- a/twms/daemon.py +++ b/twms/daemon.py @@ -8,11 +8,13 @@ from __future__ import print_function -import web +import socket import sys + +import web + from twms import * -import sys, socket try: import psyco @@ -35,7 +37,7 @@ def handler(data): urls = ( - "/(.*)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?(.*)", "tilehandler", + r"/(.*)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?(.*)", "tilehandler", "/(.*)", "mainhandler", ) diff --git a/twms/drawing.py b/twms/drawing.py index cc69d8c..acd5ef8 100644 --- a/twms/drawing.py +++ b/twms/drawing.py @@ -5,19 +5,16 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -try: - from PIL import Image, ImageDraw, ImageColor, ImageFont -except ImportError: - import Image, ImageDraw, ImageColor, ImageFont - -import urllib -import os, sys import array +import math +import os +import sys +import urllib -import projections import config +import projections from gpxparse import GPXParser -import math +from PIL import Image, ImageColor, ImageDraw, ImageFont HAVE_CAIRO = True @@ -107,7 +104,7 @@ def render_vector( if renderer == "cairo" and HAVE_CAIRO: "rendering as cairo" - imgd = img.tostring() + imgd = img.tobytes() a = array.array("B", imgd) surface = cairo.ImageSurface.create_for_data( a, cairo.FORMAT_ARGB32, W, H, W * 4 diff --git a/twms/fetchers.py b/twms/fetchers.py index 02188c9..98d3225 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -5,30 +5,231 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -from urllib.request import urlopen import filecmp -import time -import os +import hashlib import math +import os import sys -from io import BytesIO - -try: - from PIL import Image -except ImportError: - import Image - +import threading import time +from io import BytesIO +from urllib.error import HTTPError +from urllib.error import URLError +from urllib.request import Request, urlopen import config import projections -import threading +from PIL import Image +from twms.image_compat import resampling_lanczos fetching_now = {} thread_responses = {} zhash_lock = {} +_EXTENSION_FORMATS = { + "gif": "GIF", + "jpg": "JPEG", + "jpeg": "JPEG", + "png": "PNG", + "webp": "WEBP", +} + + +def _layer_extension(this_layer): + if "ext" in this_layer: + return this_layer["ext"].lower().strip(".").replace("jpeg", "jpg") + mimetype = this_layer.get("mimetype", "image/jpeg") + return mimetype.lower().replace("image/", "").replace("jpeg", "jpg") + + +def _cache_stem(z, x, y, this_layer): + cache_prefix = config.tiles_cache + this_layer["prefix"] + if this_layer.get("cache_layout") in ("zxy", "slippy", "mobac", "tms"): + return cache_prefix + "/%s/%s/%s." % (z, x, y) + return ( + cache_prefix + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) + ) + + +def _upstream_request(url, this_layer): + headers = this_layer.get("headers") + if headers: + return Request(url, headers=headers) + return url + + +def _upstream_timeout(this_layer): + if "timeout" in this_layer: + return this_layer["timeout"] + return getattr( + config, + "upstream_timeout", + min(getattr(config, "deadline", 30), 30), + ) + + +def _upstream_retries(this_layer): + return max( + 1, + int(this_layer.get("upstream_retries", getattr(config, "upstream_retries", 1))), + ) + + +def _upstream_retry_delay(this_layer): + return float( + this_layer.get( + "upstream_retry_delay", getattr(config, "upstream_retry_delay", 0) + ) + ) + + +def _read_upstream(url, this_layer): + attempts = _upstream_retries(this_layer) + for attempt in range(attempts): + try: + return urlopen( + _upstream_request(url, this_layer), + timeout=_upstream_timeout(this_layer), + ).read() + except HTTPError: + raise + except (OSError, URLError): + if attempt + 1 == attempts: + raise + delay = _upstream_retry_delay(this_layer) + if delay: + time.sleep(delay) + + + +def _outside_zoom_limits(z, this_layer): + if "min_zoom" in this_layer and z < this_layer["min_zoom"]: + return True + if "max_zoom" in this_layer and z >= this_layer["max_zoom"]: + return True + return False + + +def _format_wms_url(template, bbox, width, height, projection): + if any( + placeholder in template + for placeholder in ("{bbox}", "{width}", "{height}", "{proj}") + ): + return ( + template.replace("{bbox}", bbox) + .replace("{width}", str(width)) + .replace("{height}", str(height)) + .replace("{proj}", projection) + ) + + return template + "bbox=%s&width=%s&height=%s&srs=%s" % ( + bbox, + width, + height, + projection, + ) + + +class TileCache: + """Small filesystem cache helper. + + This keeps TWMS' historical zN/NNN/xN/NNN/yN.ext layout, but adds the useful + cache semantics from Radioxoma's fork: TTL checks, readable TNE markers, + stale-cache fallback, and atomic file replacement. + """ + + def __init__(self, z, x, y, this_layer): + self.layer = this_layer + self.cached = this_layer.get("cached", True) + if self.cached: + self.stem = _cache_stem(z, x, y, this_layer) + self.path = self.stem + _layer_extension(this_layer) + self.tne_path = self.stem + "tne" + self.lock_path = self.stem + "lock" + else: + self.stem = None + self.path = None + self.tne_path = None + self.lock_path = None + + def _fresh(self, path): + if not os.path.exists(path): + return False + ttl = self.layer.get("cache_ttl") + if not ttl: + return True + return time.time() - os.path.getmtime(path) <= ttl + + def needs_fetch(self): + if not self.cached: + return True + if self._fresh(self.tne_path): + return False + if self._fresh(self.path): + return False + return True + + def open_image(self, include_stale=False): + if not self.cached or not os.path.exists(self.path): + return None + if not include_stale and not self._fresh(self.path): + return None + return Image.open(self.path) + + def ensure_parent(self): + parent = os.path.dirname(self.stem) + if parent and not os.path.exists(parent): + os.makedirs(parent) + + def acquire(self): + if not self.cached: + return False + self.ensure_parent() + os.mkdir(self.lock_path) + return True + + def release(self): + if self.cached and os.path.exists(self.lock_path): + os.rmdir(self.lock_path) + + def wait_for_peer(self): + for _ in range(20): + time.sleep(0.1) + if not os.path.exists(self.lock_path): + if self._fresh(self.tne_path): + return None + return self.open_image() + return None + + def write_bytes(self, contents): + if not self.cached: + return + tmp_path = self.path + ".tmp.%s" % os.getpid() + with open(tmp_path, "wb") as tile_file: + tile_file.write(contents) + os.replace(tmp_path, self.path) + if os.path.exists(self.tne_path): + os.remove(self.tne_path) + + def save_image(self, image): + if not self.cached: + return + tmp_path = self.path + ".tmp.%s" % os.getpid() + image.save(tmp_path, _EXTENSION_FORMATS.get(_layer_extension(self.layer))) + os.replace(tmp_path, self.path) + if os.path.exists(self.tne_path): + os.remove(self.tne_path) + + def mark_tne(self): + if not self.cached: + return + self.ensure_parent() + if os.path.exists(self.path): + os.remove(self.path) + with open(self.tne_path, "wb") as tne: + tne.write(b"") + def fetch(z, x, y, this_layer): zhash = repr((z, x, y, this_layer)) @@ -68,113 +269,198 @@ def threadwrapper(z, x, y, this_layer, zhash): def WMS(z, x, y, this_layer): - if "max_zoom" in this_layer: - if z >= this_layer["max_zoom"]: - return None - wms = this_layer["remote_url"] + if _outside_zoom_limits(z, this_layer): + return None req_proj = this_layer.get("wms_proj", this_layer["proj"]) width = 384 # using larger source size to rescale better in python height = 384 - local = ( - config.tiles_cache - + this_layer["prefix"] - + "/z%s/%s/x%s/%s/y%s." % (z, x / 1024, x, y / 1024, y) - ) - tile_bbox = "bbox=%s,%s,%s,%s" % tuple( + cache = TileCache(z, x, y, this_layer) + tile_bbox = "%s,%s,%s,%s" % tuple( projections.from4326(projections.bbox_by_tile(z, x, y, req_proj), req_proj) ) - wms += tile_bbox + "&width=%s&height=%s&srs=%s" % (width, height, req_proj) + wms = _format_wms_url(this_layer["remote_url"], tile_bbox, width, height, req_proj) + if this_layer.get("cached", True) and not cache.needs_fetch(): + return cache.open_image() + locked = False if this_layer.get("cached", True): - if not os.path.exists("/".join(local.split("/")[:-1])): - os.makedirs("/".join(local.split("/")[:-1])) try: - os.mkdir(local + "lock") + locked = cache.acquire() except OSError: - for i in range(20): - time.sleep(0.1) - try: - if not os.path.exists(local + "lock"): - im = Image.open(local + this_layer["ext"]) - return im - except (IOError, OSError): - return None - im = Image.open(BytesIO(urlopen(wms).read())) - if width != 256 and height != 256: - im = im.resize((256, 256), Image.ANTIALIAS) - im = im.convert("RGBA") + return cache.wait_for_peer() + try: + try: + contents = _read_upstream(wms, this_layer) + im = _open_downloaded_image(contents) + if im is None: + raise OSError + except OSError: + stale = cache.open_image(include_stale=True) + if stale is not None: + return stale + return False + if width != 256 and height != 256: + im = im.resize((256, 256), resampling_lanczos(Image)) + im = im.convert("RGBA") - if this_layer.get("cached", True): - ic = Image.new( - "RGBA", (256, 256), this_layer.get("empty_color", config.default_background) - ) - if im.histogram() == ic.histogram(): - tne = open(local + "tne", "wb") - when = time.localtime() - tne.write( - "%02d.%02d.%04d %02d:%02d:%02d" - % (when[2], when[1], when[0], when[3], when[4], when[5]) + if this_layer.get("cached", True): + ic = Image.new( + "RGBA", + (256, 256), + this_layer.get("empty_color", config.default_background), ) - tne.close() - return False - im.save(local + this_layer["ext"]) - os.rmdir(local + "lock") - return im + if im.histogram() == ic.histogram(): + cache.mark_tne() + return False + cache.save_image(im) + return im + finally: + if locked: + cache.release() def Tile(z, x, y, this_layer): global OSError, IOError d_tuple = z, x, y - if "max_zoom" in this_layer: - if z >= this_layer["max_zoom"]: - return None + if _outside_zoom_limits(z, this_layer): + return None if "transform_tile_number" in this_layer: d_tuple = this_layer["transform_tile_number"](z, x, y) - remote = this_layer["remote_url"] % d_tuple + remote = _format_tile_url(this_layer["remote_url"], d_tuple) + cache = TileCache(z, x, y, this_layer) + if this_layer.get("cached", True) and not cache.needs_fetch(): + return cache.open_image() + locked = False if this_layer.get("cached", True): - local = ( - config.tiles_cache - + this_layer["prefix"] - + "/z%s/%s/x%s/%s/y%s." % (z, x / 1024, x, y / 1024, y) - ) - if not os.path.exists("/".join(local.split("/")[:-1])): - os.makedirs("/".join(local.split("/")[:-1])) try: - os.mkdir(local + "lock") + locked = cache.acquire() except OSError: - for i in range(20): - time.sleep(0.1) - try: - if not os.path.exists(local + "lock"): - im = Image.open(local + this_layer["ext"]) - return im - except (IOError, OSError): - return None + return cache.wait_for_peer() try: - contents = urlopen(remote).read() - im = Image.open(BytesIO(contents)) - except IOError: + try: + contents = _read_upstream(remote, this_layer) + im = _open_downloaded_image(contents) + if im is None: + raise OSError + except HTTPError as error: + if _is_tne_http_error(error, this_layer): + cache.mark_tne() + return False + stale = cache.open_image(include_stale=True) + if stale is not None: + return stale + return False + except OSError: + stale = cache.open_image(include_stale=True) + if stale is not None: + return stale + return False + if "dead_tile" in this_layer and _is_dead_tile(contents, this_layer["dead_tile"]): + cache.mark_tne() + return False if this_layer.get("cached", True): - os.rmdir(local + "lock") + cache.write_bytes(_cache_image_bytes(contents, im, _layer_extension(this_layer))) + return im + finally: + if locked: + cache.release() + + +def _open_downloaded_image(contents): + if not contents: + return None + image = Image.open(BytesIO(contents)) + image.load() + return image + + +def _cache_image_bytes(contents, image, extension): + target_format = _EXTENSION_FORMATS.get(extension.lower()) + if target_format is None or image.format == target_format: + return contents + + image_content = BytesIO() + if target_format == "JPEG": + image = image.convert("RGB") + image.save( + image_content, + target_format, + quality=config.output_quality, + progressive=config.output_progressive, + ) + elif target_format == "PNG": + image.save( + image_content, + target_format, + optimize=config.output_optimize, + ) + else: + image.save(image_content, target_format) + return image_content.getvalue() + + +def _format_tile_url(template, tile): + if "{" not in template: + return template % tile + + z, x, y = tile + return ( + template.replace("{z}", str(z)) + .replace("{x}", str(x)) + .replace("{y}", str(y)) + .replace("{-y}", str(tile_slippy_to_tms(z, x, y)[2])) + .replace("{q}", tile_to_quadkey(z, x, y)) + ) + + +def tile_to_quadkey(z, x, y): + quadkey = [] + for offset in range(z): + bit = z - offset + digit = ord("0") + mask = 1 << (bit - 1) + if x & mask: + digit += 1 + if y & mask: + digit += 2 + quadkey.append(chr(digit)) + return "".join(quadkey) + + +def tile_slippy_to_tms(z, x, y): + return z, x, (1 << z) - y - 1 + + +def _is_tne_http_error(error, this_layer): + status = getattr(error, "status", error.code) + if status == 404: + return True + + dead_tile = this_layer.get("dead_tile") + if not isinstance(dead_tile, dict) or "http_status" not in dead_tile: return False - if this_layer.get("cached", True): - os.rmdir(local + "lock") - open(local + this_layer["ext"], "wb").write(contents) - if "dead_tile" in this_layer: - try: - dt = open(this_layer["dead_tile"], "rb").read() - if contents == dt: - if this_layer.get("cached", True): - tne = open(local + "tne", "wb") - when = time.localtime() - tne.write( - "%02d.%02d.%04d %02d:%02d:%02d" - % (when[2], when[1], when[0], when[3], when[4], when[5]) - ) - tne.close() - os.remove(local + this_layer["ext"]) + + configured = dead_tile["http_status"] + if isinstance(configured, (list, tuple, set)): + return status in configured + return status == configured + + +def _is_dead_tile(contents, dead_tile): + if isinstance(dead_tile, dict): + if "size" in dead_tile and len(contents) != dead_tile["size"]: return False - except IOError: - pass - return im + if "md5" in dead_tile: + md5 = hashlib.md5(contents).hexdigest() + if md5 not in dead_tile["md5"]: + return False + if "sha256" in dead_tile: + sha256 = hashlib.sha256(contents).hexdigest() + if sha256 != dead_tile["sha256"]: + return False + return True + try: + return contents == open(dead_tile, "rb").read() + except IOError: + return False diff --git a/twms/filter.py b/twms/filter.py index 1dc2a4f..10480a5 100644 --- a/twms/filter.py +++ b/twms/filter.py @@ -5,10 +5,8 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -try: - from PIL import Image, ImageFilter, ImageEnhance, ImageOps -except ImportError: - import Image, ImageFilter, ImageEnhance, ImageOps +from PIL import Image, ImageEnhance, ImageFilter, ImageOps + try: import numpy @@ -17,7 +15,9 @@ except ImportError: NUMPY_AVAILABLE = False import datetime -from twms import getimg + +from twms.twms import getimg + try: import config diff --git a/twms/gpxparse.py b/twms/gpxparse.py index 5bfcd55..7f8c3fc 100644 --- a/twms/gpxparse.py +++ b/twms/gpxparse.py @@ -5,8 +5,12 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -import sys, string, bz2, gzip, os -from xml.dom import minidom, Node +import bz2 +import gzip +import os +import string +import sys +from xml.dom import Node, minidom class GPXParser: diff --git a/twms/image_compat.py b/twms/image_compat.py new file mode 100644 index 0000000..d82a8fe --- /dev/null +++ b/twms/image_compat.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# This file is part of twms. + +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms specified in COPYING. + + +def resampling_lanczos(Image): + """Return the best Pillow downsampling filter across old and new Pillow.""" + if hasattr(Image, "Resampling"): + return Image.Resampling.LANCZOS + return Image.ANTIALIAS diff --git a/twms/josm.py b/twms/josm.py new file mode 100644 index 0000000..27736e1 --- /dev/null +++ b/twms/josm.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# This file is part of twms. + +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms specified in COPYING. + +import xml.etree.ElementTree as ET + + +JOSM_MAPS = "http://josm.openstreetmap.de/maps-1.0" +ET.register_namespace("", JOSM_MAPS) + + +def _tag(name): + return "{%s}%s" % (JOSM_MAPS, name) + + +def _layer_extension(layer): + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg").lower().replace("image/", ""), + ).lower().replace("jpeg", "jpg") + + +def _layer_url(ref, layer_name, layer): + return "%s%s/{zoom}/{x}/{y}.%s" % ( + ref, + layer_name, + _layer_extension(layer), + ) + + +def _layer_bounds(config, layer): + return layer.get( + "data_bounding_box", + layer.get("bounds", layer.get("bbox", config.default_bbox)), + ) + + +def _dead_tile_md5_values(layer): + dead_tile = layer.get("dead_tile") + if not isinstance(dead_tile, dict) or "md5" not in dead_tile: + return () + md5 = dead_tile["md5"] + if isinstance(md5, str): + return (md5,) + try: + return tuple(sorted(md5)) + except TypeError: + return () + + +def document(config, ref): + root = ET.Element(_tag("imagery")) + for layer_name in sorted(config.layers): + layer = config.layers[layer_name] + attrs = {} + if layer.get("overlay"): + attrs["overlay"] = "true" + entry = ET.SubElement(root, _tag("entry"), attrs) + ET.SubElement(entry, _tag("default")).text = "true" + ET.SubElement(entry, _tag("name")).text = layer.get("name", layer_name) + ET.SubElement(entry, _tag("id")).text = "twms-%s" % layer_name + ET.SubElement(entry, _tag("type")).text = "tms" + ET.SubElement(entry, _tag("url")).text = _layer_url(ref, layer_name, layer) + ET.SubElement(entry, _tag("description")).text = layer.get("name", layer_name) + bounds = _layer_bounds(config, layer) + ET.SubElement( + entry, + _tag("bounds"), + { + "min-lon": str(bounds[0]), + "min-lat": str(bounds[1]), + "max-lon": str(bounds[2]), + "max-lat": str(bounds[3]), + }, + ) + ET.SubElement(entry, _tag("valid-georeference")).text = "true" + if "provider_url" in layer: + ET.SubElement(entry, _tag("attribution-url")).text = layer["provider_url"] + for md5 in _dead_tile_md5_values(layer): + ET.SubElement( + entry, + _tag("no-tile-checksum"), + { + "type": "MD5", + "value": md5, + }, + ) + if "max_zoom" in layer: + ET.SubElement(entry, _tag("max-zoom")).text = str(layer["max_zoom"] - 1) + if "min_zoom" in layer: + ET.SubElement(entry, _tag("min-zoom")).text = str(layer["min_zoom"]) + return root + + +def xml(config, ref): + return ET.tostring(document(config, ref), encoding="unicode", xml_declaration=True) diff --git a/twms/overview.py b/twms/overview.py index 5a96c1f..2cc91ba 100644 --- a/twms/overview.py +++ b/twms/overview.py @@ -5,8 +5,22 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -from config import * import projections +from config import * + + +def _layer_bounds(layer): + return layer.get( + "data_bounding_box", + layer.get("bounds", projections.projs[layer["proj"]]["bounds"]), + ) + + +def _layer_extension(layer): + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg").lower().replace("image/", ""), + ).lower().replace("jpeg", "jpg") def html(ref): @@ -20,9 +34,7 @@ def html(ref): resp += wms_name resp += "
" for i in layers: - bbox = layers[i].get( - "data_bounding_box", projections.projs[layers[i]["proj"]]["bounds"] - ) + bbox = _layer_bounds(layers[i]) resp += '" diff --git a/twms/projections.py b/twms/projections.py index d445391..557ccc0 100644 --- a/twms/projections.py +++ b/twms/projections.py @@ -6,6 +6,7 @@ import math + try: import pyproj except ImportError: @@ -14,15 +15,23 @@ class Proj: def __init__(self, pstring): self.pstring = pstring - def transform(self, pr1, pr2, c1, c2): + @staticmethod + def transform(pr1, pr2, c1, c2): if pr1.pstring == pr2.pstring: return c1, c2 else: raise NotImplementedError( - "Pyproj is not installed - can't convert between projectios. Install pyproj please." + "Pyproj is not installed - can't convert between projections. Install twms[proj] please." ) +def _pyproj_transformer(pr1, pr2): + if hasattr(pyproj, "Transformer"): + transformer = pyproj.Transformer.from_proj(pr1, pr2, always_xy=True) + return lambda _pr1, _pr2, c1, c2: transformer.transform(c1, c2) + return pyproj.transform + + projs = { "EPSG:4326": { "proj": pyproj.Proj("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"), @@ -93,20 +102,28 @@ def transform(self, pr1, pr2, c1, c2): "bounds": (-180.0, -90.0, 180.0, 90.0), }, } -proj_alias = {"EPSG:900913": "EPSG:3857", "EPSG:3785": "EPSG:3857"} +proj_alias = { + "CRS:84": "EPSG:4326", + "EPSG:900913": "EPSG:3857", + "EPSG:3785": "EPSG:3857", +} def _c4326t3857(t1, t2, lon, lat): """ Pure python 4326 -> 3857 transform. About 8x faster than pyproj. """ + maxbounds = 6378137 * math.pi + xtile = maxbounds / 180 * lon lat_rad = math.radians(lat) - xtile = lon * 111319.49079327358 - ytile = ( - math.log(math.tan(lat_rad) + (1 / math.cos(lat_rad))) - / math.pi - * 20037508.342789244 - ) + if abs(lat) <= 85.0511287798: + ytile = ( + math.log(math.tan(lat_rad) + (1 / math.cos(lat_rad))) + / math.pi + * maxbounds + ) + else: + ytile = math.copysign(maxbounds, lat) return (xtile, ytile) @@ -268,9 +285,10 @@ def transform(line, srs1, srs2): if (srs1, srs2) in pure_python_transformers: func = pure_python_transformers[(srs1, srs2)] # print("pure") + uses_pyproj = False else: - - func = pyproj.transform + func = None + uses_pyproj = True line = list(line) serial = False if (not isinstance(line[0], tuple)) and (not isinstance(line[0], list)): @@ -284,6 +302,8 @@ def transform(line, srs1, srs2): ans = [] pr1 = projs[srs1]["proj"] pr2 = projs[srs2]["proj"] + if uses_pyproj: + func = _pyproj_transformer(pr1, pr2) for point in line: p = func(pr1, pr2, point[0], point[1]) if serial: diff --git a/twms/reproject.py b/twms/reproject.py index 240ec55..84a27b9 100644 --- a/twms/reproject.py +++ b/twms/reproject.py @@ -5,13 +5,10 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -try: - from PIL import Image -except ImportError: - import Image +import sys import projections -import sys +from PIL import Image def reproject(image, bbox, srs_from, srs_to): diff --git a/twms/server.py b/twms/server.py new file mode 100644 index 0000000..504462f --- /dev/null +++ b/twms/server.py @@ -0,0 +1,147 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# This file is part of twms. + +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms specified in COPYING. + +import re +import sys +import textwrap +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from twms import __version__, twms_main + + +tile_route = re.compile(r"/(.*)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?(.*)") +wms_tile_route = re.compile( + r"/wms/([^/]+)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?" +) +tilejson_route = re.compile(r"/tilejson/(.+)\.json") +josm_imagery_routes = {"/josm/imagery.xml", "/josm/maps.xml", "/maps.xml"} +wmts_capabilities_route = "/wmts/1.0.0/WMTSCapabilities.xml" +wmts_tile_route = re.compile( + r"/wmts/([^/]+)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?" +) + + +def request_url(handler): + scheme = "http" + host = handler.headers.get("Host") + if not host: + host = "%s:%s" % handler.server.server_address[:2] + return "%s://%s/" % (scheme, host) + + +def startup_banner(bind_host, port): + url_host = bind_host + if not url_host or url_host in ("0.0.0.0", "::"): + url_host = "127.0.0.1" + base = "http://%s:%s" % (url_host, port) + return textwrap.dedent( + """\ + TWMS server {version} listening on {bind_host}:{port} + Overview: {base}/ + WMS: {base}/wms?SERVICE=WMS&REQUEST=GetCapabilities + WMTS: {base}/wmts/1.0.0/WMTSCapabilities.xml + JOSM imagery: {base}/josm/maps.xml + Press Ctrl-C to stop + """ + ).format( + version=__version__, + bind_host=bind_host or "0.0.0.0", + port=port, + base=base, + ).rstrip() + + +def _tile_data(match): + ext = match.group(5) or ".jpg" + return { + "request": "GetTile", + "layers": match.group(1), + "format": ext.strip(".").lower(), + "z": match.group(2), + "x": match.group(3), + "y": match.group(4), + } + + +def dispatch(path, ref=None): + parsed = urllib.parse.urlsplit(path) + tilejson_match = tilejson_route.fullmatch(parsed.path) + if tilejson_match: + data = { + "request": "GetTileJSON", + "layers": urllib.parse.unquote(tilejson_match.group(1)), + } + elif parsed.path in josm_imagery_routes: + data = { + "request": "GetJOSMImagery", + } + elif parsed.path == wmts_capabilities_route: + data = { + "request": "GetCapabilities", + "service": "WMTS", + } + else: + match = wms_tile_route.fullmatch(parsed.path) + if match: + data = _tile_data(match) + else: + match = wmts_tile_route.fullmatch(parsed.path) + if match: + data = _tile_data(match) + else: + match = tile_route.fullmatch(parsed.path) + if match: + data = _tile_data(match) + else: + if not parsed.query and parsed.path not in ("", "/", "/wms"): + return 404, "text/plain", "Not Found\n" + data = dict(urllib.parse.parse_qsl(parsed.query)) + data = dict((key.lower(), data[key]) for key in data) + if ref and parsed.path == "/wms": + ref = urllib.parse.urljoin(ref, "wms") + + if ref and "ref" not in data: + data["ref"] = ref + return twms_main(data) + + +class TWMSRequestHandler(BaseHTTPRequestHandler): + server_version = "twms/%s" % __version__ + + def do_GET(self): + status, content_type, content = dispatch(self.path, request_url(self)) + self.send_response(status) + self.send_header("Content-Type", content_type) + if "text/" in content_type or "xml" in content_type: + self.send_header("Cache-Control", "no-cache, no-store, must-revalidate") + self.send_header("Pragma", "no-cache") + self.send_header("Expires", "0") + self.end_headers() + if isinstance(content, str): + content = content.encode("utf-8") + self.wfile.write(content) + + def log_message(self, format, *args): + pass + + +def main(): + host = "" + try: + port = int(sys.argv[1]) + except IndexError: + port = 8080 + + server = ThreadingHTTPServer((host, port), TWMSRequestHandler) + print(startup_banner(host, port)) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/twms/sketch.py b/twms/sketch.py index 8a1c143..e962609 100644 --- a/twms/sketch.py +++ b/twms/sketch.py @@ -7,6 +7,7 @@ from bbox import * + string = "abcdefghijklmnopqrstuvwxyz012345ABCDEFGHIJKLMNOPQRSTUVWXYZ6789{}" diff --git a/twms/tilejson.py b/twms/tilejson.py new file mode 100644 index 0000000..c9b3b1e --- /dev/null +++ b/twms/tilejson.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# This file is part of twms. + +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms specified in COPYING. + +import json + +import bbox + + +def _layer_names(config, layers): + names = [name for name in layers.split(",") if name] + if not names: + names = [name for name in config.default_layers.split(",") if name] + if not names: + raise KeyError("TileJSON needs an existing layer name") + for name in names: + if name not in config.layers: + raise KeyError("Unknown layer: %s" % name) + return names + + +def _layer_bounds(config, layer): + return layer.get( + "data_bounding_box", + layer.get("bounds", layer.get("bbox", config.default_bbox)), + ) + + +def _tile_extension(config, layer_names, format_name): + if format_name: + return format_name.lower().replace("image/", "").replace("jpeg", "jpg") + if len(layer_names) == 1: + layer = config.layers[layer_names[0]] + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg") + .lower() + .replace("image/", "") + .replace("jpeg", "jpg"), + ) + return config.default_format.lower().replace("image/", "").replace("jpeg", "jpg") + + +def document(config, layers, ref, format_name=""): + layer_names = _layer_names(config, layers) + layer_items = [config.layers[name] for name in layer_names] + bounds = _layer_bounds(config, layer_items[0]) + for layer in layer_items[1:]: + bounds = bbox.add(bounds, _layer_bounds(config, layer)) + + minzoom = max(layer.get("min_zoom", 0) for layer in layer_items) + maxzoom = min(layer.get("max_zoom", config.default_max_zoom) for layer in layer_items) + center = [ + (bounds[0] + bounds[2]) / 2.0, + (bounds[1] + bounds[3]) / 2.0, + minzoom, + ] + ext = _tile_extension(config, layer_names, format_name) + tile_url = "%s%s/{z}/{x}/{y}.%s" % (ref, ",".join(layer_names), ext) + + return { + "tilejson": "3.0.0", + "name": ", ".join(layer["name"] for layer in layer_items), + "scheme": "xyz", + "tiles": [tile_url], + "bounds": list(bounds), + "center": center, + "minzoom": minzoom, + "maxzoom": maxzoom, + } + + +def dumps(config, layers, ref, format_name=""): + return json.dumps( + document(config, layers, ref, format_name=format_name), + sort_keys=True, + ) diff --git a/twms/twms.conf b/twms/twms.conf index ccbc766..812217d 100644 --- a/twms/twms.conf +++ b/twms/twms.conf @@ -15,6 +15,9 @@ tiles_cache = "/var/cache/twms/tiles/" # where to put cache install_path = "/usr/share/twms/" # where to look for broken tiles and other stuff gpx_cache = "/var/cache/twms/traces/" # where to store cached OSM GPX files deadline = 45 # number of seconds that are given to make up image +upstream_timeout = 30 # upstream HTTP timeout in seconds; layers may override with "timeout" +upstream_retries = 1 # total upstream HTTP attempts; layers may override with "upstream_retries" +upstream_retry_delay = 0 # seconds to wait between retry attempts; layers may override default_max_zoom = 18 # can be overridden per layer geometry_color = { # default color for overlayed vectors "LINESTRING": "#ff0000", @@ -54,19 +57,6 @@ cache_tile_responses = { ## Available layers. layers = {\ -"yhsat": { \ - "name": "Yahoo Satellite", - "prefix": "yhsat", # tile directory - "ext": "jpg", # tile images extension - "scalable": False, # could zN tile be constructed of four z(N+1) tiles - "fetch": fetchers.Tile, # function that fetches given tile. should return None if tile wasn't fetched - "remote_url": "http://aerial.maps.yimg.com/ximg?v=1.8&t=a&s=256&r=1&x=%s&y=%s&z=%s", - "transform_tile_number": lambda z,x,y: (x,((2**(z-1)/2)-1)-y,z), - "dead_tile": install_path + "yahoo_nxt.jpg", - "min_zoom": 2, - "max_zoom": 18, - "proj": "EPSG:3857", -},\ "yasat": { \ "name": "Yandex Satellite", "prefix": "yasat", # tile directory @@ -84,7 +74,7 @@ layers = {\ "ext": "png", # tile images extension "scalable": False, # could zN tile be constructed of four z(N+1) tiles "fetch": fetchers.Tile, # function that fetches given tile. should return None if tile wasn't fetched - "remote_url": "http://c.tile.openstreetmap.org/%s/%s/%s.png", + "remote_url": "https://tile.openstreetmap.org/%s/%s/%s.png", "transform_tile_number": lambda z,x,y: (z-1,x,y), "proj": "EPSG:3857", "empty_color": "#F1EEE8", @@ -102,25 +92,14 @@ layers = {\ "data_bounding_box": (23.16722,51.25930,32.82244,56.18162), },\ "landsat": { \ - "name": "Landsat from onearth.jpl.nasa.gov", + "name": "Landsat from gibs.earthdata.nasa.gov", "prefix": "landsat", # tile directory "ext": "jpg", # tile images extension "scalable": False, # could zN tile be constructed of four z(N+1) tiles - "fetch": fetchers.WMS, # function that fetches given tile. should return None if tile wasn't fetched - "remote_url": "http://onearth.jpl.nasa.gov/wms.cgi?request=GetMap&layers=global_mosaic&styles=&format=image/jpeg&", # string without srs, height, width and bbox - "max_zoom": 14, - "proj": "EPSG:4326", - "wms_proj": "EPSG:4326", # what projection to ask from wms -},\ -"latlonsat": { \ - "name": "Imagery from latlon.org", - "prefix": "latlonsat", # tile directory - "ext": "jpg", # tile images extension - "scalable": False, # could zN tile be constructed of four z(N+1) tiles - "fetch": fetchers.WMS, # function that fetches given tile. should return None if tile wasn't fetched - "remote_url": "http://dev.latlon.org/cgi-bin/ms?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=sat,plane&", # string without srs, height, width and bbox - "max_zoom": 19, + "fetch": fetchers.Tile, # function that fetches given tile. should return None if tile wasn't fetched + "transform_tile_number": lambda z,x,y: (z-1,x,y), + "remote_url": "https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/Landsat_WELD_CorrectedReflectance_TrueColor_Global_Annual/default/2000-12-01/GoogleMapsCompatible_Level12/%s/%s/%s.jpg", + "max_zoom": 12, "proj": "EPSG:3857", - "wms_proj": "EPSG:3857", # what projection to ask from wms },\ } diff --git a/twms/twms.py b/twms/twms.py index 1425203..d8adf16 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -6,57 +6,38 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -from __future__ import print_function, division - -try: - from PIL import Image, ImageOps, ImageColor -except ImportError: - import Image, ImageOps, ImageColor - -import imp -import os +import datetime import math +import os import sys +import time import urllib +from collections import OrderedDict from io import BytesIO -import time -import datetime sys.path.append(os.path.join(os.path.dirname(__file__))) -config_path = "/etc/twms/twms.conf" -if os.path.exists(config_path): - try: - config = imp.load_source("twms.config", config_path) - except: - config = imp.load_source("config", config_path) -else: - try: - config_path = os.path.join(os.path.dirname(__file__), "twms.conf") - config = imp.load_source("twms.config", config_path) - except: - config_path = os.path.join(os.path.realpath(sys.path[0]), "twms.conf") - config = imp.load_source( - "config", os.path.join(os.path.realpath(sys.path[0]), "twms.conf") - ) - sys.stderr.write( - "Configuration file not found, using defaults from %s\n" % config_path - ) - sys.stderr.flush() +from twms.config_loader import load_default_config -import correctify -import capabilities -import fetchers +config = load_default_config() -# import config import bbox -import bbox as bbox_utils -import projections +import capabilities +import correctify import drawing +import fetchers +import josm import overview +import projections +import tilejson +import wmts +from bbox import expand_to_point, zoom_for_bbox from gpxparse import GPXParser +from PIL import Image, ImageColor, ImageOps from reproject import reproject +from twms.image_compat import resampling_lanczos + try: import psyco @@ -69,8 +50,7 @@ OK = 200 ERROR = 500 -cached_objs = {} # a dict. (layer, z, x, y): PIL image -cached_hist_list = [] +cached_objs = OrderedDict() # (layer, z, x, y): PIL image, least-recent first formats = { "image/gif": "GIF", @@ -78,29 +58,84 @@ "image/jpg": "JPEG", "image/png": "PNG", "image/bmp": "BMP", + "image/webp": "WEBP", } mimetypes = dict(zip(formats.values(), formats.keys())) +def _ram_cache_key(layer, z, x, y): + return (layer["prefix"], z, x, y) + + +def _ram_cache_get(key): + if key not in cached_objs: + return None + cached_objs.move_to_end(key) + return cached_objs[key] + + +def _ram_cache_put(key, image): + limit = int(getattr(config, "max_ram_cached_tiles", 1024)) + if limit <= 0: + return + cached_objs[key] = image + cached_objs.move_to_end(key) + while len(cached_objs) > limit: + cached_objs.popitem(last=False) + + +def _response_cache_entry( + response_cache, srs, layers, filt, width, height, force, image_format +): + key_parts = (srs, tuple(layers), filt, width, height, force) + for format_key in (image_format, mimetypes.get(image_format)): + key = key_parts + (format_key,) + if key in response_cache: + return response_cache[key] + return None + + +def _layer_bounds(layer): + return layer.get( + "data_bounding_box", + layer.get("bounds", layer.get("bbox", config.default_bbox)), + ) + + +def _layer_extension(layer): + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg").lower().replace("image/", ""), + ).lower().replace("jpeg", "jpg") + + def twms_main(data): """ Do main TWMS work. data - dictionary of params. returns (error_code, content_type, resp) """ + data = dict((key.lower(), data[key]) for key in data) + # import the filter here due to a circular dependency + # TODO: break the loop + import filter start_time = datetime.datetime.now() content_type = "text/html" resp = "" - srs = data.get("srs", "EPSG:4326") + srs = data.get("crs", data.get("srs", "EPSG:4326")) gpx = data.get("gpx", "").split(",") if gpx == [""]: gpx = [] wkt = data.get("wkt", "") trackblend = float(data.get("trackblend", "0.5")) - color = data.get("color", data.get("colour", "")).split(",") + colors = [ + value + for value in data.get("color", data.get("colour", "")).split(",") + if value + ] track = False tracks = [] if len(gpx) == 0: @@ -125,23 +160,50 @@ def twms_main(data): tracks.append(track) req_type = data.get("request", "GetMap") + req_type_lower = req_type.lower() version = data.get("version", "1.1.1") ref = data.get("ref", config.service_url) - if req_type == "GetCapabilities": + if data.get("service", "").lower() == "wmts" and req_type_lower == "getcapabilities": + return (OK, "text/xml", wmts.capabilities(config, ref)) + if req_type_lower == "getwmtscapabilities": + return (OK, "text/xml", wmts.capabilities(config, ref)) + if data.get("service", "").lower() == "wmts" and req_type_lower == "gettile": + if "layers" not in data and "layer" in data: + data["layers"] = data["layer"] + if "z" not in data and "tilematrix" in data: + data["z"] = data["tilematrix"] + if "x" not in data and "tilecol" in data: + data["x"] = data["tilecol"] + if "y" not in data and "tilerow" in data: + data["y"] = data["tilerow"] + if req_type_lower == "getcapabilities": content_type, resp = capabilities.get(version, ref) return (OK, content_type, resp) + if req_type_lower in ("gettilejson", "tilejson"): + try: + resp = tilejson.dumps( + config, + data.get("layers", ""), + ref, + format_name=data.get("format", ""), + ) + return (OK, "application/json", resp) + except KeyError as exc: + return (400, "text/plain", str(exc)) + if req_type_lower in ("getjosmimagery", "josmimagery", "getjosmmaps"): + return (OK, "text/xml", josm.xml(config, ref)) layer = data.get("layers", config.default_layers).split(",") if ("layers" in data) and not layer[0]: layer = ["transparent"] - if req_type == "GetCorrections": + if req_type_lower == "getcorrections": points = data.get("points", data.get("POINTS", "")).split("=") resp = "" points = [a.split(",") for a in points] points = [(float(a[0]), float(a[1])) for a in points] - req.content_type = "text/plain" + content_type = "text/plain" for lay in layer: for point in points: resp += "%s,%s;" % tuple(correctify.rectify(config.layers[lay], point)) @@ -173,29 +235,28 @@ def twms_main(data): width = 0 height = 0 resp_cache_path, resp_ext = "", "" - if req_type == "GetTile": + if req_type_lower == "gettile": width = 256 height = 256 height = int(data.get("height", height)) width = int(data.get("width", width)) - srs = data.get("srs", "EPSG:3857") + srs = data.get("crs", data.get("srs", "EPSG:3857")) x = int(data.get("x", 0)) y = int(data.get("y", 0)) z = int(data.get("z", 1)) + 1 if "cache_tile_responses" in dir(config) and not wkt and (len(gpx) == 0): - if ( + response_cache = _response_cache_entry( + config.cache_tile_responses, srs, - tuple(layer), + layer, filt, width, height, force, format, - ) in config.cache_tile_responses: - - resp_cache_path, resp_ext = config.cache_tile_responses[ - (srs, tuple(layer), filt, width, height, force, format) - ] + ) + if response_cache: + resp_cache_path, resp_ext = response_cache resp_cache_path = resp_cache_path + "/%s/%s/%s.%s" % ( z - 1, x, @@ -203,13 +264,14 @@ def twms_main(data): resp_ext, ) if os.path.exists(resp_cache_path): - return (OK, content_type, open(resp_cache_path, "r").read()) + with open(resp_cache_path, "rb") as cached_response: + return (OK, content_type, cached_response.read()) if len(layer) == 1: if layer[0] in config.layers: if ( config.layers[layer[0]]["proj"] == srs - and width is 256 - and height is 256 + and width == 256 + and height == 256 and not filt and not force and not correctify.has_corrections(config.layers[layer[0]]) @@ -217,14 +279,14 @@ def twms_main(data): local = ( config.tiles_cache + config.layers[layer[0]]["prefix"] - + "/z%s/%s/x%s/%s/y%s." % (z, x / 1024, x, y / 1024, y) + + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) ) - ext = config.layers[layer]["ext"] + ext = _layer_extension(config.layers[layer[0]]) adds = ["", "ups."] for add in adds: if os.path.exists(local + add + ext): - tile_file = open(local + add + ext, "r") - resp = tile_file.read() + with open(local + add + ext, "rb") as tile_file: + resp = tile_file.read() return (OK, content_type, resp) req_bbox = projections.from4326(projections.bbox_by_tile(z, x, y, srs), srs) @@ -275,29 +337,29 @@ def twms_main(data): if "empty_color" in config.layers[ll]: ec = ImageColor.getcolor(config.layers[ll]["empty_color"], "RGBA") - sec = set(ec) + sec = {ec} if "empty_color_delta" in config.layers[ll]: delta = config.layers[ll]["empty_color_delta"] - for tr in range(-delta, delta): - for tg in range(-delta, delta): - for tb in range(-delta, delta): + for tr in range(-delta, delta + 1): + for tg in range(-delta, delta + 1): + for tb in range(-delta, delta + 1): if ( (ec[0] + tr) >= 0 and (ec[0] + tr) < 256 - and (ec[1] + tr) >= 0 - and (ec[1] + tr) < 256 - and (ec[2] + tr) >= 0 - and (ec[2] + tr) < 256 + and (ec[1] + tg) >= 0 + and (ec[1] + tg) < 256 + and (ec[2] + tb) >= 0 + and (ec[2] + tb) < 256 ): sec.add((ec[0] + tr, ec[1] + tg, ec[2] + tb, ec[3])) i2l = im2.load() - for x in range(0, im2.size[0]): - for y in range(0, im2.size[1]): - t = i2l[x, y] + for px in range(0, im2.size[0]): + for py in range(0, im2.size[1]): + t = i2l[px, py] if t in sec: - i2l[x, y] = (t[0], t[1], t[2], 0) + i2l[px, py] = (t[0], t[1], t[2], 0) if not im2.size == result_img.size: - im2 = im2.resize(result_img.size, Image.ANTIALIAS) + im2 = im2.resize(result_img.size, resampling_lanczos(Image)) im2 = Image.composite(im2, result_img, im2.split()[3]) # imgs/(imgs+1.)) if "noblend" in force: @@ -306,7 +368,7 @@ def twms_main(data): result_img = Image.blend(im2, result_img, 0.5) imgs += 1.0 - ##Applying filters + # Applying filters result_img = filter.raster(result_img, filt, req_bbox, srs) # print(wkt, file=sys.stderr) @@ -317,15 +379,15 @@ def twms_main(data): result_img, req_bbox, srs, - color if len(color) > 0 else None, + colors[0] if colors else None, trackblend, ) if len(gpx) > 0: last_color = None - c = iter(color) + c = iter(colors) for track in tracks: try: - last_color = c.next() + last_color = next(c) except StopIteration: pass result_img = drawing.gpx( @@ -360,7 +422,7 @@ def twms_main(data): quality=config.output_quality, progressive=config.output_progressive, ) - else: ## workaround for GIF + else: # workaround for GIF result_img = result_img.convert("RGB") result_img.save( image_content, @@ -376,9 +438,8 @@ def twms_main(data): except OSError: pass try: - a = open(resp_cache_path, "w") - a.write(resp) - a.close() + with open(resp_cache_path, "wb") as cached_response: + cached_response.write(resp) except (OSError, IOError): print( "error saving response answer to file %s." % (resp_cache_path), @@ -401,21 +462,22 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal return None if not bbox.bbox_is_in( projections.bbox_by_tile(z, x, y, layer["proj"]), - layer.get("data_bounding_box", config.default_bbox), + _layer_bounds(layer), fully=False, ): return None - global cached_objs, cached_hist_list + global cached_objs if "prefix" in layer: - if (layer["prefix"], z, x, y) in cached_objs: - return cached_objs[(layer["prefix"], z, x, y)] + cached = _ram_cache_get(_ram_cache_key(layer, z, x, y)) + if cached is not None: + return cached if layer.get("cached", True): local = ( config.tiles_cache + layer["prefix"] - + "/z%s/%s/x%s/%s/y%s." % (z, x / 1024, x, y / 1024, y) + + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) ) - ext = layer["ext"] + ext = _layer_extension(layer) if "cache_ttl" in layer: for ex in [ext, "dsc." + ext, "ups." + ext, "tne"]: f = local + ex @@ -472,7 +534,7 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal im.paste(im2, (256, 0)) im.paste(im3, (0, 256)) im.paste(im4, (256, 256)) - im = im.resize((256, 256), Image.ANTIALIAS) + im = im.resize((256, 256), resampling_lanczos(Image)) if layer.get("cached", True): try: im.save(local + "ups." + ext) @@ -524,8 +586,12 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal def getimg(bbox, request_proj, size, layer, start_time, force): + # import the filter here due to a circular dependency + # TODO: break the loop + import filter + orig_bbox = bbox - ## Making 4-corner maximal bbox + # Making 4-corner maximal bbox bbox_p = projections.from4326(bbox, request_proj) bbox_p = projections.to4326( (bbox_p[2], bbox_p[1], bbox_p[0], bbox_p[3]), request_proj @@ -542,9 +608,7 @@ def getimg(bbox, request_proj, size, layer, start_time, force): for point in bbox_4: bb4.append(correctify.rectify(layer, point)) bbox_4 = bb4 - bbox = bbox_utils.expand_to_point(bbox, bbox_4) - # print(bbox) - # print(orig_bbox) + bbox = expand_to_point(bbox, bbox_4) global cached_objs H, W = size @@ -552,7 +616,7 @@ def getimg(bbox, request_proj, size, layer, start_time, force): max_zoom = layer.get("max_zoom", config.default_max_zoom) min_zoom = layer.get("min_zoom", 1) - zoom = bbox_utils.zoom_for_bbox( + zoom = zoom_for_bbox( bbox, size, layer, min_zoom, max_zoom, (config.max_height, config.max_width) ) lo1, la1, lo2, la2 = bbox @@ -583,16 +647,12 @@ def getimg(bbox, request_proj, size, layer, start_time, force): im1 = tile_image(layer, zoom, x, y, start_time, real=True) if im1: if "prefix" in layer: - if (layer["prefix"], zoom, x, y) not in cached_objs: + cache_key = _ram_cache_key(layer, zoom, x, y) + if cache_key not in cached_objs: if im1.is_ok: - cached_objs[(layer["prefix"], zoom, x, y)] = im1 - cached_hist_list.append((layer["prefix"], zoom, x, y)) + _ram_cache_put(cache_key, im1) # print((layer["prefix"], zoom, x, y), cached_objs[(layer["prefix"], zoom, x, y)], file=sys.stderr) # sys.stderr.flush() - if len(cached_objs) >= config.max_ram_cached_tiles: - del cached_objs[cached_hist_list.pop(0)] - # print("Removed tile from cache", file=sys.stderr) - # sys.stderr.flush() else: ec = ImageColor.getcolor( layer.get("empty_color", config.default_background), "RGBA" @@ -604,7 +664,7 @@ def getimg(bbox, request_proj, size, layer, start_time, force): if "filter" in layer: out = filter.raster(out, layer["filter"], orig_bbox, request_proj) - ## TODO: Here's a room for improvement. we could drop this crop in case user doesn't need it. + # TODO: Here's a room for improvement. we could drop this crop in case user doesn't need it. out = out.crop(bbox_im) if "noresize" not in force: if (H == W) and (H == 0): @@ -631,6 +691,6 @@ def getimg(bbox, request_proj, size, layer, start_time, force): out = out.transform((W, H), Image.QUAD, quad, Image.BICUBIC) elif (W != out.size[0]) or (H != out.size[1]): "just resize" - out = out.resize((W, H), Image.ANTIALIAS) + out = out.resize((W, H), resampling_lanczos(Image)) # out = reproject(out, bbox, layer["proj"], request_proj) return out diff --git a/twms/version.py b/twms/version.py new file mode 100644 index 0000000..a798838 --- /dev/null +++ b/twms/version.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +__version__ = "0.07z" + +# Python packaging metadata must be PEP 440. Keep the public TWMS keyboard +# version above; this adapter exists only for wheel/sdist tooling. +__packaging_version__ = "0.7+z" diff --git a/twms/wmts.py b/twms/wmts.py new file mode 100644 index 0000000..c4b4822 --- /dev/null +++ b/twms/wmts.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +# This file is part of twms. + +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms specified in COPYING. + +import xml.etree.ElementTree as ET + +import projections + + +WMTS = "http://www.opengis.net/wmts/1.0" +OWS = "http://www.opengis.net/ows/1.1" +XLINK = "http://www.w3.org/1999/xlink" +XSI = "http://www.w3.org/2001/XMLSchema-instance" + + +def _tag(namespace, name): + return "{%s}%s" % (namespace, name) + + +def _extension(layer): + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg").lower().replace("image/", ""), + ).lower().replace("jpeg", "jpg") + + +def _mime_type(layer): + if "mimetype" in layer: + return layer["mimetype"] + ext = _extension(layer) + if ext == "jpg": + return "image/jpeg" + return "image/%s" % ext + + +def _layer_bounds(config, layer): + return layer.get( + "data_bounding_box", + layer.get("bounds", layer.get("bbox", config.default_bbox)), + ) + + +def _tile_url(ref, layer_name, layer): + return "%swmts/%s/{TileMatrix}/{TileCol}/{TileRow}.%s" % ( + ref, + layer_name, + _extension(layer), + ) + + +def _supported_crs(proj): + epsg = proj.split(":")[-1] + if epsg.isdigit(): + return "urn:ogc:def:crs:EPSG::%s" % epsg + return proj + + +def _projected_bounds(proj): + bounds = projections.projs[projections.proj_alias.get(proj, proj)]["bounds"] + return projections.from4326(bounds, proj) + + +def _scale_denominator(proj, z): + projected = _projected_bounds(proj) + matrix_width = 2 ** z + resolution = (projected[2] - projected[0]) / (256 * matrix_width) + return resolution / 0.00028 + + +def _max_zoom_for_proj(config, proj): + zooms = [ + layer.get("max_zoom", config.default_max_zoom) + for layer in config.layers.values() + if layer.get("proj", "EPSG:3857") == proj + ] + return max(zooms or [config.default_max_zoom]) + + +def _add_operation(parent, name, href): + operation = ET.SubElement(parent, _tag(OWS, "Operation"), {"name": name}) + dcp = ET.SubElement(operation, _tag(OWS, "DCP")) + http = ET.SubElement(dcp, _tag(OWS, "HTTP")) + ET.SubElement( + http, + _tag(OWS, "Get"), + {_tag(XLINK, "href"): href}, + ) + + +def _add_layer(parent, config, layer_name, layer, ref): + layer_element = ET.SubElement(parent, _tag(WMTS, "Layer")) + ET.SubElement(layer_element, _tag(OWS, "Title")).text = layer["name"] + ET.SubElement(layer_element, _tag(OWS, "Identifier")).text = layer_name + + wgs84_bounds = ET.SubElement(layer_element, _tag(OWS, "WGS84BoundingBox")) + bounds = _layer_bounds(config, layer) + ET.SubElement(wgs84_bounds, _tag(OWS, "LowerCorner")).text = "%s %s" % ( + bounds[0], + bounds[1], + ) + ET.SubElement(wgs84_bounds, _tag(OWS, "UpperCorner")).text = "%s %s" % ( + bounds[2], + bounds[3], + ) + + style = ET.SubElement(layer_element, _tag(WMTS, "Style"), {"isDefault": "true"}) + ET.SubElement(style, _tag(OWS, "Identifier")).text = "default" + ET.SubElement(layer_element, _tag(WMTS, "Format")).text = _mime_type(layer) + + link = ET.SubElement(layer_element, _tag(WMTS, "TileMatrixSetLink")) + ET.SubElement(link, _tag(WMTS, "TileMatrixSet")).text = layer.get("proj", "EPSG:3857") + ET.SubElement( + layer_element, + _tag(WMTS, "ResourceURL"), + { + "format": _mime_type(layer), + "resourceType": "tile", + "template": _tile_url(ref, layer_name, layer), + }, + ) + + +def _add_tile_matrix_set(parent, config, proj): + projected = _projected_bounds(proj) + tile_matrix_set = ET.SubElement(parent, _tag(WMTS, "TileMatrixSet")) + ET.SubElement(tile_matrix_set, _tag(OWS, "Identifier")).text = proj + ET.SubElement(tile_matrix_set, _tag(OWS, "SupportedCRS")).text = _supported_crs(proj) + + for z in range(_max_zoom_for_proj(config, proj) + 1): + matrix_size = 2 ** z + tile_matrix = ET.SubElement(tile_matrix_set, _tag(WMTS, "TileMatrix")) + ET.SubElement(tile_matrix, _tag(OWS, "Identifier")).text = str(z) + ET.SubElement(tile_matrix, _tag(WMTS, "ScaleDenominator")).text = str( + _scale_denominator(proj, z) + ) + ET.SubElement(tile_matrix, _tag(WMTS, "TopLeftCorner")).text = "%s %s" % ( + projected[0], + projected[3], + ) + ET.SubElement(tile_matrix, _tag(WMTS, "TileWidth")).text = "256" + ET.SubElement(tile_matrix, _tag(WMTS, "TileHeight")).text = "256" + ET.SubElement(tile_matrix, _tag(WMTS, "MatrixWidth")).text = str(matrix_size) + ET.SubElement(tile_matrix, _tag(WMTS, "MatrixHeight")).text = str(matrix_size) + + +def capabilities(config, ref): + ET.register_namespace("", WMTS) + ET.register_namespace("ows", OWS) + ET.register_namespace("xlink", XLINK) + ET.register_namespace("xsi", XSI) + + root = ET.Element( + _tag(WMTS, "Capabilities"), + { + "version": "1.0.0", + _tag(XSI, "schemaLocation"): ( + "http://www.opengis.net/wmts/1.0 " + "http://schemas.opengis.net/wmts/1.0/wmtsGetCapabilities_response.xsd" + ), + }, + ) + service = ET.SubElement(root, _tag(OWS, "ServiceIdentification")) + ET.SubElement(service, _tag(OWS, "Title")).text = config.wms_name + ET.SubElement(service, _tag(OWS, "ServiceType")).text = "OGC WMTS" + ET.SubElement(service, _tag(OWS, "ServiceTypeVersion")).text = "1.0.0" + + operations = ET.SubElement(root, _tag(OWS, "OperationsMetadata")) + _add_operation( + operations, + "GetCapabilities", + "%swmts/1.0.0/WMTSCapabilities.xml" % ref, + ) + _add_operation(operations, "GetTile", ref) + + contents = ET.SubElement(root, _tag(WMTS, "Contents")) + projections_in_use = set() + for layer_name in sorted(config.layers.keys()): + layer = config.layers[layer_name] + proj = layer.get("proj", "EPSG:3857") + if proj not in projections.projs: + continue + projections_in_use.add(proj) + _add_layer(contents, config, layer_name, layer, ref) + + for proj in sorted(projections_in_use): + _add_tile_matrix_set(contents, config, proj) + + ET.SubElement( + root, + _tag(WMTS, "ServiceMetadataURL"), + {_tag(XLINK, "href"): "%swmts/1.0.0/WMTSCapabilities.xml" % ref}, + ) + + return '\n' + ET.tostring( + root, + encoding="unicode", + ) diff --git a/yahoo_nxt.jpg b/yahoo_nxt.jpg deleted file mode 100644 index 9e1970f..0000000 Binary files a/yahoo_nxt.jpg and /dev/null differ

' ) - resp += layers[i]["name"] + if "provider_url" in layers[i]: + resp += 'Bounding box: " + str(bbox) @@ -47,7 +66,7 @@ def html(ref): + "" + i + "/!/!/!." - + layers[i].get("ext", "jpg") + + _layer_extension(layers[i]) + "
" ) resp += "