From 26cf6e7d83d1039df468d2aa0effa3f26c3d2247 Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Mon, 1 Mar 2021 17:02:07 +0100 Subject: [PATCH 01/51] Move as much as possible metadata from setup.py to setup.cfg --- setup.cfg | 36 ++++++++++++++++++++++++++++++++++++ setup.py | 32 -------------------------------- 2 files changed, 36 insertions(+), 32 deletions(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..8aeeafb --- /dev/null +++ b/setup.cfg @@ -0,0 +1,36 @@ +[metadata] +name = twms +version = 0.07z +author = Darafei Praliaskouski +author_email = me@komzpa.net +url = https://github.com/komzpa/twms +description = tiny web map service +long_description = file: README.md +license = Public Domain or ISC +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Web Environment + Intended Audience :: Developers + Intended Audience :: End Users/Desktop + License :: Public Domain + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 3.6 + Topic :: Internet :: WWW/HTTP + Topic :: Scientific/Engineering :: GIS + +[options] +python_requires = >= 3.6 +packages = find: +install_requires = + Pillow + web.py +include_package_data = True + +[options.extras_require] +proj = pyproj +cairo = pycairo + +[options.entry_points] +console_scripts = + twms = twms.daemon:main diff --git a/setup.py b/setup.py index 476becd..94a0037 100755 --- a/setup.py +++ b/setup.py @@ -55,42 +55,10 @@ def initialize_options(self): setup( name = __name__, - version = "0.06y", - author = 'Darafei Praliaskoiski', - author_email = 'me@komzpa.net', - url = 'https://github.com/komzpa/twms', - description = 'tiny web map service', - long_description = read('README.md'), - license = 'Public Domain or ISC', - packages = find_packages(), - install_requires = ['Pillow', 'web.py'], - extras_require = { - 'proj': ['pyproj'], - 'cairo': ['pycairo'], - }, - classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Web Environment', - 'Intended Audience :: Developers', - 'Intended Audience :: End Users/Desktop', - 'License :: Public Domain', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.6', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Scientific/Engineering :: GIS', - ], - include_package_data = True, data_files = [ (os.path.join('share', 'doc', __name__), ['COPYING']), (os.path.join('share', 'doc', __name__), glob('*.md')), (os.path.join('share', __name__), glob('*.jpg')), (os.path.join('share', __name__, 'tools'), glob(os.path.join('tools', '*.py'))) ] + man_files('*.1') + config_files(), - entry_points = { - 'console_scripts': [ - 'twms = twms.daemon:main' - ] - } ) From aae666fa6561cd26bda8e0b76d42e57d1362144f Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Mon, 1 Mar 2021 17:04:48 +0100 Subject: [PATCH 02/51] Reformat imports with isort --- index.py | 3 ++- setup.cfg | 38 +++++++++++++++++++++++++++++++++++ setup.py | 5 ++++- tools/compile_correction.py | 2 ++ tools/decompile_correction.py | 1 + twms/bbox.py | 1 + twms/canvas.py | 14 +++++-------- twms/correctify.py | 3 ++- twms/daemon.py | 6 ++++-- twms/drawing.py | 15 ++++++-------- twms/fetchers.py | 16 +++++---------- twms/filter.py | 8 ++++---- twms/gpxparse.py | 8 ++++++-- twms/overview.py | 2 +- twms/projections.py | 1 + twms/reproject.py | 7 ++----- twms/sketch.py | 1 + twms/twms.py | 26 ++++++++++-------------- 18 files changed, 96 insertions(+), 61 deletions(-) diff --git a/index.py b/index.py index d3321c0..f2b79d6 100644 --- a/index.py +++ b/index.py @@ -10,8 +10,9 @@ if __name__ != "__main__": try: - from mod_python import apache, util import datetime + + from mod_python import apache, util except ImportError: pass diff --git a/setup.cfg b/setup.cfg index 8aeeafb..49c5e6a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,3 +34,41 @@ cairo = pycairo [options.entry_points] console_scripts = twms = twms.daemon:main + +[flake8] +doctests = yes +max-line-length = 130 +exclude = .git,build,__pycache__,setup.py +ignore = E121,E123,E126,E133,E226,E241,E242,E704,E501,E301,E261,E127,E128,W391,W503,W504 +# https://pep8.readthedocs.org/en/latest/intro.html#error-codes +# These are ignored by default: +# - E121: continuation line under-indented for hanging indent +# - E123: closing bracket does not match indentation of opening bracket's line +# - E126: continuation line over-indented for hanging indent +# - E133: closing bracket does not match visual indentation +# - E226: missing whitespace around arithmetic operator +# - E241: multiple spaces after ',' +# - E242: tab after ',' +# - E704: multiple statements on one line (def) +# These were added because PEP-8 allows exceptions, but pep8 doesn't: +# - E501: line too long +# - E301: expected 1 blank line, found 0 +# - E261: at least two spaces before inline comment +# - E127: continuation line over-indented for visual indent +# - E128: continuation line under-indented for visual indent +# - W391: blank line at end of file +# These were added because PEP-8 is wrong sometimes +# - W503: line break before binary operator +# - W504: line break after binary operator + +[isort] +# from X import ( +# a, +# b, +# ) +multi_line_output = 3 +include_trailing_comma = true +lines_after_imports = 2 +line_length = 130 +reverse_relative = true +default_section = THIRDPARTY diff --git a/setup.py b/setup.py index 94a0037..b86b7ce 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,9 @@ import os import platform from glob import glob as abs_glob -from setuptools import setup, find_packages + +from setuptools import find_packages, setup + __platform__ = platform.system() is_windows = __platform__ in ['Windows'] @@ -42,6 +44,7 @@ def config_files(): # monkey patch setuptools to use distutils owner/group functionality from setuptools.command import sdist + sdist_org = sdist.sdist diff --git a/tools/compile_correction.py b/tools/compile_correction.py index 506571c..b7013de 100644 --- a/tools/compile_correction.py +++ b/tools/compile_correction.py @@ -4,8 +4,10 @@ import os import sys + from lxml import etree + tiles_cache = "/var/www/latlon/wms/cache/" layers = ["irs", "yhsat", "DGsat", "yasat", "SAT"] default_user = "Komzpa" diff --git a/tools/decompile_correction.py b/tools/decompile_correction.py index 6186ba5..1e44988 100644 --- a/tools/decompile_correction.py +++ b/tools/decompile_correction.py @@ -4,6 +4,7 @@ import sys + tiles_cache = "/var/www/latlon/wms/cache/" layers = ["irs", "yhsat", "DGsat", "yasat"] diff --git a/twms/bbox.py b/twms/bbox.py index ccf0c30..6a54740 100644 --- a/twms/bbox.py +++ b/twms/bbox.py @@ -6,6 +6,7 @@ # and/or modify it under the terms specified in COPYING. import sys + import projections diff --git a/twms/canvas.py b/twms/canvas.py index 5fa622b..5e01726 100644 --- a/twms/canvas.py +++ b/twms/canvas.py @@ -12,18 +12,14 @@ ## -import projections - -try: - from PIL import Image, ImageFilter -except ImportError: - import Image, ImageFilter - -import urllib -from io import BytesIO import datetime import sys import threading +import urllib +from io import BytesIO + +import projections +from PIL import Image, ImageFilter def debug(st): diff --git a/twms/correctify.py b/twms/correctify.py index 6d17736..17ebbb6 100644 --- a/twms/correctify.py +++ b/twms/correctify.py @@ -5,9 +5,10 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -import projections import os + import config +import projections distance = lambda z, x, y, g: ((z - y) ** 2 + (x - g) ** 2) ** (0.5) diff --git a/twms/daemon.py b/twms/daemon.py index 97e61b5..83a679e 100755 --- a/twms/daemon.py +++ b/twms/daemon.py @@ -8,11 +8,13 @@ from __future__ import print_function -import web +import socket import sys + +import web + from twms import * -import sys, socket try: import psyco diff --git a/twms/drawing.py b/twms/drawing.py index cc69d8c..f1960e6 100644 --- a/twms/drawing.py +++ b/twms/drawing.py @@ -5,19 +5,16 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -try: - from PIL import Image, ImageDraw, ImageColor, ImageFont -except ImportError: - import Image, ImageDraw, ImageColor, ImageFont - -import urllib -import os, sys import array +import math +import os +import sys +import urllib -import projections import config +import projections from gpxparse import GPXParser -import math +from PIL import Image, ImageColor, ImageDraw, ImageFont HAVE_CAIRO = True diff --git a/twms/fetchers.py b/twms/fetchers.py index 02188c9..3ddda94 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -5,24 +5,18 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -from urllib.request import urlopen import filecmp -import time -import os import math +import os import sys -from io import BytesIO - -try: - from PIL import Image -except ImportError: - import Image - +import threading import time +from io import BytesIO +from urllib.request import urlopen import config import projections -import threading +from PIL import Image fetching_now = {} diff --git a/twms/filter.py b/twms/filter.py index 1dc2a4f..edff76a 100644 --- a/twms/filter.py +++ b/twms/filter.py @@ -5,10 +5,8 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -try: - from PIL import Image, ImageFilter, ImageEnhance, ImageOps -except ImportError: - import Image, ImageFilter, ImageEnhance, ImageOps +from PIL import Image, ImageEnhance, ImageFilter, ImageOps + try: import numpy @@ -17,8 +15,10 @@ except ImportError: NUMPY_AVAILABLE = False import datetime + from twms import getimg + try: import config except: diff --git a/twms/gpxparse.py b/twms/gpxparse.py index 5bfcd55..7f8c3fc 100644 --- a/twms/gpxparse.py +++ b/twms/gpxparse.py @@ -5,8 +5,12 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -import sys, string, bz2, gzip, os -from xml.dom import minidom, Node +import bz2 +import gzip +import os +import string +import sys +from xml.dom import Node, minidom class GPXParser: diff --git a/twms/overview.py b/twms/overview.py index 5a96c1f..f2f646f 100644 --- a/twms/overview.py +++ b/twms/overview.py @@ -5,8 +5,8 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -from config import * import projections +from config import * def html(ref): diff --git a/twms/projections.py b/twms/projections.py index d445391..d6316bb 100644 --- a/twms/projections.py +++ b/twms/projections.py @@ -6,6 +6,7 @@ import math + try: import pyproj except ImportError: diff --git a/twms/reproject.py b/twms/reproject.py index 240ec55..84a27b9 100644 --- a/twms/reproject.py +++ b/twms/reproject.py @@ -5,13 +5,10 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -try: - from PIL import Image -except ImportError: - import Image +import sys import projections -import sys +from PIL import Image def reproject(image, bbox, srs_from, srs_to): diff --git a/twms/sketch.py b/twms/sketch.py index 8a1c143..e962609 100644 --- a/twms/sketch.py +++ b/twms/sketch.py @@ -7,6 +7,7 @@ from bbox import * + string = "abcdefghijklmnopqrstuvwxyz012345ABCDEFGHIJKLMNOPQRSTUVWXYZ6789{}" diff --git a/twms/twms.py b/twms/twms.py index 1425203..9fa272b 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -6,21 +6,17 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. -from __future__ import print_function, division - -try: - from PIL import Image, ImageOps, ImageColor -except ImportError: - import Image, ImageOps, ImageColor - +import datetime import imp -import os import math +import os import sys +import time import urllib from io import BytesIO -import time -import datetime + +from PIL import Image, ImageColor, ImageOps + sys.path.append(os.path.join(os.path.dirname(__file__))) @@ -45,19 +41,19 @@ sys.stderr.flush() -import correctify -import capabilities -import fetchers - # import config import bbox import bbox as bbox_utils -import projections +import capabilities +import correctify import drawing +import fetchers import overview +import projections from gpxparse import GPXParser from reproject import reproject + try: import psyco From 0b51cfaa42e5dca0539f2f977c03cbc20c1bd87b Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Mon, 1 Mar 2021 17:12:49 +0100 Subject: [PATCH 03/51] AutoPEP everything --- twms/correctify.py | 3 ++- twms/twms.py | 23 +++++++++-------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/twms/correctify.py b/twms/correctify.py index 17ebbb6..1f8254c 100644 --- a/twms/correctify.py +++ b/twms/correctify.py @@ -11,7 +11,8 @@ import projections -distance = lambda z, x, y, g: ((z - y) ** 2 + (x - g) ** 2) ** (0.5) +def distance(z, x, y, g): + return ((z - y) ** 2 + (x - g) ** 2) ** (0.5) def has_corrections(layer): diff --git a/twms/twms.py b/twms/twms.py index 9fa272b..33d746f 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -15,9 +15,7 @@ import urllib from io import BytesIO -from PIL import Image, ImageColor, ImageOps - - +# import config sys.path.append(os.path.join(os.path.dirname(__file__))) config_path = "/etc/twms/twms.conf" @@ -40,17 +38,16 @@ ) sys.stderr.flush() - -# import config import bbox -import bbox as bbox_utils import capabilities import correctify import drawing import fetchers import overview import projections +from bbox import expand_to_point, zoom_for_bbox from gpxparse import GPXParser +from PIL import Image, ImageColor, ImageOps from reproject import reproject @@ -302,7 +299,7 @@ def twms_main(data): result_img = Image.blend(im2, result_img, 0.5) imgs += 1.0 - ##Applying filters + # Applying filters result_img = filter.raster(result_img, filt, req_bbox, srs) # print(wkt, file=sys.stderr) @@ -356,7 +353,7 @@ def twms_main(data): quality=config.output_quality, progressive=config.output_progressive, ) - else: ## workaround for GIF + else: # workaround for GIF result_img = result_img.convert("RGB") result_img.save( image_content, @@ -521,7 +518,7 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal def getimg(bbox, request_proj, size, layer, start_time, force): orig_bbox = bbox - ## Making 4-corner maximal bbox + # Making 4-corner maximal bbox bbox_p = projections.from4326(bbox, request_proj) bbox_p = projections.to4326( (bbox_p[2], bbox_p[1], bbox_p[0], bbox_p[3]), request_proj @@ -538,9 +535,7 @@ def getimg(bbox, request_proj, size, layer, start_time, force): for point in bbox_4: bb4.append(correctify.rectify(layer, point)) bbox_4 = bb4 - bbox = bbox_utils.expand_to_point(bbox, bbox_4) - # print(bbox) - # print(orig_bbox) + bbox = expand_to_point(bbox, bbox_4) global cached_objs H, W = size @@ -548,7 +543,7 @@ def getimg(bbox, request_proj, size, layer, start_time, force): max_zoom = layer.get("max_zoom", config.default_max_zoom) min_zoom = layer.get("min_zoom", 1) - zoom = bbox_utils.zoom_for_bbox( + zoom = zoom_for_bbox( bbox, size, layer, min_zoom, max_zoom, (config.max_height, config.max_width) ) lo1, la1, lo2, la2 = bbox @@ -600,7 +595,7 @@ def getimg(bbox, request_proj, size, layer, start_time, force): if "filter" in layer: out = filter.raster(out, layer["filter"], orig_bbox, request_proj) - ## TODO: Here's a room for improvement. we could drop this crop in case user doesn't need it. + # TODO: Here's a room for improvement. we could drop this crop in case user doesn't need it. out = out.crop(bbox_im) if "noresize" not in force: if (H == W) and (H == 0): From b1e085839344b523dbd18cf09d542b12fe71cef9 Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Mon, 1 Mar 2021 17:56:51 +0100 Subject: [PATCH 04/51] Make filters work again Commit cb7d39a erroneously removed a circular import of "filter" which was necessary for filters to work. This needs to be dealt with properly in future. --- twms/twms.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/twms/twms.py b/twms/twms.py index 33d746f..d455a1e 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -82,6 +82,9 @@ def twms_main(data): data - dictionary of params. returns (error_code, content_type, resp) """ + # import the filter here due to a circular dependency + # TODO: break the loop + import filter start_time = datetime.datetime.now() @@ -517,6 +520,10 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal def getimg(bbox, request_proj, size, layer, start_time, force): + # import the filter here due to a circular dependency + # TODO: break the loop + import filter + orig_bbox = bbox # Making 4-corner maximal bbox bbox_p = projections.from4326(bbox, request_proj) From 8d99f381223e8ed3759c0060f197a175bd3b507f Mon Sep 17 00:00:00 2001 From: Andrej Shadura Date: Mon, 1 Mar 2021 17:58:13 +0100 Subject: [PATCH 05/51] =?UTF-8?q?Drop=20old=20sources,=20update=20OSM=20an?= =?UTF-8?q?d=20Landsat=20(still=20doesn=E2=80=99t=20work)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- irs_nxt.jpg | 0 twms/twms.conf | 36 ++++++------------------------------ yahoo_nxt.jpg | Bin 4768 -> 0 bytes 3 files changed, 6 insertions(+), 30 deletions(-) delete mode 100644 irs_nxt.jpg delete mode 100644 yahoo_nxt.jpg diff --git a/irs_nxt.jpg b/irs_nxt.jpg deleted file mode 100644 index e69de29..0000000 diff --git a/twms/twms.conf b/twms/twms.conf index ccbc766..9d4ab48 100644 --- a/twms/twms.conf +++ b/twms/twms.conf @@ -54,19 +54,6 @@ cache_tile_responses = { ## Available layers. layers = {\ -"yhsat": { \ - "name": "Yahoo Satellite", - "prefix": "yhsat", # tile directory - "ext": "jpg", # tile images extension - "scalable": False, # could zN tile be constructed of four z(N+1) tiles - "fetch": fetchers.Tile, # function that fetches given tile. should return None if tile wasn't fetched - "remote_url": "http://aerial.maps.yimg.com/ximg?v=1.8&t=a&s=256&r=1&x=%s&y=%s&z=%s", - "transform_tile_number": lambda z,x,y: (x,((2**(z-1)/2)-1)-y,z), - "dead_tile": install_path + "yahoo_nxt.jpg", - "min_zoom": 2, - "max_zoom": 18, - "proj": "EPSG:3857", -},\ "yasat": { \ "name": "Yandex Satellite", "prefix": "yasat", # tile directory @@ -84,7 +71,7 @@ layers = {\ "ext": "png", # tile images extension "scalable": False, # could zN tile be constructed of four z(N+1) tiles "fetch": fetchers.Tile, # function that fetches given tile. should return None if tile wasn't fetched - "remote_url": "http://c.tile.openstreetmap.org/%s/%s/%s.png", + "remote_url": "https://tile.openstreetmap.org/%s/%s/%s.png", "transform_tile_number": lambda z,x,y: (z-1,x,y), "proj": "EPSG:3857", "empty_color": "#F1EEE8", @@ -102,25 +89,14 @@ layers = {\ "data_bounding_box": (23.16722,51.25930,32.82244,56.18162), },\ "landsat": { \ - "name": "Landsat from onearth.jpl.nasa.gov", + "name": "Landsat from gibs.earthdata.nasa.gov", "prefix": "landsat", # tile directory "ext": "jpg", # tile images extension "scalable": False, # could zN tile be constructed of four z(N+1) tiles - "fetch": fetchers.WMS, # function that fetches given tile. should return None if tile wasn't fetched - "remote_url": "http://onearth.jpl.nasa.gov/wms.cgi?request=GetMap&layers=global_mosaic&styles=&format=image/jpeg&", # string without srs, height, width and bbox - "max_zoom": 14, - "proj": "EPSG:4326", - "wms_proj": "EPSG:4326", # what projection to ask from wms -},\ -"latlonsat": { \ - "name": "Imagery from latlon.org", - "prefix": "latlonsat", # tile directory - "ext": "jpg", # tile images extension - "scalable": False, # could zN tile be constructed of four z(N+1) tiles - "fetch": fetchers.WMS, # function that fetches given tile. should return None if tile wasn't fetched - "remote_url": "http://dev.latlon.org/cgi-bin/ms?FORMAT=image/jpeg&VERSION=1.1.1&SERVICE=WMS&REQUEST=GetMap&Layers=sat,plane&", # string without srs, height, width and bbox - "max_zoom": 19, + "fetch": fetchers.Tile, # function that fetches given tile. should return None if tile wasn't fetched + "transform_tile_number": lambda z,x,y: (z-1,x,y), + "remote_url": "https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/Landsat_WELD_CorrectedReflectance_TrueColor_Global_Annual/default/2000-12-01/GoogleMapsCompatible_Level12/%s/%s/%s.jpg", + "max_zoom": 12, "proj": "EPSG:3857", - "wms_proj": "EPSG:3857", # what projection to ask from wms },\ } diff --git a/yahoo_nxt.jpg b/yahoo_nxt.jpg deleted file mode 100644 index 9e1970f0cafb35ba74c312941e2a473f9d4b8405..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4768 zcmd^Bi96Ko*Z*3_zLQ}hMz&-d`yOFrPj;2;)L63&5lT;qVMaoR5+YGyY@sY;hOv{~ zh%95$RQ7Cn{eIW`Jiqt-7vA^W=f1A{y6$t%y?#FD98Vt413Z_^EX)8pIsl+MX~6Le zU;?l(GlQ6!SU?~UD=P~dhX5xBJ39xMkDpuMw1~LaX%PtIjFh6x8A$~w1R|>`r=YB& zrluw?qph!{s;3B3Q$212*jfG+j*gBOIK~6wC&8KM{w=_N3mrWJBNH=-g_Vu{#GsA` zpr>PCpl4)YVq#=Gv5r5P2N-#o_#{*=F!S5r07-@ks3xI{SfIw$?Shwwe@Vf-!joCq zgiZ;AMb1jg$jZs9scUFzY3rC=G&M7aTU>Iu;^^e;;_Bw@}7FD>8rBx*A+FjxVrj=#-bJXZQEs{_!-x@~>+%1hz=BFH`@|>WlU9e;1QPS7pOHQd)APcl%*X}sTb{+LyEMRQCz3sEsc*@{NOd$md11i9f^g$9_hV1p?R z=IM!PTHI1UFAzopIRkN&o*k! zGcRpKIEkCx6DfM8ma1*f2X>j4uaISoAb`z}HuUB86(;y3LeZPt? z>OR8})+W3o=r%p_^@o-^dwvKGiSJeaB$i~Ci04u%yxU5HPbU-!whSNTlKiRzm?!L) zF@2;mnHL-?8J9*rR^M#I{VpW?ptPTm(v#`G2NUlNfQH6>SMu)ci2I|ygcsW`Eo9c* zX*=;CCba5!l_Dg!$d(Yqa+`^VRLZ`MbBGq@-5@bG|KjMgk?ZOUjY)^CX_dpbyouI{ z{zc18EA^}YV3iP4!CE=ld#jcx7f|p})XU38TkA9&LZ5)Ey(z>)T+v%1@W2PEYb#!_ zT0l5<`Y(l4B_T${pWdL`v)%7Z<4!21+NZnq+%FjrFe{zRz9o4264rBZMy*F()Kw;V zY$3a=+Yji#)3YqpXthHNXZ@?hXWWqHl8bB)A^f5oueC&D_8!E`jh|ZC`ATy>^0>^D zmMx#KDsyIYcJ#cui*G6;CK6d}-O-&e)#aRo0Z;@6&Z zR!mh5W{$ki>f`nAm54@Tg9#u|l6AZn2_};dHS-qD)l+N&&k)Sdh?Uq!mb)nB3(y)R z#QePc?T3=#!l%tCc0I4*IlXLpiVv10+c>kCRml;_F%Z<}hFH9cYp(+5CVu#VEAJDU zpFK8G--f$RC*EE!cbdzhP`1F=xYQ)mU@aV0$mx>&XKuwoAUxQUJ${@nQHzw-4xmRW zhyTMFjP%N}P9_0xgu++9RP(yaiTx~*q2}5i+w;-oKXGv>)yN1dBDd!MP9%2%00_%%;&iu0C5*!Uw7cN!f^vcj zDR#`~*P253yXj1|Ms+r&_+s0+_YXJPo1T{aiQ0;MnW}TrPmBzk}8_=965ypZDJ$_(Fa!`h=30 zkR2HLSUP=DFV~o%Ec4(2a)8`KJRM|GKArPO=(=)u#i}EF8h@e%gf267|8je|`_?qJ>R$&9I!!dxP-6_k8oHE)?ul<>di0pDHT;E-w|1F(P*gM~cRm+}m@{y)bJpsXDIY9V+9@dd zaG;gHo1Yp#KGS%Y>rdX!nq7b86w!Yt#XoX`vqQevr%XJ@o*0ayZo?7Z`L;3fX{219 z`h~_t^vv@?$%3QTelji`CGbmcN79u0Ny)axaSRR?ksPotR__(dU8Bn}1KPw3-D`H7 z*YNcmcn%dOU(YI>&wL>lFerGwYP*#hnKD_!+o+?8F2wRaCse#`ef2@^6zHaGsFe-# zT;5VRC}QZ!W>D1AT`?<6cBs)U<_^lM6^PQJwMZRWHRVY)%2sX+>SejPk(8FzPNl6m zc^2yCUd;UXd%z6&O(#vv)Z%Zb#m72Y()882YZaPZC7!_TKP%tBrFkAaOWn12utg|s zX&h&joV+K@bqY;QZ&eOz96AOz=D$}6OXE#exgz;%R;BZ7ld^E0 ztKgqLAx{=PArRJo$nnU~R!H{HJ$WYlMxtpZEP3)9@)%fiv@b#jy^k*|&7T&Unus#E zFVrvCB(!)p2~gD^@zYe<9S_v&M>fC)UX?{P%-mM{ zo2gk2EkZdeJ@w{OjGG8PK?fX{uqWjm<5q>vgR<})KD?wm^5LVJth>FMh$i7{TxKdi zGp*f3D$xYjs@j{74)PK#$8(C!hNTDC9f0d=tKr9kO9Sd%Z zAM1UrM`kMJz0pH?s;njt1{p#O#y~sdzJ7X>pvMwT8$Nux9Lt&6D13%4d2>7KAshZL~yX;E<{G8f|{iO9#_1Kl| z5z7EX5npd8yJRzs&P>$;yzP+sj=B1L*H{a$+Z_rrQ!Nqc?jiR*;x~8IsSlm59)o<- zd6G#yIeZbDKC%Su9zi$vwKr6Sd7%xX8V^;ix%Cjc=8q0_F5GR9a!{4t%}t)?t5SQY zHK7T=S)NQh%?#S$jXhwy6T6fVh7hQ?4VGqDDMhA|wAdn4v$Gbi-# zI<28txdDTqaT`Sk+gn@sU&LbLIGHsVL^Zs_55N6n|g3A02u@*e8EkE&^1L`WU0 z!>C~#!^%L=_uU4KRUwMK#sXseVh+iHQa#}O$%YxEw^Eel{F%1YNBSaRnHT!POBxVl z0CyvZe;CHGA5ylWHO`&yXv!AIk4Xhx#$$QR<(Zi;Fh-Vpf3CAonDdSYqFC4x!3$BD z4B3;)%J6oXZ|9ty`OVARkAadNu|i-;POqu7mhfY%I0)6A$Jp31K-k_zR(L4X2Dk8Bsxhvf zxqm%RG}s0%wXFp$8_x@S0$RvD%Ls7{l)`7J*6){!_*z{}@wT*eFqi4)&FYwM2s^KhFd7i5e(? zhWZTQz1MA61)=EqYY*DQ4JYb^g2ew?FozZa$3Wp$*ksr!xOijmwSf1`q(3OtD(ag) zcWiV*%dTYB-a*viH`epeXWw)mfqOE3NN79iZDx3gU^gc+s8@cdtw$;c2Fs{`T=f#2 yZ$Mwg3|# Date: Mon, 1 Mar 2021 17:59:30 +0100 Subject: [PATCH 06/51] Drop no longer useful .hgtags --- .hgtags | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .hgtags diff --git a/.hgtags b/.hgtags deleted file mode 100644 index c0e3b5e..0000000 --- a/.hgtags +++ /dev/null @@ -1,5 +0,0 @@ -0a1cc6bba99284ea984710c18cf9bc116d791928 0.01q -bfab1f075a7f33cfd760956b94f3de3fbcbefdf2 0.02w -962296dc4deb9b4821e21d0dd0e17e1fde2ef2d1 0.03e -e40a1db64469baab230c1b37d0069bf15d2abf90 0.04r -413c35c82b67468f77e35663dc37040c681ccaa2 0.05t From 88cfbb9d5cc022fe6276f84b37a16c4dd2c0a09c Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 13:18:45 +0400 Subject: [PATCH 07/51] chore: bootstrap packaging and CI --- .github/workflows/ci.yml | 54 ++++++++++++++++++++++++++++++++++++++++ COPYING | 4 ++- README.md | 7 ++++++ pyproject.toml | 3 +++ setup.cfg | 8 +++--- twms/__init__.py | 4 +++ twms/twms.py | 25 ++++++++++++------- twms/version.py | 7 ++++++ 8 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 pyproject.toml create mode 100644 twms/version.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..dc11d9b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI + +on: + push: + branches: + - master + - "ai/**" + pull_request: + workflow_dispatch: + +jobs: + package-smoke: + name: Package smoke (${{ matrix.os }}, Python ${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + python-version: + - "3.11" + - "3.13" + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install build tools + run: python -m pip install --upgrade pip build + + - name: Install package + run: python -m pip install -e . + + - name: Compile sources + run: python -m compileall -q setup.py index.py tools twms + + - name: Import WSGI application + run: | + python - <<'PY' + import importlib.metadata + import twms + import twms.daemon + + assert twms.__version__ == "0.07z" + assert importlib.metadata.version("twms") == "0.7+z" + print(type(twms.daemon.application).__name__) + PY + + - name: Build source and wheel distributions + run: python -m build diff --git a/COPYING b/COPYING index f41b044..1d14a85 100644 --- a/COPYING +++ b/COPYING @@ -1,4 +1,5 @@ -The authors, Darafei Praliaskouski (Komzpa) and Andrew Shadura, +The authors, Darafei Praliaskouski (Komzpa), Andrew Shadura, and +Eugene Dvoretsky (Radioxoma), explicitly disclaim copyright in all jurisdictions which recognise such a disclaimer. In such jurisdictions, this software is released into the Public Domain. @@ -7,6 +8,7 @@ In jurisdictions which do not recognise Public Domain property, this software is Copyright © 2009—2013 Darafei Praliaskouski Copyright © 2010—2013, 2016 Andrew Shadura + Copyright © 2020—2021, 2023—2024 Eugene Dvoretsky and is released under the terms of the ISC License (see below). diff --git a/README.md b/README.md index 942386c..2668174 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,13 @@ so GIS applications that support WMS protocol can access this tile set. Also, twms can act as a proxy and perform WMS requests to external services and serve the tile cache +## Credits + +twms was originally written by Darafei Praliaskouski (Komzpa). +Andrew Shadura maintains Debian packaging and wrote the Debian manpage. +Eugene Dvoretsky (Radioxoma) contributed modernization work around +packaging, serving, caching, and tile protocols. + TODO ==== diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..301a9c6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg index 49c5e6a..f92de8a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = twms -version = 0.07z +version = attr: twms.version.__packaging_version__ author = Darafei Praliaskouski author_email = me@komzpa.net url = https://github.com/komzpa/twms @@ -15,12 +15,14 @@ classifiers = License :: Public Domain Operating System :: OS Independent Programming Language :: Python - Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 Topic :: Internet :: WWW/HTTP Topic :: Scientific/Engineering :: GIS [options] -python_requires = >= 3.6 +python_requires = >= 3.11 packages = find: install_requires = Pillow diff --git a/twms/__init__.py b/twms/__init__.py index 144a350..934c69e 100644 --- a/twms/__init__.py +++ b/twms/__init__.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- +from .version import __version__ + + __all__ = [ + "__version__", "twms", "bbox", "canvas", diff --git a/twms/twms.py b/twms/twms.py index d455a1e..a9a1d88 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -7,7 +7,8 @@ # and/or modify it under the terms specified in COPYING. import datetime -import imp +import importlib.machinery +import importlib.util import math import os import sys @@ -18,21 +19,27 @@ # import config sys.path.append(os.path.join(os.path.dirname(__file__))) + +def load_config(path): + loader = importlib.machinery.SourceFileLoader("twms.config", path) + spec = importlib.util.spec_from_loader("twms.config", loader) + module = importlib.util.module_from_spec(spec) + sys.modules["twms.config"] = module + sys.modules["config"] = module + loader.exec_module(module) + return module + + config_path = "/etc/twms/twms.conf" if os.path.exists(config_path): - try: - config = imp.load_source("twms.config", config_path) - except: - config = imp.load_source("config", config_path) + config = load_config(config_path) else: try: config_path = os.path.join(os.path.dirname(__file__), "twms.conf") - config = imp.load_source("twms.config", config_path) + config = load_config(config_path) except: config_path = os.path.join(os.path.realpath(sys.path[0]), "twms.conf") - config = imp.load_source( - "config", os.path.join(os.path.realpath(sys.path[0]), "twms.conf") - ) + config = load_config(config_path) sys.stderr.write( "Configuration file not found, using defaults from %s\n" % config_path ) diff --git a/twms/version.py b/twms/version.py new file mode 100644 index 0000000..a798838 --- /dev/null +++ b/twms/version.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +__version__ = "0.07z" + +# Python packaging metadata must be PEP 440. Keep the public TWMS keyboard +# version above; this adapter exists only for wheel/sdist tooling. +__packaging_version__ = "0.7+z" From 1320671149c76bc5f64b5e4d8e831d0f64d3b9b0 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 13:40:30 +0400 Subject: [PATCH 08/51] test: cover legacy package smoke --- .github/workflows/ci.yml | 3 ++ tests/test_legacy_smoke.py | 59 ++++++++++++++++++++++++++++++++++++++ twms/__init__.py | 19 ++++++++++++ twms/config_loader.py | 32 +++++++++++++++++++++ twms/filter.py | 2 +- twms/twms.py | 28 ++---------------- 6 files changed, 116 insertions(+), 27 deletions(-) create mode 100644 tests/test_legacy_smoke.py create mode 100644 twms/config_loader.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc11d9b..5e6feca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,5 +50,8 @@ jobs: print(type(twms.daemon.application).__name__) PY + - name: Run legacy smoke tests + run: python -m unittest discover -s tests + - name: Build source and wheel distributions run: python -m build diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py new file mode 100644 index 0000000..eb901e2 --- /dev/null +++ b/tests/test_legacy_smoke.py @@ -0,0 +1,59 @@ +import importlib +import importlib.metadata +import unittest + +import twms +import twms.daemon +import twms.twms + + +class LegacySmokeTest(unittest.TestCase): + def test_public_version_keeps_keyboard_suffix(self): + self.assertEqual(twms.__version__, "0.07z") + self.assertEqual(importlib.metadata.version("twms"), "0.7+z") + + def test_legacy_modules_import_as_package_modules(self): + modules = [ + "twms.bbox", + "twms.capabilities", + "twms.correctify", + "twms.drawing", + "twms.filter", + "twms.gpxparse", + "twms.overview", + "twms.projections", + "twms.reproject", + "twms.sketch", + ] + for module in modules: + with self.subTest(module=module): + importlib.import_module(module) + + def test_wms_capabilities_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetCapabilities", + "version": "1.1.1", + "ref": "http://example.test/wms", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/vnd.ogc.wms_xml") + self.assertIn("", body) + self.assertIn("Yandex Satellite", body) + + def test_wsgi_application_imports(self): + self.assertTrue(callable(twms.daemon.application)) + + +if __name__ == "__main__": + unittest.main() diff --git a/twms/__init__.py b/twms/__init__.py index 934c69e..ec99052 100644 --- a/twms/__init__.py +++ b/twms/__init__.py @@ -1,8 +1,27 @@ # -*- coding: utf-8 -*- +import os +import sys + +package_dir = os.path.dirname(__file__) +if package_dir not in sys.path: + sys.path.append(package_dir) + +from .config_loader import load_default_config from .version import __version__ +load_default_config() + + +def __getattr__(name): + if name in {"getimg", "tile_image", "twms_main"}: + from . import twms as twms_module + + return getattr(twms_module, name) + raise AttributeError(name) + + __all__ = [ "__version__", "twms", diff --git a/twms/config_loader.py b/twms/config_loader.py new file mode 100644 index 0000000..039281f --- /dev/null +++ b/twms/config_loader.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import importlib.machinery +import importlib.util +import os +import sys + + +def load_config(path): + loader = importlib.machinery.SourceFileLoader("twms.config", path) + spec = importlib.util.spec_from_loader("twms.config", loader) + module = importlib.util.module_from_spec(spec) + sys.modules["twms.config"] = module + sys.modules["config"] = module + loader.exec_module(module) + return module + + +def load_default_config(): + if "config" in sys.modules: + return sys.modules["config"] + + package_dir = os.path.dirname(__file__) + paths = [ + "/etc/twms/twms.conf", + os.path.join(package_dir, "twms.conf"), + os.path.join(os.path.realpath(sys.path[0]), "twms.conf"), + ] + for path in paths: + if os.path.exists(path): + return load_config(path) + return load_config(paths[-1]) diff --git a/twms/filter.py b/twms/filter.py index edff76a..10480a5 100644 --- a/twms/filter.py +++ b/twms/filter.py @@ -16,7 +16,7 @@ NUMPY_AVAILABLE = False import datetime -from twms import getimg +from twms.twms import getimg try: diff --git a/twms/twms.py b/twms/twms.py index a9a1d88..5de0513 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -7,8 +7,6 @@ # and/or modify it under the terms specified in COPYING. import datetime -import importlib.machinery -import importlib.util import math import os import sys @@ -16,34 +14,12 @@ import urllib from io import BytesIO -# import config sys.path.append(os.path.join(os.path.dirname(__file__))) +from twms.config_loader import load_default_config -def load_config(path): - loader = importlib.machinery.SourceFileLoader("twms.config", path) - spec = importlib.util.spec_from_loader("twms.config", loader) - module = importlib.util.module_from_spec(spec) - sys.modules["twms.config"] = module - sys.modules["config"] = module - loader.exec_module(module) - return module - -config_path = "/etc/twms/twms.conf" -if os.path.exists(config_path): - config = load_config(config_path) -else: - try: - config_path = os.path.join(os.path.dirname(__file__), "twms.conf") - config = load_config(config_path) - except: - config_path = os.path.join(os.path.realpath(sys.path[0]), "twms.conf") - config = load_config(config_path) - sys.stderr.write( - "Configuration file not found, using defaults from %s\n" % config_path - ) - sys.stderr.flush() +config = load_default_config() import bbox import capabilities From 47455de9e00bff61f7ced168c3bf7c6a22238e64 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 13:42:17 +0400 Subject: [PATCH 09/51] fix: clear Python syntax warnings --- twms/daemon.py | 2 +- twms/twms.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/twms/daemon.py b/twms/daemon.py index 83a679e..15e06fa 100755 --- a/twms/daemon.py +++ b/twms/daemon.py @@ -37,7 +37,7 @@ def handler(data): urls = ( - "/(.*)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?(.*)", "tilehandler", + r"/(.*)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?(.*)", "tilehandler", "/(.*)", "mainhandler", ) diff --git a/twms/twms.py b/twms/twms.py index 5de0513..35f85e8 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -187,8 +187,8 @@ def twms_main(data): if layer[0] in config.layers: if ( config.layers[layer[0]]["proj"] == srs - and width is 256 - and height is 256 + and width == 256 + and height == 256 and not filt and not force and not correctify.has_corrections(config.layers[layer[0]]) From 996ce0b350e79d4a9a59f4df549768f0cfa99cd9 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 13:44:27 +0400 Subject: [PATCH 10/51] test: cover legacy GetTile smoke --- tests/test_legacy_smoke.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index eb901e2..f7c2ef1 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -1,7 +1,10 @@ import importlib import importlib.metadata +from io import BytesIO import unittest +from PIL import Image + import twms import twms.daemon import twms.twms @@ -51,6 +54,24 @@ def test_overview_smoke(self): self.assertIn("", body) self.assertIn("Yandex Satellite", body) + def test_gettile_transparent_layer_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "0", + "x": "0", + "y": "0", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (256, 256)) + self.assertEqual(image.mode, "RGBA") + def test_wsgi_application_imports(self): self.assertTrue(callable(twms.daemon.application)) From 60bd6199a72adf50df8e552c82eb090ddff1f5ac Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 13:48:57 +0400 Subject: [PATCH 11/51] feat: add stdlib HTTP server --- setup.cfg | 3 +- tests/test_legacy_smoke.py | 31 ++++++++++++++ twms/__main__.py | 7 ++++ twms/server.py | 82 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 twms/__main__.py create mode 100644 twms/server.py diff --git a/setup.cfg b/setup.cfg index f92de8a..57f9133 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,8 @@ cairo = pycairo [options.entry_points] console_scripts = - twms = twms.daemon:main + twms = twms.server:main + twms-webpy = twms.daemon:main [flake8] doctests = yes diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index f7c2ef1..9bdf034 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -1,12 +1,16 @@ import importlib import importlib.metadata from io import BytesIO +import threading import unittest +import urllib.request +from http.server import ThreadingHTTPServer from PIL import Image import twms import twms.daemon +import twms.server import twms.twms @@ -75,6 +79,33 @@ def test_gettile_transparent_layer_smoke(self): def test_wsgi_application_imports(self): self.assertTrue(callable(twms.daemon.application)) + def test_stdlib_server_serves_wms_and_gettile(self): + httpd = ThreadingHTTPServer(("127.0.0.1", 0), twms.server.TWMSRequestHandler) + thread = threading.Thread(target=httpd.serve_forever) + thread.daemon = True + thread.start() + base = "http://127.0.0.1:%s" % httpd.server_address[1] + try: + with urllib.request.urlopen( + base + "/?request=GetCapabilities&version=1.1.1" + ) as response: + body = response.read().decode("utf-8") + self.assertEqual(response.status, 200) + self.assertIn("application/vnd.ogc.wms_xml", response.headers["Content-Type"]) + self.assertIn(" Date: Sun, 26 Jul 2026 13:54:36 +0400 Subject: [PATCH 12/51] feat: expose TileJSON metadata --- tests/test_legacy_smoke.py | 27 ++++++++++++++ twms/server.py | 30 ++++++++++------ twms/tilejson.py | 73 ++++++++++++++++++++++++++++++++++++++ twms/twms.py | 12 +++++++ 4 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 twms/tilejson.py diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 9bdf034..eaa7f0f 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -1,5 +1,6 @@ import importlib import importlib.metadata +import json from io import BytesIO import threading import unittest @@ -76,6 +77,26 @@ def test_gettile_transparent_layer_smoke(self): self.assertEqual(image.size, (256, 256)) self.assertEqual(image.mode, "RGBA") + def test_tilejson_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTileJSON", + "layers": "osm", + "ref": "http://example.test/", + } + ) + + doc = json.loads(body) + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/json") + self.assertEqual(doc["tilejson"], "3.0.0") + self.assertEqual(doc["name"], "OpenStreetMap mapnik") + self.assertEqual(doc["scheme"], "xyz") + self.assertEqual(doc["tiles"], ["http://example.test/osm/{z}/{x}/{y}.png"]) + self.assertEqual(doc["bounds"], [-180.0, -85.0511287798, 180.0, 85.0511287798]) + self.assertEqual(doc["minzoom"], 0) + self.assertEqual(doc["maxzoom"], 18) + def test_wsgi_application_imports(self): self.assertTrue(callable(twms.daemon.application)) @@ -101,6 +122,12 @@ def test_stdlib_server_serves_wms_and_gettile(self): with Image.open(BytesIO(body)) as image: self.assertEqual(image.size, (256, 256)) self.assertEqual(image.mode, "RGBA") + + with urllib.request.urlopen(base + "/tilejson/osm.json") as response: + doc = json.loads(response.read().decode("utf-8")) + self.assertEqual(response.status, 200) + self.assertIn("application/json", response.headers["Content-Type"]) + self.assertEqual(doc["tiles"], [base + "/osm/{z}/{x}/{y}.png"]) finally: httpd.shutdown() httpd.server_close() diff --git a/twms/server.py b/twms/server.py index 8f1733f..a849a24 100644 --- a/twms/server.py +++ b/twms/server.py @@ -15,6 +15,7 @@ tile_route = re.compile(r"/(.*)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?(.*)") +tilejson_route = re.compile(r"/tilejson/(.*)\.json") def request_url(handler): @@ -27,20 +28,27 @@ def request_url(handler): def dispatch(path, ref=None): parsed = urllib.parse.urlsplit(path) - match = tile_route.fullmatch(parsed.path) - if match: - ext = match.group(5) or ".jpg" + tilejson_match = tilejson_route.fullmatch(parsed.path) + if tilejson_match: data = { - "request": "GetTile", - "layers": match.group(1), - "format": ext.strip(".").lower(), - "z": match.group(2), - "x": match.group(3), - "y": match.group(4), + "request": "GetTileJSON", + "layers": urllib.parse.unquote(tilejson_match.group(1)), } else: - data = dict(urllib.parse.parse_qsl(parsed.query)) - data = dict((key.lower(), data[key]) for key in data) + match = tile_route.fullmatch(parsed.path) + if match: + ext = match.group(5) or ".jpg" + data = { + "request": "GetTile", + "layers": match.group(1), + "format": ext.strip(".").lower(), + "z": match.group(2), + "x": match.group(3), + "y": match.group(4), + } + else: + data = dict(urllib.parse.parse_qsl(parsed.query)) + data = dict((key.lower(), data[key]) for key in data) if ref and "ref" not in data: data["ref"] = ref diff --git a/twms/tilejson.py b/twms/tilejson.py new file mode 100644 index 0000000..29bc1bb --- /dev/null +++ b/twms/tilejson.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# This file is part of twms. + +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms specified in COPYING. + +import json + +import bbox + + +def _layer_names(config, layers): + names = [name for name in layers.split(",") if name] + if not names: + names = [name for name in config.default_layers.split(",") if name] + if not names: + raise KeyError("TileJSON needs an existing layer name") + for name in names: + if name not in config.layers: + raise KeyError("Unknown layer: %s" % name) + return names + + +def _layer_bounds(config, layer): + return layer.get( + "data_bounding_box", + layer.get("bbox", config.default_bbox), + ) + + +def _tile_extension(config, layer_names, format_name): + if format_name: + return format_name.lower().replace("image/", "").replace("jpeg", "jpg") + if len(layer_names) == 1: + return config.layers[layer_names[0]].get("ext", "jpg") + return config.default_format.lower().replace("image/", "").replace("jpeg", "jpg") + + +def document(config, layers, ref, format_name=""): + layer_names = _layer_names(config, layers) + layer_items = [config.layers[name] for name in layer_names] + bounds = _layer_bounds(config, layer_items[0]) + for layer in layer_items[1:]: + bounds = bbox.add(bounds, _layer_bounds(config, layer)) + + minzoom = max(layer.get("min_zoom", 0) for layer in layer_items) + maxzoom = min(layer.get("max_zoom", config.default_max_zoom) for layer in layer_items) + center = [ + (bounds[0] + bounds[2]) / 2.0, + (bounds[1] + bounds[3]) / 2.0, + minzoom, + ] + ext = _tile_extension(config, layer_names, format_name) + tile_url = "%s%s/{z}/{x}/{y}.%s" % (ref, ",".join(layer_names), ext) + + return { + "tilejson": "3.0.0", + "name": ", ".join(layer["name"] for layer in layer_items), + "scheme": "xyz", + "tiles": [tile_url], + "bounds": list(bounds), + "center": center, + "minzoom": minzoom, + "maxzoom": maxzoom, + } + + +def dumps(config, layers, ref, format_name=""): + return json.dumps( + document(config, layers, ref, format_name=format_name), + sort_keys=True, + ) diff --git a/twms/twms.py b/twms/twms.py index 35f85e8..f1901d6 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -28,6 +28,7 @@ import fetchers import overview import projections +import tilejson from bbox import expand_to_point, zoom_for_bbox from gpxparse import GPXParser from PIL import Image, ImageColor, ImageOps @@ -109,6 +110,17 @@ def twms_main(data): if req_type == "GetCapabilities": content_type, resp = capabilities.get(version, ref) return (OK, content_type, resp) + if req_type.lower() in ("gettilejson", "tilejson"): + try: + resp = tilejson.dumps( + config, + data.get("layers", ""), + ref, + format_name=data.get("format", ""), + ) + return (OK, "application/json", resp) + except KeyError as exc: + return (400, "text/plain", str(exc)) layer = data.get("layers", config.default_layers).split(",") if ("layers" in data) and not layer[0]: From 1ddb2bb6c7a587e97e2413dd1357e581bc5d2439 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:02:07 +0400 Subject: [PATCH 13/51] feat: expose WMTS tile metadata Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 75 ++++++++++++++ twms/__init__.py | 1 + twms/server.py | 27 ++++- twms/twms.py | 14 +++ twms/wmts.py | 195 +++++++++++++++++++++++++++++++++++++ 5 files changed, 309 insertions(+), 3 deletions(-) create mode 100644 twms/wmts.py diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index eaa7f0f..5ae22da 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -6,6 +6,7 @@ import unittest import urllib.request from http.server import ThreadingHTTPServer +import xml.etree.ElementTree as ET from PIL import Image @@ -32,6 +33,7 @@ def test_legacy_modules_import_as_package_modules(self): "twms.projections", "twms.reproject", "twms.sketch", + "twms.wmts", ] for module in modules: with self.subTest(module=module): @@ -97,6 +99,60 @@ def test_tilejson_smoke(self): self.assertEqual(doc["minzoom"], 0) self.assertEqual(doc["maxzoom"], 18) + def test_wmts_capabilities_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "service": "WMTS", + "request": "GetCapabilities", + "ref": "http://example.test/", + } + ) + + root = ET.fromstring(body) + namespaces = { + "wmts": "http://www.opengis.net/wmts/1.0", + "ows": "http://www.opengis.net/ows/1.1", + } + layer_ids = [ + element.text + for element in root.findall( + "./wmts:Contents/wmts:Layer/ows:Identifier", + namespaces, + ) + ] + resource = root.find( + "./wmts:Contents/wmts:Layer[ows:Identifier='osm']/wmts:ResourceURL", + namespaces, + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(root.tag, "{http://www.opengis.net/wmts/1.0}Capabilities") + self.assertIn("osm", layer_ids) + self.assertEqual( + resource.attrib["template"], + "http://example.test/wmts/osm/{TileMatrix}/{TileCol}/{TileRow}.png", + ) + + def test_wmts_kvp_gettile_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "service": "WMTS", + "request": "GetTile", + "layer": "transparent", + "format": "image/png", + "tilematrix": "0", + "tilecol": "0", + "tilerow": "0", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (256, 256)) + self.assertEqual(image.mode, "RGBA") + def test_wsgi_application_imports(self): self.assertTrue(callable(twms.daemon.application)) @@ -128,6 +184,25 @@ def test_stdlib_server_serves_wms_and_gettile(self): self.assertEqual(response.status, 200) self.assertIn("application/json", response.headers["Content-Type"]) self.assertEqual(doc["tiles"], [base + "/osm/{z}/{x}/{y}.png"]) + + with urllib.request.urlopen( + base + "/wmts/1.0.0/WMTSCapabilities.xml" + ) as response: + root = ET.fromstring(response.read().decode("utf-8")) + self.assertEqual(response.status, 200) + self.assertIn("text/xml", response.headers["Content-Type"]) + self.assertEqual( + root.tag, + "{http://www.opengis.net/wmts/1.0}Capabilities", + ) + + with urllib.request.urlopen(base + "/wmts/transparent/0/0/0.png") as response: + body = response.read() + self.assertEqual(response.status, 200) + self.assertIn("image/png", response.headers["Content-Type"]) + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (256, 256)) + self.assertEqual(image.mode, "RGBA") finally: httpd.shutdown() httpd.server_close() diff --git a/twms/__init__.py b/twms/__init__.py index ec99052..e7bf4a8 100644 --- a/twms/__init__.py +++ b/twms/__init__.py @@ -31,4 +31,5 @@ def __getattr__(name): "drawing", "projections", "reproject", + "wmts", ] diff --git a/twms/server.py b/twms/server.py index a849a24..8d04a50 100644 --- a/twms/server.py +++ b/twms/server.py @@ -16,6 +16,10 @@ tile_route = re.compile(r"/(.*)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?(.*)") tilejson_route = re.compile(r"/tilejson/(.*)\.json") +wmts_capabilities_route = "/wmts/1.0.0/WMTSCapabilities.xml" +wmts_tile_route = re.compile( + r"/wmts/([^/]+)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?" +) def request_url(handler): @@ -34,8 +38,13 @@ def dispatch(path, ref=None): "request": "GetTileJSON", "layers": urllib.parse.unquote(tilejson_match.group(1)), } + elif parsed.path == wmts_capabilities_route: + data = { + "request": "GetCapabilities", + "service": "WMTS", + } else: - match = tile_route.fullmatch(parsed.path) + match = wmts_tile_route.fullmatch(parsed.path) if match: ext = match.group(5) or ".jpg" data = { @@ -47,8 +56,20 @@ def dispatch(path, ref=None): "y": match.group(4), } else: - data = dict(urllib.parse.parse_qsl(parsed.query)) - data = dict((key.lower(), data[key]) for key in data) + match = tile_route.fullmatch(parsed.path) + if match: + ext = match.group(5) or ".jpg" + data = { + "request": "GetTile", + "layers": match.group(1), + "format": ext.strip(".").lower(), + "z": match.group(2), + "x": match.group(3), + "y": match.group(4), + } + else: + data = dict(urllib.parse.parse_qsl(parsed.query)) + data = dict((key.lower(), data[key]) for key in data) if ref and "ref" not in data: data["ref"] = ref diff --git a/twms/twms.py b/twms/twms.py index f1901d6..5b0c074 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -29,6 +29,7 @@ import overview import projections import tilejson +import wmts from bbox import expand_to_point, zoom_for_bbox from gpxparse import GPXParser from PIL import Image, ImageColor, ImageOps @@ -107,6 +108,19 @@ def twms_main(data): req_type = data.get("request", "GetMap") version = data.get("version", "1.1.1") ref = data.get("ref", config.service_url) + if data.get("service", "").lower() == "wmts" and req_type.lower() == "getcapabilities": + return (OK, "text/xml", wmts.capabilities(config, ref)) + if req_type.lower() == "getwmtscapabilities": + return (OK, "text/xml", wmts.capabilities(config, ref)) + if data.get("service", "").lower() == "wmts" and req_type.lower() == "gettile": + if "layers" not in data and "layer" in data: + data["layers"] = data["layer"] + if "z" not in data and "tilematrix" in data: + data["z"] = data["tilematrix"] + if "x" not in data and "tilecol" in data: + data["x"] = data["tilecol"] + if "y" not in data and "tilerow" in data: + data["y"] = data["tilerow"] if req_type == "GetCapabilities": content_type, resp = capabilities.get(version, ref) return (OK, content_type, resp) diff --git a/twms/wmts.py b/twms/wmts.py new file mode 100644 index 0000000..289f79a --- /dev/null +++ b/twms/wmts.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +# This file is part of twms. + +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms specified in COPYING. + +import xml.etree.ElementTree as ET + +import projections + + +WMTS = "http://www.opengis.net/wmts/1.0" +OWS = "http://www.opengis.net/ows/1.1" +XLINK = "http://www.w3.org/1999/xlink" +XSI = "http://www.w3.org/2001/XMLSchema-instance" + + +def _tag(namespace, name): + return "{%s}%s" % (namespace, name) + + +def _extension(layer): + return layer.get("ext", "jpg").lower().replace("jpeg", "jpg") + + +def _mime_type(layer): + ext = _extension(layer) + if ext == "jpg": + return "image/jpeg" + return "image/%s" % ext + + +def _layer_bounds(config, layer): + return layer.get( + "data_bounding_box", + layer.get("bbox", config.default_bbox), + ) + + +def _tile_url(ref, layer_name, layer): + return "%swmts/%s/{TileMatrix}/{TileCol}/{TileRow}.%s" % ( + ref, + layer_name, + _extension(layer), + ) + + +def _supported_crs(proj): + epsg = proj.split(":")[-1] + if epsg.isdigit(): + return "urn:ogc:def:crs:EPSG::%s" % epsg + return proj + + +def _projected_bounds(proj): + bounds = projections.projs[projections.proj_alias.get(proj, proj)]["bounds"] + return projections.from4326(bounds, proj) + + +def _scale_denominator(proj, z): + projected = _projected_bounds(proj) + matrix_width = 2 ** z + resolution = (projected[2] - projected[0]) / (256 * matrix_width) + return resolution / 0.00028 + + +def _max_zoom_for_proj(config, proj): + zooms = [ + layer.get("max_zoom", config.default_max_zoom) + for layer in config.layers.values() + if layer.get("proj", "EPSG:3857") == proj + ] + return max(zooms or [config.default_max_zoom]) + + +def _add_operation(parent, name, href): + operation = ET.SubElement(parent, _tag(OWS, "Operation"), {"name": name}) + dcp = ET.SubElement(operation, _tag(OWS, "DCP")) + http = ET.SubElement(dcp, _tag(OWS, "HTTP")) + ET.SubElement( + http, + _tag(OWS, "Get"), + {_tag(XLINK, "href"): href}, + ) + + +def _add_layer(parent, config, layer_name, layer, ref): + layer_element = ET.SubElement(parent, _tag(WMTS, "Layer")) + ET.SubElement(layer_element, _tag(OWS, "Title")).text = layer["name"] + ET.SubElement(layer_element, _tag(OWS, "Identifier")).text = layer_name + + wgs84_bounds = ET.SubElement(layer_element, _tag(OWS, "WGS84BoundingBox")) + bounds = _layer_bounds(config, layer) + ET.SubElement(wgs84_bounds, _tag(OWS, "LowerCorner")).text = "%s %s" % ( + bounds[0], + bounds[1], + ) + ET.SubElement(wgs84_bounds, _tag(OWS, "UpperCorner")).text = "%s %s" % ( + bounds[2], + bounds[3], + ) + + style = ET.SubElement(layer_element, _tag(WMTS, "Style"), {"isDefault": "true"}) + ET.SubElement(style, _tag(OWS, "Identifier")).text = "default" + ET.SubElement(layer_element, _tag(WMTS, "Format")).text = _mime_type(layer) + + link = ET.SubElement(layer_element, _tag(WMTS, "TileMatrixSetLink")) + ET.SubElement(link, _tag(WMTS, "TileMatrixSet")).text = layer.get("proj", "EPSG:3857") + ET.SubElement( + layer_element, + _tag(WMTS, "ResourceURL"), + { + "format": _mime_type(layer), + "resourceType": "tile", + "template": _tile_url(ref, layer_name, layer), + }, + ) + + +def _add_tile_matrix_set(parent, config, proj): + projected = _projected_bounds(proj) + tile_matrix_set = ET.SubElement(parent, _tag(WMTS, "TileMatrixSet")) + ET.SubElement(tile_matrix_set, _tag(OWS, "Identifier")).text = proj + ET.SubElement(tile_matrix_set, _tag(OWS, "SupportedCRS")).text = _supported_crs(proj) + + for z in range(_max_zoom_for_proj(config, proj) + 1): + matrix_size = 2 ** z + tile_matrix = ET.SubElement(tile_matrix_set, _tag(WMTS, "TileMatrix")) + ET.SubElement(tile_matrix, _tag(OWS, "Identifier")).text = str(z) + ET.SubElement(tile_matrix, _tag(WMTS, "ScaleDenominator")).text = str( + _scale_denominator(proj, z) + ) + ET.SubElement(tile_matrix, _tag(WMTS, "TopLeftCorner")).text = "%s %s" % ( + projected[0], + projected[3], + ) + ET.SubElement(tile_matrix, _tag(WMTS, "TileWidth")).text = "256" + ET.SubElement(tile_matrix, _tag(WMTS, "TileHeight")).text = "256" + ET.SubElement(tile_matrix, _tag(WMTS, "MatrixWidth")).text = str(matrix_size) + ET.SubElement(tile_matrix, _tag(WMTS, "MatrixHeight")).text = str(matrix_size) + + +def capabilities(config, ref): + ET.register_namespace("", WMTS) + ET.register_namespace("ows", OWS) + ET.register_namespace("xlink", XLINK) + ET.register_namespace("xsi", XSI) + + root = ET.Element( + _tag(WMTS, "Capabilities"), + { + "version": "1.0.0", + _tag(XSI, "schemaLocation"): ( + "http://www.opengis.net/wmts/1.0 " + "http://schemas.opengis.net/wmts/1.0/wmtsGetCapabilities_response.xsd" + ), + }, + ) + service = ET.SubElement(root, _tag(OWS, "ServiceIdentification")) + ET.SubElement(service, _tag(OWS, "Title")).text = config.wms_name + ET.SubElement(service, _tag(OWS, "ServiceType")).text = "OGC WMTS" + ET.SubElement(service, _tag(OWS, "ServiceTypeVersion")).text = "1.0.0" + + operations = ET.SubElement(root, _tag(OWS, "OperationsMetadata")) + _add_operation( + operations, + "GetCapabilities", + "%swmts/1.0.0/WMTSCapabilities.xml" % ref, + ) + _add_operation(operations, "GetTile", ref) + + contents = ET.SubElement(root, _tag(WMTS, "Contents")) + projections_in_use = set() + for layer_name in sorted(config.layers.keys()): + layer = config.layers[layer_name] + proj = layer.get("proj", "EPSG:3857") + if proj not in projections.projs: + continue + projections_in_use.add(proj) + _add_layer(contents, config, layer_name, layer, ref) + + for proj in sorted(projections_in_use): + _add_tile_matrix_set(contents, config, proj) + + ET.SubElement( + root, + _tag(WMTS, "ServiceMetadataURL"), + {_tag(XLINK, "href"): "%swmts/1.0.0/WMTSCapabilities.xml" % ref}, + ) + + return '\n' + ET.tostring( + root, + encoding="unicode", + ) From 3989521de56314781fe4de989ec176714496a06f Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:10:58 +0400 Subject: [PATCH 14/51] feat: modernize tile cache semantics Keep TWMS' historical filesystem cache layout, but teach it the useful cache behavior from Radioxoma's fork: cache_ttl checks, reusable .tne markers, stale fallback after network errors, and atomic replacement for freshly downloaded tiles. Also accept Radioxoma-style dead_tile dictionaries while preserving the legacy dead_tile file comparison. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 131 +++++++++++++++++++ twms/fetchers.py | 254 ++++++++++++++++++++++++++----------- 2 files changed, 310 insertions(+), 75 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 5ae22da..fed88c6 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -1,22 +1,43 @@ import importlib import importlib.metadata +import hashlib import json +import os from io import BytesIO +import tempfile import threading import unittest import urllib.request from http.server import ThreadingHTTPServer import xml.etree.ElementTree as ET +from unittest import mock from PIL import Image import twms import twms.daemon +import twms.fetchers import twms.server import twms.twms class LegacySmokeTest(unittest.TestCase): + def image_bytes(self, color, image_format="PNG"): + buffer = BytesIO() + Image.new("RGBA", (256, 256), color).save(buffer, image_format) + return buffer.getvalue() + + def cache_path(self, cache_root, layer, z, x, y): + return os.path.join( + cache_root, + layer["prefix"], + "z%s" % z, + "%s" % (x // 1024), + "x%s" % x, + "%s" % (y // 1024), + "y%s.%s" % (y, layer["ext"]), + ) + def test_public_version_keeps_keyboard_suffix(self): self.assertEqual(twms.__version__, "0.07z") self.assertEqual(importlib.metadata.version("twms"), "0.7+z") @@ -153,6 +174,116 @@ def test_wmts_kvp_gettile_smoke(self): self.assertEqual(image.size, (256, 256)) self.assertEqual(image.mode, "RGBA") + def test_tile_cache_uses_fresh_file_without_network(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "ttl", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_ttl": 3600, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_not_called() + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_cache_refetches_expired_file_and_keeps_stale_on_error(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "ttl", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_ttl": 1, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + old_time = 946684800 + os.utime(path, (old_time, old_time)) + + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (40, 50, 60, 255) + ) + image = twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_called_once() + self.assertEqual(image.getpixel((0, 0)), (40, 50, 60, 255)) + + os.utime(path, (old_time, old_time)) + with mock.patch("twms.fetchers.urlopen", side_effect=OSError): + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.getpixel((0, 0)), (40, 50, 60, 255)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_cache_tne_suppresses_fetch_until_ttl_expires(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "ttl", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_ttl": 3600, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + os.makedirs(os.path.dirname(tne_path)) + open(tne_path, "wb").close() + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_not_called() + self.assertIsNone(image) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_dead_tile_dict_is_recorded_as_tne(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + body = self.image_bytes((255, 0, 0, 255)) + layer = { + "prefix": "dead", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "dead_tile": { + "size": len(body), + "md5": {hashlib.md5(body).hexdigest()}, + }, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = body + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertFalse(image) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(tne_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_wsgi_application_imports(self): self.assertTrue(callable(twms.daemon.application)) diff --git a/twms/fetchers.py b/twms/fetchers.py index 3ddda94..ca4c83b 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -6,6 +6,7 @@ # and/or modify it under the terms specified in COPYING. import filecmp +import hashlib import math import os import sys @@ -24,6 +25,114 @@ zhash_lock = {} +def _cache_stem(z, x, y, this_layer): + return ( + config.tiles_cache + + this_layer["prefix"] + + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) + ) + + +class TileCache: + """Small filesystem cache helper. + + This keeps TWMS' historical zN/NNN/xN/NNN/yN.ext layout, but adds the useful + cache semantics from Radioxoma's fork: TTL checks, readable TNE markers, + stale-cache fallback, and atomic file replacement. + """ + + def __init__(self, z, x, y, this_layer): + self.layer = this_layer + self.cached = this_layer.get("cached", True) + if self.cached: + self.stem = _cache_stem(z, x, y, this_layer) + self.path = self.stem + this_layer["ext"] + self.tne_path = self.stem + "tne" + self.lock_path = self.stem + "lock" + else: + self.stem = None + self.path = None + self.tne_path = None + self.lock_path = None + + def _fresh(self, path): + if not os.path.exists(path): + return False + ttl = self.layer.get("cache_ttl") + if not ttl: + return True + return time.time() - os.path.getmtime(path) <= ttl + + def needs_fetch(self): + if not self.cached: + return True + if self._fresh(self.tne_path): + return False + if self._fresh(self.path): + return False + return True + + def open_image(self, include_stale=False): + if not self.cached or not os.path.exists(self.path): + return None + if not include_stale and not self._fresh(self.path): + return None + return Image.open(self.path) + + def ensure_parent(self): + parent = os.path.dirname(self.stem) + if parent and not os.path.exists(parent): + os.makedirs(parent) + + def acquire(self): + if not self.cached: + return False + self.ensure_parent() + os.mkdir(self.lock_path) + return True + + def release(self): + if self.cached and os.path.exists(self.lock_path): + os.rmdir(self.lock_path) + + def wait_for_peer(self): + for _ in range(20): + time.sleep(0.1) + if not os.path.exists(self.lock_path): + if self._fresh(self.tne_path): + return None + return self.open_image() + return None + + def write_bytes(self, contents): + if not self.cached: + return + tmp_path = self.path + ".tmp.%s" % os.getpid() + with open(tmp_path, "wb") as tile_file: + tile_file.write(contents) + os.replace(tmp_path, self.path) + if os.path.exists(self.tne_path): + os.remove(self.tne_path) + + def save_image(self, image): + if not self.cached: + return + tmp_path = self.path + ".tmp.%s" % os.getpid() + image.save(tmp_path) + os.replace(tmp_path, self.path) + if os.path.exists(self.tne_path): + os.remove(self.tne_path) + + def mark_tne(self): + if not self.cached: + return + self.ensure_parent() + if os.path.exists(self.path): + os.remove(self.path) + with open(self.tne_path, "wb") as tne: + tne.write(b"") + + def fetch(z, x, y, this_layer): zhash = repr((z, x, y, this_layer)) try: @@ -69,51 +178,46 @@ def WMS(z, x, y, this_layer): req_proj = this_layer.get("wms_proj", this_layer["proj"]) width = 384 # using larger source size to rescale better in python height = 384 - local = ( - config.tiles_cache - + this_layer["prefix"] - + "/z%s/%s/x%s/%s/y%s." % (z, x / 1024, x, y / 1024, y) - ) + cache = TileCache(z, x, y, this_layer) tile_bbox = "bbox=%s,%s,%s,%s" % tuple( projections.from4326(projections.bbox_by_tile(z, x, y, req_proj), req_proj) ) wms += tile_bbox + "&width=%s&height=%s&srs=%s" % (width, height, req_proj) + if this_layer.get("cached", True) and not cache.needs_fetch(): + return cache.open_image() + locked = False if this_layer.get("cached", True): - if not os.path.exists("/".join(local.split("/")[:-1])): - os.makedirs("/".join(local.split("/")[:-1])) try: - os.mkdir(local + "lock") + locked = cache.acquire() except OSError: - for i in range(20): - time.sleep(0.1) - try: - if not os.path.exists(local + "lock"): - im = Image.open(local + this_layer["ext"]) - return im - except (IOError, OSError): - return None - im = Image.open(BytesIO(urlopen(wms).read())) - if width != 256 and height != 256: - im = im.resize((256, 256), Image.ANTIALIAS) - im = im.convert("RGBA") + return cache.wait_for_peer() + try: + try: + im = Image.open(BytesIO(urlopen(wms).read())) + except OSError: + stale = cache.open_image(include_stale=True) + if stale is not None: + return stale + return False + if width != 256 and height != 256: + im = im.resize((256, 256), Image.ANTIALIAS) + im = im.convert("RGBA") - if this_layer.get("cached", True): - ic = Image.new( - "RGBA", (256, 256), this_layer.get("empty_color", config.default_background) - ) - if im.histogram() == ic.histogram(): - tne = open(local + "tne", "wb") - when = time.localtime() - tne.write( - "%02d.%02d.%04d %02d:%02d:%02d" - % (when[2], when[1], when[0], when[3], when[4], when[5]) + if this_layer.get("cached", True): + ic = Image.new( + "RGBA", + (256, 256), + this_layer.get("empty_color", config.default_background), ) - tne.close() - return False - im.save(local + this_layer["ext"]) - os.rmdir(local + "lock") - return im + if im.histogram() == ic.histogram(): + cache.mark_tne() + return False + cache.save_image(im) + return im + finally: + if locked: + cache.release() def Tile(z, x, y, this_layer): @@ -126,49 +230,49 @@ def Tile(z, x, y, this_layer): d_tuple = this_layer["transform_tile_number"](z, x, y) remote = this_layer["remote_url"] % d_tuple + cache = TileCache(z, x, y, this_layer) + if this_layer.get("cached", True) and not cache.needs_fetch(): + return cache.open_image() + locked = False if this_layer.get("cached", True): - local = ( - config.tiles_cache - + this_layer["prefix"] - + "/z%s/%s/x%s/%s/y%s." % (z, x / 1024, x, y / 1024, y) - ) - if not os.path.exists("/".join(local.split("/")[:-1])): - os.makedirs("/".join(local.split("/")[:-1])) try: - os.mkdir(local + "lock") + locked = cache.acquire() except OSError: - for i in range(20): - time.sleep(0.1) - try: - if not os.path.exists(local + "lock"): - im = Image.open(local + this_layer["ext"]) - return im - except (IOError, OSError): - return None + return cache.wait_for_peer() try: - contents = urlopen(remote).read() - im = Image.open(BytesIO(contents)) - except IOError: - if this_layer.get("cached", True): - os.rmdir(local + "lock") - return False - if this_layer.get("cached", True): - os.rmdir(local + "lock") - open(local + this_layer["ext"], "wb").write(contents) - if "dead_tile" in this_layer: try: - dt = open(this_layer["dead_tile"], "rb").read() - if contents == dt: - if this_layer.get("cached", True): - tne = open(local + "tne", "wb") - when = time.localtime() - tne.write( - "%02d.%02d.%04d %02d:%02d:%02d" - % (when[2], when[1], when[0], when[3], when[4], when[5]) - ) - tne.close() - os.remove(local + this_layer["ext"]) + contents = urlopen(remote).read() + im = Image.open(BytesIO(contents)) + except OSError: + stale = cache.open_image(include_stale=True) + if stale is not None: + return stale + return False + if "dead_tile" in this_layer and _is_dead_tile(contents, this_layer["dead_tile"]): + cache.mark_tne() return False - except IOError: - pass - return im + if this_layer.get("cached", True): + cache.write_bytes(contents) + return im + finally: + if locked: + cache.release() + + +def _is_dead_tile(contents, dead_tile): + if isinstance(dead_tile, dict): + if "size" in dead_tile and len(contents) != dead_tile["size"]: + return False + if "md5" in dead_tile: + md5 = hashlib.md5(contents).hexdigest() + if md5 not in dead_tile["md5"]: + return False + if "sha256" in dead_tile: + sha256 = hashlib.sha256(contents).hexdigest() + if sha256 != dead_tile["sha256"]: + return False + return True + try: + return contents == open(dead_tile, "rb").read() + except IOError: + return False From c7c3b494637a771d81aacec565149d2223316081 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:17:24 +0400 Subject: [PATCH 15/51] fix: modernize projection fallback Keep TWMS' built-in projection shortlist tiny, but make the core WebMercator path tolerate pole-edge bounds and route optional pyproj transforms through the modern Transformer API. This deliberately preserves the existing optional twms[proj] behavior for non-core projections instead of taking Radioxoma's hard pyproj removal. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 39 ++++++++++++++++++++++++++++++++++++++ twms/projections.py | 35 ++++++++++++++++++++++++---------- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index fed88c6..8c35e7f 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -2,12 +2,14 @@ import importlib.metadata import hashlib import json +import math import os from io import BytesIO import tempfile import threading import unittest import urllib.request +import warnings from http.server import ThreadingHTTPServer import xml.etree.ElementTree as ET from unittest import mock @@ -17,6 +19,7 @@ import twms import twms.daemon import twms.fetchers +import twms.projections import twms.server import twms.twms @@ -174,6 +177,42 @@ def test_wmts_kvp_gettile_smoke(self): self.assertEqual(image.size, (256, 256)) self.assertEqual(image.mode, "RGBA") + def test_webmercator_projection_clamps_poles(self): + maxbounds = 6378137 * math.pi + + projected = twms.projections.from4326( + (-180.0, -90.0, 180.0, 90.0), + "EPSG:3857", + ) + + self.assertEqual(projected[0], -maxbounds) + self.assertEqual(projected[1], -maxbounds) + self.assertEqual(projected[2], maxbounds) + self.assertEqual(projected[3], maxbounds) + + def test_optional_pyproj_projection_uses_modern_transformer(self): + if not hasattr(twms.projections.pyproj, "Transformer"): + self.skipTest("pyproj extra is not installed") + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + point = twms.projections.from4326((27.6, 53.2), "EPSG:32635") + + self.assertTrue(540000 < point[0] < 541000) + self.assertTrue(5894000 < point[1] < 5895000) + self.assertFalse( + [warning for warning in caught if warning.category is FutureWarning], + ) + + def test_non_core_projection_reports_missing_pyproj_extra(self): + if hasattr(twms.projections.pyproj, "Transformer"): + self.skipTest("pyproj extra is installed") + + with self.assertRaises(NotImplementedError) as raised: + twms.projections.from4326((27.6, 53.2), "EPSG:32635") + + self.assertIn("twms[proj]", str(raised.exception)) + def test_tile_cache_uses_fresh_file_without_network(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/projections.py b/twms/projections.py index d6316bb..92a7dbe 100644 --- a/twms/projections.py +++ b/twms/projections.py @@ -15,15 +15,23 @@ class Proj: def __init__(self, pstring): self.pstring = pstring - def transform(self, pr1, pr2, c1, c2): + @staticmethod + def transform(pr1, pr2, c1, c2): if pr1.pstring == pr2.pstring: return c1, c2 else: raise NotImplementedError( - "Pyproj is not installed - can't convert between projectios. Install pyproj please." + "Pyproj is not installed - can't convert between projections. Install twms[proj] please." ) +def _pyproj_transformer(pr1, pr2): + if hasattr(pyproj, "Transformer"): + transformer = pyproj.Transformer.from_proj(pr1, pr2, always_xy=True) + return lambda _pr1, _pr2, c1, c2: transformer.transform(c1, c2) + return pyproj.transform + + projs = { "EPSG:4326": { "proj": pyproj.Proj("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"), @@ -101,13 +109,17 @@ def _c4326t3857(t1, t2, lon, lat): """ Pure python 4326 -> 3857 transform. About 8x faster than pyproj. """ + maxbounds = 6378137 * math.pi + xtile = maxbounds / 180 * lon lat_rad = math.radians(lat) - xtile = lon * 111319.49079327358 - ytile = ( - math.log(math.tan(lat_rad) + (1 / math.cos(lat_rad))) - / math.pi - * 20037508.342789244 - ) + if abs(lat) <= 85.0511287798: + ytile = ( + math.log(math.tan(lat_rad) + (1 / math.cos(lat_rad))) + / math.pi + * maxbounds + ) + else: + ytile = math.copysign(maxbounds, lat) return (xtile, ytile) @@ -269,9 +281,10 @@ def transform(line, srs1, srs2): if (srs1, srs2) in pure_python_transformers: func = pure_python_transformers[(srs1, srs2)] # print("pure") + uses_pyproj = False else: - - func = pyproj.transform + func = None + uses_pyproj = True line = list(line) serial = False if (not isinstance(line[0], tuple)) and (not isinstance(line[0], list)): @@ -285,6 +298,8 @@ def transform(line, srs1, srs2): ans = [] pr1 = projs[srs1]["proj"] pr2 = projs[srs2]["proj"] + if uses_pyproj: + func = _pyproj_transformer(pr1, pr2) for point in line: p = func(pr1, pr2, point[0], point[1]) if serial: From 317dc88d9acd44f95a9f7469b5eb5549344a15a0 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:25:04 +0400 Subject: [PATCH 16/51] feat: expose WMS 1.3 capabilities Harvest Radioxoma's WMS 1.3.0 capability work without taking the fork's API/config rewrite or legacy module removals. Keep the existing WMS 1.1.1 generator intact, add a tiny ElementTree WMS 1.3.0 response, accept case-insensitive request keys, and support CRS:84 as the lon/lat WMS 1.3.0 request alias without advertising it as a WMS 1.1.1 SRS. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 48 ++++++++++++++ twms/capabilities.py | 130 +++++++++++++++++++++++++++++++++++-- twms/projections.py | 6 +- twms/twms.py | 20 +++--- 4 files changed, 188 insertions(+), 16 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 8c35e7f..dfdfc9d 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -76,6 +76,54 @@ def test_wms_capabilities_smoke(self): self.assertEqual(content_type, "application/vnd.ogc.wms_xml") self.assertIn("CRS:84", body) + + def test_wms_130_capabilities_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "SERVICE": "WMS", + "REQUEST": "GetCapabilities", + "VERSION": "1.3.0", + "ref": "http://example.test/wms", + } + ) + + root = ET.fromstring(body) + namespaces = {"wms": "http://www.opengis.net/wms"} + osm = root.find( + "./wms:Capability/wms:Layer/wms:Layer[wms:Name='osm']", + namespaces, + ) + crs_values = [element.text for element in osm.findall("wms:CRS", namespaces)] + bbox = osm.find("wms:BoundingBox", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(root.tag, "{http://www.opengis.net/wms}WMS_Capabilities") + self.assertEqual(root.attrib["version"], "1.3.0") + self.assertIn("CRS:84", crs_values) + self.assertIn("EPSG:3857", crs_values) + self.assertEqual(bbox.attrib["CRS"], "EPSG:3857") + self.assertLess(float(bbox.attrib["minx"]), -20000000) + self.assertGreater(float(bbox.attrib["maxx"]), 20000000) + + def test_wms_getmap_accepts_crs_parameter(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "transparent", + "format": "image/png", + "width": "32", + "height": "32", + "crs": "CRS:84", + "bbox": "-1,-1,1,1", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (32, 32)) def test_overview_smoke(self): status, content_type, body = twms.twms.twms_main({"ref": "http://example.test/"}) diff --git a/twms/capabilities.py b/twms/capabilities.py index 621e2a8..033f091 100644 --- a/twms/capabilities.py +++ b/twms/capabilities.py @@ -5,13 +5,135 @@ # the extent permitted by applicable law. You can redistribute it # and/or modify it under the terms specified in COPYING. +import xml.etree.ElementTree as ET + import config import projections +WMS = "http://www.opengis.net/wms" +XLINK = "http://www.w3.org/1999/xlink" + + +def _wms130_bbox(layer): + bbox = layer.get("bbox", config.default_bbox) + proj = layer.get("proj", "EPSG:3857") + if projections.proj_alias.get(proj, proj) == "EPSG:4326": + return "CRS:84", bbox + return proj, projections.from4326(bbox, proj) + + +def _legacy_srs_ids(): + return sorted( + proj + for proj in projections.projs.keys() | projections.proj_alias.keys() + if not proj.startswith("CRS:") + ) + + +def _wms130(ref): + content_type = "text/xml" + + ET.register_namespace("", WMS) + ET.register_namespace("xlink", XLINK) + + root = ET.Element( + "{%s}WMS_Capabilities" % WMS, + attrib={"version": "1.3.0"}, + ) + service = ET.SubElement(root, "Service") + ET.SubElement(service, "Name").text = "WMS" + ET.SubElement(service, "Title").text = config.wms_name + ET.SubElement( + service, + "OnlineResource", + attrib={ + "{%s}type" % XLINK: "simple", + "{%s}href" % XLINK: ref, + }, + ) + ET.SubElement(service, "Fees").text = "none" + ET.SubElement(service, "AccessConstraints").text = "none" + + capability = ET.SubElement(root, "Capability") + request = ET.SubElement(capability, "Request") + get_capabilities = ET.SubElement(request, "GetCapabilities") + ET.SubElement(get_capabilities, "Format").text = "text/xml" + ET.SubElement( + ET.SubElement( + ET.SubElement(ET.SubElement(get_capabilities, "DCPType"), "HTTP"), + "Get", + ), + "OnlineResource", + attrib={ + "{%s}type" % XLINK: "simple", + "{%s}href" % XLINK: ref, + }, + ) + + get_map = ET.SubElement(request, "GetMap") + for image_format in ("image/png", "image/jpeg", "image/gif", "image/bmp"): + ET.SubElement(get_map, "Format").text = image_format + ET.SubElement( + ET.SubElement(ET.SubElement(ET.SubElement(get_map, "DCPType"), "HTTP"), "Get"), + "OnlineResource", + attrib={ + "{%s}type" % XLINK: "simple", + "{%s}href" % XLINK: ref, + }, + ) + + exceptions = ET.SubElement(capability, "Exception") + ET.SubElement(exceptions, "Format").text = "XML" + + parent = ET.SubElement(capability, "Layer") + ET.SubElement(parent, "Title").text = config.wms_name + ET.SubElement(parent, "CRS").text = "CRS:84" + west, south, east, north = config.default_bbox + geo_bbox = ET.SubElement(parent, "EX_GeographicBoundingBox") + ET.SubElement(geo_bbox, "westBoundLongitude").text = str(west) + ET.SubElement(geo_bbox, "eastBoundLongitude").text = str(east) + ET.SubElement(geo_bbox, "southBoundLatitude").text = str(south) + ET.SubElement(geo_bbox, "northBoundLatitude").text = str(north) + + for layer_id in config.layers.keys(): + layer_config = config.layers[layer_id] + layer = ET.SubElement(parent, "Layer", attrib={"queryable": "0", "opaque": "1"}) + ET.SubElement(layer, "Name").text = layer_id + ET.SubElement(layer, "Title").text = layer_config["name"] + ET.SubElement(layer, "CRS").text = "CRS:84" + bbox = layer_config.get("bbox", config.default_bbox) + west, south, east, north = bbox + geo_bbox = ET.SubElement(layer, "EX_GeographicBoundingBox") + ET.SubElement(geo_bbox, "westBoundLongitude").text = str(west) + ET.SubElement(geo_bbox, "eastBoundLongitude").text = str(east) + ET.SubElement(geo_bbox, "southBoundLatitude").text = str(south) + ET.SubElement(geo_bbox, "northBoundLatitude").text = str(north) + + bbox_crs, native_bbox = _wms130_bbox(layer_config) + ET.SubElement(layer, "CRS").text = bbox_crs + ET.SubElement( + layer, + "BoundingBox", + attrib={ + "CRS": bbox_crs, + "minx": str(native_bbox[0]), + "miny": str(native_bbox[1]), + "maxx": str(native_bbox[2]), + "maxy": str(native_bbox[3]), + }, + ) + + ET.indent(root) + return content_type, ET.tostring(root, encoding="unicode", xml_declaration=True) + + def get(version, ref): content_type = "text/xml" + if version == "1.3.0": + return _wms130(ref) + if version == "1.0.0": req = ( """ @@ -103,9 +225,7 @@ def get(version, ref): + """ """ ) - pset = set(projections.projs.keys()) - pset = pset.union(set(projections.proj_alias.keys())) - for proj in pset: + for proj in _legacy_srs_ids(): req += "%s" % proj req += """ @@ -216,9 +336,7 @@ def get(version, ref): World Map""" ) - pset = set(projections.projs.keys()) - pset = pset.union(set(projections.proj_alias.keys())) - for proj in pset: + for proj in _legacy_srs_ids(): req += "%s" % proj req += """ diff --git a/twms/projections.py b/twms/projections.py index 92a7dbe..557ccc0 100644 --- a/twms/projections.py +++ b/twms/projections.py @@ -102,7 +102,11 @@ def _pyproj_transformer(pr1, pr2): "bounds": (-180.0, -90.0, 180.0, 90.0), }, } -proj_alias = {"EPSG:900913": "EPSG:3857", "EPSG:3785": "EPSG:3857"} +proj_alias = { + "CRS:84": "EPSG:4326", + "EPSG:900913": "EPSG:3857", + "EPSG:3785": "EPSG:3857", +} def _c4326t3857(t1, t2, lon, lat): diff --git a/twms/twms.py b/twms/twms.py index 5b0c074..19b541d 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -67,6 +67,7 @@ def twms_main(data): data - dictionary of params. returns (error_code, content_type, resp) """ + data = dict((key.lower(), data[key]) for key in data) # import the filter here due to a circular dependency # TODO: break the loop import filter @@ -75,7 +76,7 @@ def twms_main(data): content_type = "text/html" resp = "" - srs = data.get("srs", "EPSG:4326") + srs = data.get("crs", data.get("srs", "EPSG:4326")) gpx = data.get("gpx", "").split(",") if gpx == [""]: gpx = [] @@ -106,13 +107,14 @@ def twms_main(data): tracks.append(track) req_type = data.get("request", "GetMap") + req_type_lower = req_type.lower() version = data.get("version", "1.1.1") ref = data.get("ref", config.service_url) - if data.get("service", "").lower() == "wmts" and req_type.lower() == "getcapabilities": + if data.get("service", "").lower() == "wmts" and req_type_lower == "getcapabilities": return (OK, "text/xml", wmts.capabilities(config, ref)) - if req_type.lower() == "getwmtscapabilities": + if req_type_lower == "getwmtscapabilities": return (OK, "text/xml", wmts.capabilities(config, ref)) - if data.get("service", "").lower() == "wmts" and req_type.lower() == "gettile": + if data.get("service", "").lower() == "wmts" and req_type_lower == "gettile": if "layers" not in data and "layer" in data: data["layers"] = data["layer"] if "z" not in data and "tilematrix" in data: @@ -121,10 +123,10 @@ def twms_main(data): data["x"] = data["tilecol"] if "y" not in data and "tilerow" in data: data["y"] = data["tilerow"] - if req_type == "GetCapabilities": + if req_type_lower == "getcapabilities": content_type, resp = capabilities.get(version, ref) return (OK, content_type, resp) - if req_type.lower() in ("gettilejson", "tilejson"): + if req_type_lower in ("gettilejson", "tilejson"): try: resp = tilejson.dumps( config, @@ -140,7 +142,7 @@ def twms_main(data): if ("layers" in data) and not layer[0]: layer = ["transparent"] - if req_type == "GetCorrections": + if req_type_lower == "getcorrections": points = data.get("points", data.get("POINTS", "")).split("=") resp = "" points = [a.split(",") for a in points] @@ -178,12 +180,12 @@ def twms_main(data): width = 0 height = 0 resp_cache_path, resp_ext = "", "" - if req_type == "GetTile": + if req_type_lower == "gettile": width = 256 height = 256 height = int(data.get("height", height)) width = int(data.get("width", width)) - srs = data.get("srs", "EPSG:3857") + srs = data.get("crs", data.get("srs", "EPSG:3857")) x = int(data.get("x", 0)) y = int(data.get("y", 0)) z = int(data.get("z", 1)) + 1 From 4aa6b65643d8c10b678d68707c7ec1d06560274b Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:30:48 +0400 Subject: [PATCH 17/51] docs: document modern client setup Harvest the useful deployment and client guidance from Radioxoma's README without taking the fork framing, unsupported-feature removals, or repo layout rewrite. Document the current upstream entry points, QGIS WMS/WMTS URLs, JOSM TMS/GetTile/file-cache forms, cache behavior, and optional dependencies while keeping TWMS described as a general tiny WMS/tile service. Co-authored-by: Eugene Dvoretsky --- README.md | 208 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 189 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 2668174..d74201e 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,202 @@ -About -===== +# twms -twms is a script that connects World of Tiles and World of WMS. -The name ‘twms’ stands for twms web map server. +twms is a tiny web map service that connects the world of tiles and +the world of WMS. -The primary purpose of twms is to export your map tiles to the -WMS-enabled applications. +The primary purpose of twms is to export raster tile sets to +WMS-enabled GIS applications. It can also act as a small tile proxy: +fetching remote tiles, keeping a filesystem cache, and serving that +cache back through WMS, WMTS, TileJSON, or direct slippy-map tile URLs. -twms can export a set of raster tiles as a WMS service -so GIS applications that support WMS protocol can access -this tile set. Also, twms can act as a proxy and perform -WMS requests to external services and serve the tile cache +twms intentionally stays small. The default server uses the Python +standard library, while the older `web.py` WSGI/standalone path is +still available for deployments that already use it. + +## Install and run + +Install from a checkout: + +```sh +python -m pip install -e . +``` + +Run the default stdlib server: + +```sh +twms 8080 +``` + +or: + +```sh +python -m twms 8080 +``` + +The previous `web.py` server and WSGI adapter are still installed as: + +```sh +twms-webpy +``` + +That path is kept for older deployments, including Windows/JOSM proxy +bundles that depended on `web.py`. + +## Configuration + +twms loads Python configuration from: + +1. `/etc/twms/twms.conf` +2. the packaged `twms/twms.conf` +3. `twms.conf` in the current script directory + +Important settings: + +- `tiles_cache`: root directory for the filesystem tile cache +- `gpx_cache`: cache for downloaded OSM GPX traces +- `service_url`: externally visible base URL used in generated capabilities +- `default_layers`: layer list used when a request does not name layers +- `default_format`: output image MIME type, usually `image/jpeg` +- `layers`: configured imagery layers and their fetchers + +Layer dictionaries usually define: + +- `name`: human readable title +- `prefix`: cache subdirectory +- `ext`: tile extension such as `jpg` or `png` +- `proj`: tile pyramid projection, commonly `EPSG:3857` or `EPSG:3395` +- `remote_url`: upstream tile URL template +- `fetch`: fetcher function, normally `fetchers.Tile` +- `min_zoom` / `max_zoom`: optional zoom limits +- `cache_ttl`: optional fresh-cache lifetime in seconds +- `dead_tile`: optional dead-tile marker, either a legacy file path or a + `{ "size": ..., "md5": {...} }` dictionary + +## Client URLs + +Assuming the server runs at `http://127.0.0.1:8080/`: + +- overview page: + `http://127.0.0.1:8080/` +- WMS 1.1.1/1.3.0 endpoint: + `http://127.0.0.1:8080/` +- WMS capabilities: + `http://127.0.0.1:8080/?service=WMS&request=GetCapabilities&version=1.3.0` +- WMTS capabilities: + `http://127.0.0.1:8080/wmts/1.0.0/WMTSCapabilities.xml` +- TileJSON for a layer: + `http://127.0.0.1:8080/tilejson/osm.json` +- direct tile URL: + `http://127.0.0.1:8080/osm/{z}/{x}/{y}.png` +- WMTS REST tile URL: + `http://127.0.0.1:8080/wmts/osm/{z}/{x}/{y}.png` + +The legacy `GetTile` request is still supported because it is useful +when clients need filters or other TWMS-specific request parameters in +the tile URL: + +```text +http://127.0.0.1:8080/?request=GetTile&layers=osm&z={z}&x={x}&y={y}&format=png +``` + +## QGIS + +For WMS, create a WMS/WMTS connection pointing at: + +```text +http://127.0.0.1:8080/ +``` + +QGIS may request WMS 1.3.0 capabilities with uppercase parameter names +such as `SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0`; twms +accepts those. WMS 1.3.0 also accepts the `crs` parameter and advertises +`CRS:84` for lon/lat bounds, while the old WMS 1.1.1 capabilities keep +their existing `SRS` listings. + +For WMTS, use: + +```text +http://127.0.0.1:8080/wmts/1.0.0/WMTSCapabilities.xml +``` + +## JOSM + +For a normal local proxy, add a TMS imagery entry such as: + +```text +tms:http://127.0.0.1:8080/osm/{zoom}/{x}/{y}.png +``` + +For TWMS-specific parameters, use the WMS-style `GetTile` URL instead: + +```text +tms:http://127.0.0.1:8080/?request=GetTile&layers=osm&z={zoom}&x={x}&y={y}&format=png +``` + +JOSM can also point directly at a compatible local slippy-map cache with +`file://` if no proxy or reprojection is needed: + +```text +tms:file:///home/user/SAS.Planet/cache_ma/osm/{zoom}/{x}/{y}.png +``` + +On Windows the same idea uses a Windows path: + +```text +tms:file:///C:/SAS.Planet/cache_ma/osm/{zoom}/{x}/{y}.png +``` + +## Shared tile caches + +twms keeps the historical filesystem cache layout under `tiles_cache`: + +```text +//z//x//y. +``` + +Fresh cached tiles are served without network access. When `cache_ttl` +expires, twms tries to refresh the tile; if the remote fetch fails, the +stale cached tile can still be used. Missing/dead tiles can be recorded +as `.tne` files so repeated requests do not hammer upstream services. + +This makes twms useful with tools that share a slippy-map/MOBAC-style +cache, including SAS.Planet and similar offline tile workflows. + +## Optional dependencies + +The base install keeps dependencies small: + +- `Pillow` +- `web.py` for the legacy WSGI/standalone server path + +Optional extras: + +```sh +python -m pip install -e '.[proj]' +python -m pip install -e '.[cairo]' +``` + +`twms[proj]` enables pyproj-backed transformations for configured +non-core projections. Without it, twms still has built-in pure-Python +support for the common EPSG:4326/EPSG:3857/EPSG:3395 path. + +`twms[cairo]` enables Cairo-backed vector rendering where the old +rendering path uses it. ## Credits twms was originally written by Darafei Praliaskouski (Komzpa). Andrew Shadura maintains Debian packaging and wrote the Debian manpage. Eugene Dvoretsky (Radioxoma) contributed modernization work around -packaging, serving, caching, and tile protocols. +packaging, serving, caching, projections, documentation, and tile +protocols. -TODO -==== +## TODO - - Make fetchers work with proxy - - Full reprojection support - - Imagery realignment +- Make fetchers work with proxy. +- Full reprojection support. +- Imagery realignment. -Conventions -=========== +## Conventions - - Inside tWMS, only EPSG:4326 latlon should be used for transmitting coordinates. +- Inside twms, EPSG:4326 lon/lat should be used for transmitting + coordinates. From daed6f68f549bbdc5eaa1ec18bd477fa9e58a5d0 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:33:47 +0400 Subject: [PATCH 18/51] ci: build Windows executable artifacts Preserve the Windows/JOSM single-executable deployment path without adding runtime framework code to TWMS. Build both the stdlib server entry point and the preserved web.py entry point with PyInstaller in GitHub Actions, upload the executables as CI artifacts, and run the workflow for branches, tags, releases, PRs, and manual dispatch. Co-authored-by: Eugene Dvoretsky --- .github/workflows/ci.yml | 51 ++++++++++++++++++++++++++++++++++++++++ README.md | 5 ++++ 2 files changed, 56 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e6feca..a983747 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,12 @@ on: branches: - master - "ai/**" + tags: + - "*" pull_request: + release: + types: + - published workflow_dispatch: jobs: @@ -55,3 +60,49 @@ jobs: - name: Build source and wheel distributions run: python -m build + + windows-exe: + name: Windows executable artifact + runs-on: windows-latest + needs: package-smoke + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install build tools + run: python -m pip install --upgrade pip pyinstaller + + - name: Install package + run: python -m pip install -e . + + - name: Write PyInstaller entry points + shell: pwsh + run: | + @' + from twms.server import main + + if __name__ == "__main__": + main() + '@ | Set-Content -Encoding UTF8 twms-stdlib-entry.py + + @' + from twms.daemon import main + + if __name__ == "__main__": + main() + '@ | Set-Content -Encoding UTF8 twms-webpy-entry.py + + - name: Build executables + run: | + python -m PyInstaller --onefile --name twms twms-stdlib-entry.py + python -m PyInstaller --onefile --name twms-webpy twms-webpy-entry.py + + - name: Upload executable artifact + uses: actions/upload-artifact@v4 + with: + name: twms-windows-exe-${{ github.sha }} + path: dist/*.exe diff --git a/README.md b/README.md index d74201e..ec5c5ff 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,11 @@ twms-webpy That path is kept for older deployments, including Windows/JOSM proxy bundles that depended on `web.py`. +On GitHub, CI builds Windows executable artifacts for both entry points: + +- `twms.exe` +- `twms-webpy.exe` + ## Configuration twms loads Python configuration from: From 35c847e0dd4b05902b29c77c559d3187b99ca2f6 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:38:38 +0400 Subject: [PATCH 19/51] fix: preserve legacy rendering helpers Do not take Radioxoma's hard removal of filters, corrections, canvas, and reprojection helpers. Instead keep the legacy surfaces covered and fix current Python/Pillow breakage: use a Pillow resampling compatibility helper, keep canvas usable on Python 3, keep GetCorrections returning text/plain, pass WKT colors correctly when no color parameter is provided, and use Python 3 iterator semantics for GPX colors. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 78 ++++++++++++++++++++++++++++++++++++++ twms/__init__.py | 1 + twms/canvas.py | 6 +-- twms/drawing.py | 2 +- twms/fetchers.py | 3 +- twms/image_compat.py | 13 +++++++ twms/twms.py | 29 ++++++++------ 7 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 twms/image_compat.py diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index dfdfc9d..d7026e5 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -1,3 +1,4 @@ +import datetime import importlib import importlib.metadata import hashlib @@ -17,8 +18,10 @@ from PIL import Image import twms +import twms.canvas import twms.daemon import twms.fetchers +import twms.filter import twms.projections import twms.server import twms.twms @@ -53,6 +56,7 @@ def test_legacy_modules_import_as_package_modules(self): "twms.drawing", "twms.filter", "twms.gpxparse", + "twms.image_compat", "twms.overview", "twms.projections", "twms.reproject", @@ -151,6 +155,80 @@ def test_gettile_transparent_layer_smoke(self): self.assertEqual(image.size, (256, 256)) self.assertEqual(image.mode, "RGBA") + def test_legacy_getcorrections_without_rectify_file(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetCorrections", + "layers": "osm", + "points": "27.6,53.2", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/plain") + self.assertEqual(body, "27.6,53.2;\n") + + def test_legacy_filter_smoke(self): + image = Image.new("RGBA", (2, 1), (10, 20, 30, 255)) + + filtered = twms.filter.raster(image, ("swaprb", "brightness:2")) + + self.assertEqual(filtered.getpixel((0, 0)), (60, 40, 20, 255)) + + def test_legacy_wkt_drawing_without_color_parameter(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "transparent", + "format": "image/png", + "width": "32", + "height": "32", + "bbox": "-1,-1,1,1", + "wkt": "LINESTRING(-1 -1,1 1)", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (32, 32)) + self.assertTrue( + any(image.getchannel("A").tobytes()), + "WKT overlay should draw visible pixels", + ) + + def test_legacy_canvas_blank_tile_smoke(self): + canvas = twms.canvas.WmsCanvas(tile_size=(32, 32)) + + canvas.FetchTile(0, 0) + + self.assertEqual(canvas.tiles[(0, 0)]["im"].size, (32, 32)) + self.assertEqual(canvas.tiles[(0, 0)]["im"].mode, "RGBA") + + def test_getimg_resize_works_with_current_pillow(self): + tile = Image.new("RGBA", (256, 256), (1, 2, 3, 255)) + tile.is_ok = True + layer = { + "name": "Resize smoke", + "prefix": "resize", + "proj": "EPSG:3857", + "cached": False, + "max_zoom": 1, + } + + with mock.patch("twms.twms.tile_image", return_value=tile): + image = twms.twms.getimg( + (-1.0, -1.0, 1.0, 1.0), + "EPSG:3857", + (32, 32), + layer, + datetime.datetime.now(), + (), + ) + + self.assertEqual(image.size, (32, 32)) + self.assertEqual(image.getpixel((0, 0)), (1, 2, 3, 255)) + def test_tilejson_smoke(self): status, content_type, body = twms.twms.twms_main( { diff --git a/twms/__init__.py b/twms/__init__.py index e7bf4a8..38f0e46 100644 --- a/twms/__init__.py +++ b/twms/__init__.py @@ -29,6 +29,7 @@ def __getattr__(name): "canvas", "correctify", "drawing", + "image_compat", "projections", "reproject", "wmts", diff --git a/twms/canvas.py b/twms/canvas.py index 5e01726..7a09cae 100644 --- a/twms/canvas.py +++ b/twms/canvas.py @@ -15,7 +15,7 @@ import datetime import sys import threading -import urllib +from urllib.request import urlopen from io import BytesIO import projections @@ -105,7 +105,7 @@ def FetchTile(self, x, y): remote = self.ConstructTileUrl(x, y) debug(remote) ttz = datetime.datetime.now() - contents = urllib.urlopen(remote).read() + contents = urlopen(remote).read() debug("Download took %s" % str(datetime.datetime.now() - ttz)) im = Image.open(BytesIO(contents)) if im.mode != self.mode: @@ -133,7 +133,7 @@ def PreparePixel(self, x, y): group=None, target=self.FetchTile, name=None, - args=(self, tile_x, tile_y), + args=(tile_x, tile_y), kwargs={}, ) self.tiles[(tile_x, tile_y)]["thread"].start() diff --git a/twms/drawing.py b/twms/drawing.py index f1960e6..acd5ef8 100644 --- a/twms/drawing.py +++ b/twms/drawing.py @@ -104,7 +104,7 @@ def render_vector( if renderer == "cairo" and HAVE_CAIRO: "rendering as cairo" - imgd = img.tostring() + imgd = img.tobytes() a = array.array("B", imgd) surface = cairo.ImageSurface.create_for_data( a, cairo.FORMAT_ARGB32, W, H, W * 4 diff --git a/twms/fetchers.py b/twms/fetchers.py index ca4c83b..52de29a 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -18,6 +18,7 @@ import config import projections from PIL import Image +from twms.image_compat import resampling_lanczos fetching_now = {} @@ -201,7 +202,7 @@ def WMS(z, x, y, this_layer): return stale return False if width != 256 and height != 256: - im = im.resize((256, 256), Image.ANTIALIAS) + im = im.resize((256, 256), resampling_lanczos(Image)) im = im.convert("RGBA") if this_layer.get("cached", True): diff --git a/twms/image_compat.py b/twms/image_compat.py new file mode 100644 index 0000000..d82a8fe --- /dev/null +++ b/twms/image_compat.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# This file is part of twms. + +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms specified in COPYING. + + +def resampling_lanczos(Image): + """Return the best Pillow downsampling filter across old and new Pillow.""" + if hasattr(Image, "Resampling"): + return Image.Resampling.LANCZOS + return Image.ANTIALIAS diff --git a/twms/twms.py b/twms/twms.py index 19b541d..98e1c52 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -34,6 +34,7 @@ from gpxparse import GPXParser from PIL import Image, ImageColor, ImageOps from reproject import reproject +from twms.image_compat import resampling_lanczos try: @@ -82,7 +83,11 @@ def twms_main(data): gpx = [] wkt = data.get("wkt", "") trackblend = float(data.get("trackblend", "0.5")) - color = data.get("color", data.get("colour", "")).split(",") + colors = [ + value + for value in data.get("color", data.get("colour", "")).split(",") + if value + ] track = False tracks = [] if len(gpx) == 0: @@ -148,7 +153,7 @@ def twms_main(data): points = [a.split(",") for a in points] points = [(float(a[0]), float(a[1])) for a in points] - req.content_type = "text/plain" + content_type = "text/plain" for lay in layer: for point in points: resp += "%s,%s;" % tuple(correctify.rectify(config.layers[lay], point)) @@ -298,13 +303,13 @@ def twms_main(data): ): sec.add((ec[0] + tr, ec[1] + tg, ec[2] + tb, ec[3])) i2l = im2.load() - for x in range(0, im2.size[0]): - for y in range(0, im2.size[1]): - t = i2l[x, y] + for px in range(0, im2.size[0]): + for py in range(0, im2.size[1]): + t = i2l[px, py] if t in sec: - i2l[x, y] = (t[0], t[1], t[2], 0) + i2l[px, py] = (t[0], t[1], t[2], 0) if not im2.size == result_img.size: - im2 = im2.resize(result_img.size, Image.ANTIALIAS) + im2 = im2.resize(result_img.size, resampling_lanczos(Image)) im2 = Image.composite(im2, result_img, im2.split()[3]) # imgs/(imgs+1.)) if "noblend" in force: @@ -324,15 +329,15 @@ def twms_main(data): result_img, req_bbox, srs, - color if len(color) > 0 else None, + colors[0] if colors else None, trackblend, ) if len(gpx) > 0: last_color = None - c = iter(color) + c = iter(colors) for track in tracks: try: - last_color = c.next() + last_color = next(c) except StopIteration: pass result_img = drawing.gpx( @@ -479,7 +484,7 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal im.paste(im2, (256, 0)) im.paste(im3, (0, 256)) im.paste(im4, (256, 256)) - im = im.resize((256, 256), Image.ANTIALIAS) + im = im.resize((256, 256), resampling_lanczos(Image)) if layer.get("cached", True): try: im.save(local + "ups." + ext) @@ -640,6 +645,6 @@ def getimg(bbox, request_proj, size, layer, start_time, force): out = out.transform((W, H), Image.QUAD, quad, Image.BICUBIC) elif (W != out.size[0]) or (H != out.size[1]): "just resize" - out = out.resize((W, H), Image.ANTIALIAS) + out = out.resize((W, H), resampling_lanczos(Image)) # out = reproject(out, bbox, layer["proj"], request_proj) return out From c24f53df895cb18d4a8f575aa844e5a579e779d0 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:44:30 +0400 Subject: [PATCH 20/51] fix: validate downloaded tile images Validate downloaded WMS/TMS image bytes before saving them to the filesystem cache, keep stale cached tiles on invalid downloads, and convert mismatched tile responses to the configured layer extension when possible. This keeps the useful cache-safety behavior from Radioxoma's fork without adopting the larger fetcher rewrite. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 48 ++++++++++++++++++++++++++++++++++- twms/fetchers.py | 52 +++++++++++++++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index d7026e5..1fec415 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -30,7 +30,10 @@ class LegacySmokeTest(unittest.TestCase): def image_bytes(self, color, image_format="PNG"): buffer = BytesIO() - Image.new("RGBA", (256, 256), color).save(buffer, image_format) + image = Image.new("RGBA", (256, 256), color) + if image_format == "JPEG": + image = image.convert("RGB") + image.save(buffer, image_format) return buffer.getvalue() def cache_path(self, cache_root, layer, z, x, y): @@ -449,6 +452,49 @@ def test_dead_tile_dict_is_recorded_as_tne(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_invalid_downloaded_tile_is_not_cached(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "invalid", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = b"not an image" + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertFalse(image) + self.assertFalse(os.path.exists(path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_cache_converts_download_to_layer_extension(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "format", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.jpg", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (20, 30, 40), image_format="JPEG" + ) + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.format, "JPEG") + with Image.open(path) as cached_image: + self.assertEqual(cached_image.format, "PNG") + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_wsgi_application_imports(self): self.assertTrue(callable(twms.daemon.application)) diff --git a/twms/fetchers.py b/twms/fetchers.py index 52de29a..2b784f9 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -25,6 +25,14 @@ thread_responses = {} zhash_lock = {} +_EXTENSION_FORMATS = { + "gif": "GIF", + "jpg": "JPEG", + "jpeg": "JPEG", + "png": "PNG", + "webp": "WEBP", +} + def _cache_stem(z, x, y, this_layer): return ( @@ -195,7 +203,10 @@ def WMS(z, x, y, this_layer): return cache.wait_for_peer() try: try: - im = Image.open(BytesIO(urlopen(wms).read())) + contents = urlopen(wms).read() + im = _open_downloaded_image(contents) + if im is None: + raise OSError except OSError: stale = cache.open_image(include_stale=True) if stale is not None: @@ -243,7 +254,9 @@ def Tile(z, x, y, this_layer): try: try: contents = urlopen(remote).read() - im = Image.open(BytesIO(contents)) + im = _open_downloaded_image(contents) + if im is None: + raise OSError except OSError: stale = cache.open_image(include_stale=True) if stale is not None: @@ -253,13 +266,46 @@ def Tile(z, x, y, this_layer): cache.mark_tne() return False if this_layer.get("cached", True): - cache.write_bytes(contents) + cache.write_bytes(_cache_image_bytes(contents, im, this_layer["ext"])) return im finally: if locked: cache.release() +def _open_downloaded_image(contents): + if not contents: + return None + image = Image.open(BytesIO(contents)) + image.load() + return image + + +def _cache_image_bytes(contents, image, extension): + target_format = _EXTENSION_FORMATS.get(extension.lower()) + if target_format is None or image.format == target_format: + return contents + + image_content = BytesIO() + if target_format == "JPEG": + image = image.convert("RGB") + image.save( + image_content, + target_format, + quality=config.output_quality, + progressive=config.output_progressive, + ) + elif target_format == "PNG": + image.save( + image_content, + target_format, + optimize=config.output_optimize, + ) + else: + image.save(image_content, target_format) + return image_content.getvalue() + + def _is_dead_tile(contents, dead_tile): if isinstance(dead_tile, dict): if "size" in dead_tile and len(contents) != dead_tile["size"]: From 237dac575f1c47977be64339a30bc5da398d4890 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:47:09 +0400 Subject: [PATCH 21/51] fix: record HTTP tile misses Record HTTP 404 tile responses as .tne cache misses and allow layer configs to opt in additional HTTP miss statuses with dead_tile.http_status. This keeps the useful HTTP-status TNE behavior from Radioxoma's fork while avoiding service-specific heuristics and noisy response-body logging. Co-authored-by: Eugene Dvoretsky --- README.md | 3 +- tests/test_legacy_smoke.py | 58 ++++++++++++++++++++++++++++++++++++++ twms/fetchers.py | 24 ++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ec5c5ff..5b2463d 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,8 @@ Layer dictionaries usually define: - `min_zoom` / `max_zoom`: optional zoom limits - `cache_ttl`: optional fresh-cache lifetime in seconds - `dead_tile`: optional dead-tile marker, either a legacy file path or a - `{ "size": ..., "md5": {...} }` dictionary + `{ "size": ..., "md5": {...} }` dictionary; dictionaries may also set + `http_status` for an upstream status code that should be cached as `.tne` ## Client URLs diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 1fec415..b0e55ca 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -9,6 +9,7 @@ import tempfile import threading import unittest +import urllib.error import urllib.request import warnings from http.server import ThreadingHTTPServer @@ -495,6 +496,63 @@ def test_tile_cache_converts_download_to_layer_extension(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_http_404_tile_is_recorded_as_tne(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "http-tne", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + error = urllib.error.HTTPError( + "http://example.test/2/3/4.png", + 404, + "Not Found", + hdrs={}, + fp=None, + ) + with mock.patch("twms.fetchers.urlopen", side_effect=error): + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertFalse(image) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(tne_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_configured_http_status_tile_is_recorded_as_tne(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "http-tne", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "dead_tile": {"http_status": 410}, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + error = urllib.error.HTTPError( + "http://example.test/2/3/4.png", + 410, + "Gone", + hdrs={}, + fp=None, + ) + with mock.patch("twms.fetchers.urlopen", side_effect=error): + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertFalse(image) + self.assertFalse(os.path.exists(path)) + self.assertTrue(os.path.exists(tne_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_wsgi_application_imports(self): self.assertTrue(callable(twms.daemon.application)) diff --git a/twms/fetchers.py b/twms/fetchers.py index 2b784f9..e3d3193 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -13,6 +13,7 @@ import threading import time from io import BytesIO +from urllib.error import HTTPError from urllib.request import urlopen import config @@ -257,6 +258,14 @@ def Tile(z, x, y, this_layer): im = _open_downloaded_image(contents) if im is None: raise OSError + except HTTPError as error: + if _is_tne_http_error(error, this_layer): + cache.mark_tne() + return False + stale = cache.open_image(include_stale=True) + if stale is not None: + return stale + return False except OSError: stale = cache.open_image(include_stale=True) if stale is not None: @@ -306,6 +315,21 @@ def _cache_image_bytes(contents, image, extension): return image_content.getvalue() +def _is_tne_http_error(error, this_layer): + status = getattr(error, "status", error.code) + if status == 404: + return True + + dead_tile = this_layer.get("dead_tile") + if not isinstance(dead_tile, dict) or "http_status" not in dead_tile: + return False + + configured = dead_tile["http_status"] + if isinstance(configured, (list, tuple, set)): + return status in configured + return status == configured + + def _is_dead_tile(contents, dead_tile): if isinstance(dead_tile, dict): if "size" in dead_tile and len(contents) != dead_tile["size"]: From 2aa87ce9dc143507301e94660363db4ca02d8171 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:50:44 +0400 Subject: [PATCH 22/51] feat: support named tile URL placeholders Accept readable tile URL templates with {z}, {x}, {y}, {-y}, and {q} while preserving the legacy percent-template path and transform_tile_number behavior. This takes the useful URL placeholder idea from Radioxoma's fork without migrating existing configs or adopting the larger fetcher rewrite. Co-authored-by: Eugene Dvoretsky --- README.md | 4 ++- tests/test_legacy_smoke.py | 64 ++++++++++++++++++++++++++++++++++++++ twms/fetchers.py | 34 +++++++++++++++++++- 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5b2463d..7c3d670 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ Layer dictionaries usually define: - `prefix`: cache subdirectory - `ext`: tile extension such as `jpg` or `png` - `proj`: tile pyramid projection, commonly `EPSG:3857` or `EPSG:3395` -- `remote_url`: upstream tile URL template +- `remote_url`: upstream tile URL template; legacy `%s/%s/%s` templates still + work, and named placeholders `{z}`, `{x}`, `{y}`, `{-y}`, and `{q}` are also + accepted for readable Slippy/TMS/Bing URLs - `fetch`: fetcher function, normally `fetchers.Tile` - `min_zoom` / `max_zoom`: optional zoom limits - `cache_ttl`: optional fresh-cache lifetime in seconds diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index b0e55ca..0276b08 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -553,6 +553,70 @@ def test_configured_http_status_tile_is_recorded_as_tne(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_legacy_percent_tile_template_still_uses_transform_tuple(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "url", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "transform_tile_number": lambda z, x, y: (x, y, z - 1), + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_called_once_with("http://example.test/3/4/1.png") + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_named_tile_template_placeholders(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "url", + "ext": "png", + "remote_url": "http://example.test/{z}/{x}/{y}/{-y}/{q}.png", + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(4, 9, 5, layer) + + urlopen.assert_called_once_with( + "http://example.test/4/9/5/10/1203.png" + ) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_named_tile_template_uses_transform_tuple(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "url", + "ext": "png", + "remote_url": "http://example.test/z{z}/x{x}/y{y}.png", + "transform_tile_number": lambda z, x, y: (z - 1, x + 1, y + 2), + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(4, 5, 6, layer) + + urlopen.assert_called_once_with("http://example.test/z3/x6/y8.png") + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_wsgi_application_imports(self): self.assertTrue(callable(twms.daemon.application)) diff --git a/twms/fetchers.py b/twms/fetchers.py index e3d3193..64e6e08 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -242,7 +242,7 @@ def Tile(z, x, y, this_layer): if "transform_tile_number" in this_layer: d_tuple = this_layer["transform_tile_number"](z, x, y) - remote = this_layer["remote_url"] % d_tuple + remote = _format_tile_url(this_layer["remote_url"], d_tuple) cache = TileCache(z, x, y, this_layer) if this_layer.get("cached", True) and not cache.needs_fetch(): return cache.open_image() @@ -315,6 +315,38 @@ def _cache_image_bytes(contents, image, extension): return image_content.getvalue() +def _format_tile_url(template, tile): + if "{" not in template: + return template % tile + + z, x, y = tile + return ( + template.replace("{z}", str(z)) + .replace("{x}", str(x)) + .replace("{y}", str(y)) + .replace("{-y}", str(tile_slippy_to_tms(z, x, y)[2])) + .replace("{q}", tile_to_quadkey(z, x, y)) + ) + + +def tile_to_quadkey(z, x, y): + quadkey = [] + for offset in range(z): + bit = z - offset + digit = ord("0") + mask = 1 << (bit - 1) + if x & mask: + digit += 1 + if y & mask: + digit += 2 + quadkey.append(chr(digit)) + return "".join(quadkey) + + +def tile_slippy_to_tms(z, x, y): + return z, x, (1 << z) - y - 1 + + def _is_tne_http_error(error, this_layer): status = getattr(error, "status", error.code) if status == 404: From 2980eaae34b04418bc95f310b1e6708f20ffeaaa Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 14:55:49 +0400 Subject: [PATCH 23/51] feat: expose JOSM imagery list Generate a JOSM maps-1.0 imagery XML document from configured TWMS layers and expose it through both request=GetJOSMImagery and stdlib routes at /josm/imagery.xml and /maps.xml. This keeps the useful JOSM imagery-list idea from Radioxoma's fork without taking the heavy route/config reshuffle or launching JOSM remote-control side effects. Co-authored-by: Eugene Dvoretsky --- README.md | 6 +++++ tests/test_legacy_smoke.py | 36 +++++++++++++++++++++++++++ twms/__init__.py | 1 + twms/josm.py | 50 ++++++++++++++++++++++++++++++++++++++ twms/server.py | 5 ++++ twms/twms.py | 3 +++ 6 files changed, 101 insertions(+) create mode 100644 twms/josm.py diff --git a/README.md b/README.md index 7c3d670..0d06b19 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,12 @@ For a normal local proxy, add a TMS imagery entry such as: tms:http://127.0.0.1:8080/osm/{zoom}/{x}/{y}.png ``` +JOSM can also consume the generated imagery list: + +```text +http://127.0.0.1:8080/josm/imagery.xml +``` + For TWMS-specific parameters, use the WMS-style `GetTile` URL instead: ```text diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 0276b08..acd8148 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -61,6 +61,7 @@ def test_legacy_modules_import_as_package_modules(self): "twms.filter", "twms.gpxparse", "twms.image_compat", + "twms.josm", "twms.overview", "twms.projections", "twms.reproject", @@ -253,6 +254,30 @@ def test_tilejson_smoke(self): self.assertEqual(doc["minzoom"], 0) self.assertEqual(doc["maxzoom"], 18) + def test_josm_imagery_xml_smoke(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetJOSMImagery", + "ref": "http://example.test/", + } + ) + + namespaces = {"josm": "http://josm.openstreetmap.de/maps-1.0"} + root = ET.fromstring(body) + osm = root.find("./josm:entry[josm:id='twms-osm']", namespaces) + landsat = root.find("./josm:entry[josm:id='twms-landsat']", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(root.tag, "{http://josm.openstreetmap.de/maps-1.0}imagery") + self.assertEqual(osm.find("josm:name", namespaces).text, "OpenStreetMap mapnik") + self.assertEqual(osm.find("josm:type", namespaces).text, "tms") + self.assertEqual( + osm.find("josm:url", namespaces).text, + "http://example.test/osm/{zoom}/{x}/{y}.png", + ) + self.assertEqual(landsat.find("josm:max-zoom", namespaces).text, "11") + def test_wmts_capabilities_smoke(self): status, content_type, body = twms.twms.twms_main( { @@ -649,6 +674,17 @@ def test_stdlib_server_serves_wms_and_gettile(self): self.assertIn("application/json", response.headers["Content-Type"]) self.assertEqual(doc["tiles"], [base + "/osm/{z}/{x}/{y}.png"]) + with urllib.request.urlopen(base + "/josm/imagery.xml") as response: + root = ET.fromstring(response.read().decode("utf-8")) + namespaces = {"josm": "http://josm.openstreetmap.de/maps-1.0"} + osm = root.find("./josm:entry[josm:id='twms-osm']", namespaces) + self.assertEqual(response.status, 200) + self.assertIn("text/xml", response.headers["Content-Type"]) + self.assertEqual( + osm.find("josm:url", namespaces).text, + base + "/osm/{zoom}/{x}/{y}.png", + ) + with urllib.request.urlopen( base + "/wmts/1.0.0/WMTSCapabilities.xml" ) as response: diff --git a/twms/__init__.py b/twms/__init__.py index 38f0e46..d001e59 100644 --- a/twms/__init__.py +++ b/twms/__init__.py @@ -30,6 +30,7 @@ def __getattr__(name): "correctify", "drawing", "image_compat", + "josm", "projections", "reproject", "wmts", diff --git a/twms/josm.py b/twms/josm.py new file mode 100644 index 0000000..e21dd94 --- /dev/null +++ b/twms/josm.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# This file is part of twms. + +# This program is free software. It comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms specified in COPYING. + +import xml.etree.ElementTree as ET + + +JOSM_MAPS = "http://josm.openstreetmap.de/maps-1.0" +ET.register_namespace("", JOSM_MAPS) + + +def _tag(name): + return "{%s}%s" % (JOSM_MAPS, name) + + +def _layer_extension(layer): + return layer.get("ext", "jpg").lower().replace("jpeg", "jpg") + + +def _layer_url(ref, layer_name, layer): + return "%s%s/{zoom}/{x}/{y}.%s" % ( + ref, + layer_name, + _layer_extension(layer), + ) + + +def document(config, ref): + root = ET.Element(_tag("imagery")) + for layer_name in sorted(config.layers): + layer = config.layers[layer_name] + entry = ET.SubElement(root, _tag("entry")) + ET.SubElement(entry, _tag("name")).text = layer.get("name", layer_name) + ET.SubElement(entry, _tag("id")).text = "twms-%s" % layer_name + ET.SubElement(entry, _tag("type")).text = "tms" + ET.SubElement(entry, _tag("url")).text = _layer_url(ref, layer_name, layer) + if "provider_url" in layer: + ET.SubElement(entry, _tag("attribution-url")).text = layer["provider_url"] + if "max_zoom" in layer: + ET.SubElement(entry, _tag("max-zoom")).text = str(layer["max_zoom"] - 1) + if "min_zoom" in layer: + ET.SubElement(entry, _tag("min-zoom")).text = str(layer["min_zoom"]) + return root + + +def xml(config, ref): + return ET.tostring(document(config, ref), encoding="unicode", xml_declaration=True) diff --git a/twms/server.py b/twms/server.py index 8d04a50..5197c4e 100644 --- a/twms/server.py +++ b/twms/server.py @@ -16,6 +16,7 @@ tile_route = re.compile(r"/(.*)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?(.*)") tilejson_route = re.compile(r"/tilejson/(.*)\.json") +josm_imagery_routes = {"/josm/imagery.xml", "/maps.xml"} wmts_capabilities_route = "/wmts/1.0.0/WMTSCapabilities.xml" wmts_tile_route = re.compile( r"/wmts/([^/]+)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?" @@ -38,6 +39,10 @@ def dispatch(path, ref=None): "request": "GetTileJSON", "layers": urllib.parse.unquote(tilejson_match.group(1)), } + elif parsed.path in josm_imagery_routes: + data = { + "request": "GetJOSMImagery", + } elif parsed.path == wmts_capabilities_route: data = { "request": "GetCapabilities", diff --git a/twms/twms.py b/twms/twms.py index 98e1c52..bee2bad 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -26,6 +26,7 @@ import correctify import drawing import fetchers +import josm import overview import projections import tilejson @@ -142,6 +143,8 @@ def twms_main(data): return (OK, "application/json", resp) except KeyError as exc: return (400, "text/plain", str(exc)) + if req_type_lower in ("getjosmimagery", "josmimagery", "getjosmmaps"): + return (OK, "text/xml", josm.xml(config, ref)) layer = data.get("layers", config.default_layers).split(",") if ("layers" in data) and not layer[0]: From 041580cfbfa6d967bd39f422ca2c7c5c4d592994 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:02:06 +0400 Subject: [PATCH 24/51] fix: save WMS cache images atomically Pass the target image format explicitly when saving through a temporary cache path, so Pillow does not infer the format from the .tmp. suffix. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 24 ++++++++++++++++++++++++ twms/fetchers.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index acd8148..29e93c1 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -549,6 +549,30 @@ def test_http_404_tile_is_recorded_as_tne(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_wms_fetcher_caches_downloaded_image(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "wms-cache", + "ext": "png", + "remote_url": "http://example.test/wms?", + "proj": "EPSG:3857", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + image = twms.fetchers.WMS(2, 3, 4, layer) + + self.assertEqual(image.getpixel((0, 0)), (1, 2, 3, 255)) + self.assertTrue(os.path.exists(path)) + urlopen.assert_called_once_with(mock.ANY) + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_configured_http_status_tile_is_recorded_as_tne(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/fetchers.py b/twms/fetchers.py index 64e6e08..d47a2f3 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -128,7 +128,7 @@ def save_image(self, image): if not self.cached: return tmp_path = self.path + ".tmp.%s" % os.getpid() - image.save(tmp_path) + image.save(tmp_path, _EXTENSION_FORMATS.get(self.layer["ext"].lower())) os.replace(tmp_path, self.path) if os.path.exists(self.tne_path): os.remove(self.tne_path) From 61df8a7884487e268105a8d93b0c75a8bfc2065e Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:02:53 +0400 Subject: [PATCH 25/51] feat: send configured upstream headers Pass optional per-layer HTTP headers to tile and WMS upstream downloads while leaving layers without headers on the legacy string-url path. Co-authored-by: Eugene Dvoretsky --- README.md | 2 ++ tests/test_legacy_smoke.py | 60 ++++++++++++++++++++++++++++++++++++++ twms/fetchers.py | 13 +++++++-- 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0d06b19..142801c 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ Layer dictionaries usually define: - `remote_url`: upstream tile URL template; legacy `%s/%s/%s` templates still work, and named placeholders `{z}`, `{x}`, `{y}`, `{-y}`, and `{q}` are also accepted for readable Slippy/TMS/Bing URLs +- `headers`: optional upstream HTTP request headers, such as `Referer`, + `User-Agent`, or authentication cookies required by a particular source - `fetch`: fetcher function, normally `fetchers.Tile` - `min_zoom` / `max_zoom`: optional zoom limits - `cache_ttl`: optional fresh-cache lifetime in seconds diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 29e93c1..da1e4ed 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -573,6 +573,66 @@ def test_wms_fetcher_caches_downloaded_image(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_tile_fetcher_sends_configured_headers(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "headers", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "headers": { + "Referer": "https://example.test/map/", + "User-Agent": "twms-test", + }, + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(2, 3, 4, layer) + + request = urlopen.call_args.args[0] + headers = { + key.lower(): value for key, value in request.header_items() + } + self.assertEqual(request.full_url, "http://example.test/2/3/4.png") + self.assertEqual(headers["referer"], "https://example.test/map/") + self.assertEqual(headers["user-agent"], "twms-test") + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_wms_fetcher_sends_configured_headers(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "wms-headers", + "ext": "png", + "remote_url": "http://example.test/wms?", + "proj": "EPSG:3857", + "headers": { + "Referer": "https://example.test/wms-client/", + }, + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.WMS(2, 3, 4, layer) + + request = urlopen.call_args.args[0] + headers = { + key.lower(): value for key, value in request.header_items() + } + self.assertTrue(request.full_url.startswith("http://example.test/wms?")) + self.assertIn("bbox=", request.full_url) + self.assertEqual(headers["referer"], "https://example.test/wms-client/") + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_configured_http_status_tile_is_recorded_as_tne(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/fetchers.py b/twms/fetchers.py index d47a2f3..c2a3be1 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -14,7 +14,7 @@ import time from io import BytesIO from urllib.error import HTTPError -from urllib.request import urlopen +from urllib.request import Request, urlopen import config import projections @@ -43,6 +43,13 @@ def _cache_stem(z, x, y, this_layer): ) +def _upstream_request(url, this_layer): + headers = this_layer.get("headers") + if headers: + return Request(url, headers=headers) + return url + + class TileCache: """Small filesystem cache helper. @@ -204,7 +211,7 @@ def WMS(z, x, y, this_layer): return cache.wait_for_peer() try: try: - contents = urlopen(wms).read() + contents = urlopen(_upstream_request(wms, this_layer)).read() im = _open_downloaded_image(contents) if im is None: raise OSError @@ -254,7 +261,7 @@ def Tile(z, x, y, this_layer): return cache.wait_for_peer() try: try: - contents = urlopen(remote).read() + contents = urlopen(_upstream_request(remote, this_layer)).read() im = _open_downloaded_image(contents) if im is None: raise OSError From a1339fad92b63ac99001921ea836dae553885b4a Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:06:38 +0400 Subject: [PATCH 26/51] fix: enforce minimum layer zoom Honor per-layer min_zoom in both tile and WMS fetchers while preserving the existing exclusive max_zoom behavior. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 43 ++++++++++++++++++++++++++++++++++++++ twms/fetchers.py | 18 ++++++++++------ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index da1e4ed..d76666d 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -573,6 +573,49 @@ def test_wms_fetcher_caches_downloaded_image(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_tile_fetcher_respects_min_zoom(self): + layer = { + "prefix": "minzoom", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "min_zoom": 3, + } + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertIsNone(image) + urlopen.assert_not_called() + + def test_wms_fetcher_respects_min_zoom(self): + layer = { + "prefix": "wms-minzoom", + "ext": "png", + "remote_url": "http://example.test/wms?", + "proj": "EPSG:3857", + "min_zoom": 3, + } + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.WMS(2, 3, 4, layer) + + self.assertIsNone(image) + urlopen.assert_not_called() + + def test_tile_fetcher_keeps_legacy_exclusive_max_zoom(self): + layer = { + "prefix": "maxzoom", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "max_zoom": 2, + } + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertIsNone(image) + urlopen.assert_not_called() + def test_tile_fetcher_sends_configured_headers(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/fetchers.py b/twms/fetchers.py index c2a3be1..5b6917f 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -50,6 +50,14 @@ def _upstream_request(url, this_layer): return url +def _outside_zoom_limits(z, this_layer): + if "min_zoom" in this_layer and z < this_layer["min_zoom"]: + return True + if "max_zoom" in this_layer and z >= this_layer["max_zoom"]: + return True + return False + + class TileCache: """Small filesystem cache helper. @@ -188,9 +196,8 @@ def threadwrapper(z, x, y, this_layer, zhash): def WMS(z, x, y, this_layer): - if "max_zoom" in this_layer: - if z >= this_layer["max_zoom"]: - return None + if _outside_zoom_limits(z, this_layer): + return None wms = this_layer["remote_url"] req_proj = this_layer.get("wms_proj", this_layer["proj"]) width = 384 # using larger source size to rescale better in python @@ -243,9 +250,8 @@ def WMS(z, x, y, this_layer): def Tile(z, x, y, this_layer): global OSError, IOError d_tuple = z, x, y - if "max_zoom" in this_layer: - if z >= this_layer["max_zoom"]: - return None + if _outside_zoom_limits(z, this_layer): + return None if "transform_tile_number" in this_layer: d_tuple = this_layer["transform_tile_number"](z, x, y) From 7852d8ae13e53343fa7739a7edc0c3e634ce84d1 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:10:01 +0400 Subject: [PATCH 27/51] feat: support zxy cache layout Add an opt-in cache_layout for slippy/MOBAC-style // cache paths while keeping TWMS' historical grouped cache layout as the default. Co-authored-by: Eugene Dvoretsky --- README.md | 18 +++++++++--- tests/test_legacy_smoke.py | 59 ++++++++++++++++++++++++++++++++++++++ twms/fetchers.py | 7 +++-- 3 files changed, 77 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 142801c..960e9a7 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,9 @@ Layer dictionaries usually define: - `fetch`: fetcher function, normally `fetchers.Tile` - `min_zoom` / `max_zoom`: optional zoom limits - `cache_ttl`: optional fresh-cache lifetime in seconds +- `cache_layout`: optional cache path layout; the default is TWMS' + historical grouped layout, while `zxy` stores slippy/MOBAC-style + `////.` tiles - `dead_tile`: optional dead-tile marker, either a legacy file path or a `{ "size": ..., "md5": {...} }` dictionary; dictionaries may also set `http_status` for an upstream status code that should be cached as `.tne` @@ -149,7 +152,8 @@ tms:http://127.0.0.1:8080/?request=GetTile&layers=osm&z={zoom}&x={x}&y={y}&forma ``` JOSM can also point directly at a compatible local slippy-map cache with -`file://` if no proxy or reprojection is needed: +`file://` if no proxy or reprojection is needed. Configure that layer with +`cache_layout: "zxy"` so TWMS uses the same `//` path: ```text tms:file:///home/user/SAS.Planet/cache_ma/osm/{zoom}/{x}/{y}.png @@ -163,7 +167,8 @@ tms:file:///C:/SAS.Planet/cache_ma/osm/{zoom}/{x}/{y}.png ## Shared tile caches -twms keeps the historical filesystem cache layout under `tiles_cache`: +By default, twms keeps the historical filesystem cache layout under +`tiles_cache`: ```text //z//x//y. @@ -174,8 +179,13 @@ expires, twms tries to refresh the tile; if the remote fetch fails, the stale cached tile can still be used. Missing/dead tiles can be recorded as `.tne` files so repeated requests do not hammer upstream services. -This makes twms useful with tools that share a slippy-map/MOBAC-style -cache, including SAS.Planet and similar offline tile workflows. +Layers that need to share a slippy-map/MOBAC-style cache, including +SAS.Planet and similar offline tile workflows, can opt in with +`cache_layout: "zxy"`: + +```text +////. +``` ## Optional dependencies diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index d76666d..29aed00 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -48,6 +48,15 @@ def cache_path(self, cache_root, layer, z, x, y): "y%s.%s" % (y, layer["ext"]), ) + def zxy_cache_path(self, cache_root, layer, z, x, y): + return os.path.join( + cache_root, + layer["prefix"], + "%s" % z, + "%s" % x, + "%s.%s" % (y, layer["ext"]), + ) + def test_public_version_keeps_keyboard_suffix(self): self.assertEqual(twms.__version__, "0.07z") self.assertEqual(importlib.metadata.version("twms"), "0.7+z") @@ -392,6 +401,56 @@ def test_tile_cache_uses_fresh_file_without_network(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_tile_cache_can_use_zxy_layout(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "zxy", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_layout": "zxy", + } + try: + path = self.zxy_cache_path(cache_root, layer, 2, 3, 4) + legacy_path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (10, 20, 30, 255) + ) + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + self.assertTrue(os.path.exists(path)) + self.assertFalse(os.path.exists(legacy_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_cache_can_reuse_fresh_zxy_file_without_network(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "zxy-hit", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "cache_layout": "zxy", + "cache_ttl": 3600, + } + try: + path = self.zxy_cache_path(cache_root, layer, 2, 3, 4) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as tile_file: + tile_file.write(self.image_bytes((70, 80, 90, 255))) + + with mock.patch("twms.fetchers.urlopen") as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.getpixel((0, 0)), (70, 80, 90, 255)) + urlopen.assert_not_called() + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_tile_cache_refetches_expired_file_and_keeps_stale_on_error(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/fetchers.py b/twms/fetchers.py index 5b6917f..f7a4ddd 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -36,10 +36,11 @@ def _cache_stem(z, x, y, this_layer): + cache_prefix = config.tiles_cache + this_layer["prefix"] + if this_layer.get("cache_layout") in ("zxy", "slippy", "mobac", "tms"): + return cache_prefix + "/%s/%s/%s." % (z, x, y) return ( - config.tiles_cache - + this_layer["prefix"] - + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) + cache_prefix + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) ) From 98c39bd2c4b687308aca05ad0f6fa5e3e7399bc7 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:15:22 +0400 Subject: [PATCH 28/51] feat: support WMS URL placeholders Allow upstream WMS remote_url templates to supply {bbox}, {width}, {height}, and {proj} while preserving the legacy base-URL append behavior when no WMS placeholders are present. Co-authored-by: Eugene Dvoretsky --- README.md | 7 ++--- tests/test_legacy_smoke.py | 52 ++++++++++++++++++++++++++++++++++++++ twms/fetchers.py | 25 +++++++++++++++--- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 960e9a7..3d590cb 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,10 @@ Layer dictionaries usually define: - `prefix`: cache subdirectory - `ext`: tile extension such as `jpg` or `png` - `proj`: tile pyramid projection, commonly `EPSG:3857` or `EPSG:3395` -- `remote_url`: upstream tile URL template; legacy `%s/%s/%s` templates still - work, and named placeholders `{z}`, `{x}`, `{y}`, `{-y}`, and `{q}` are also - accepted for readable Slippy/TMS/Bing URLs +- `remote_url`: upstream tile/WMS URL template; legacy tile `%s/%s/%s` + templates still work, named tile placeholders `{z}`, `{x}`, `{y}`, `{-y}`, + and `{q}` are accepted for readable Slippy/TMS/Bing URLs, and WMS upstream + templates may use `{bbox}`, `{width}`, `{height}`, and `{proj}` - `headers`: optional upstream HTTP request headers, such as `Referer`, `User-Agent`, or authentication cookies required by a particular source - `fetch`: fetcher function, normally `fetchers.Tile` diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 29aed00..65f25eb 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -632,6 +632,58 @@ def test_wms_fetcher_caches_downloaded_image(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_wms_fetcher_keeps_legacy_url_append(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "wms-legacy-url", + "ext": "png", + "remote_url": "http://example.test/wms?", + "proj": "EPSG:3857", + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.WMS(2, 3, 4, layer) + + url = urlopen.call_args.args[0] + self.assertIsInstance(url, str) + self.assertTrue(url.startswith("http://example.test/wms?bbox=")) + self.assertIn("&width=384&height=384&srs=EPSG:3857", url) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_wms_fetcher_formats_named_url_placeholders(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "wms-template-url", + "ext": "png", + "remote_url": ( + "http://example.test/wms?SERVICE=WMS&REQUEST=GetMap" + "&WIDTH={width}&HEIGHT={height}&CRS={proj}&BBOX={bbox}" + ), + "proj": "EPSG:3857", + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.WMS(2, 3, 4, layer) + + url = urlopen.call_args.args[0] + self.assertIsInstance(url, str) + self.assertIn("WIDTH=384&HEIGHT=384&CRS=EPSG:3857&BBOX=", url) + self.assertNotIn("srs=EPSG:3857", url) + self.assertEqual(url.count("BBOX="), 1) + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_tile_fetcher_respects_min_zoom(self): layer = { "prefix": "minzoom", diff --git a/twms/fetchers.py b/twms/fetchers.py index f7a4ddd..f2de405 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -59,6 +59,26 @@ def _outside_zoom_limits(z, this_layer): return False +def _format_wms_url(template, bbox, width, height, projection): + if any( + placeholder in template + for placeholder in ("{bbox}", "{width}", "{height}", "{proj}") + ): + return ( + template.replace("{bbox}", bbox) + .replace("{width}", str(width)) + .replace("{height}", str(height)) + .replace("{proj}", projection) + ) + + return template + "bbox=%s&width=%s&height=%s&srs=%s" % ( + bbox, + width, + height, + projection, + ) + + class TileCache: """Small filesystem cache helper. @@ -199,16 +219,15 @@ def threadwrapper(z, x, y, this_layer, zhash): def WMS(z, x, y, this_layer): if _outside_zoom_limits(z, this_layer): return None - wms = this_layer["remote_url"] req_proj = this_layer.get("wms_proj", this_layer["proj"]) width = 384 # using larger source size to rescale better in python height = 384 cache = TileCache(z, x, y, this_layer) - tile_bbox = "bbox=%s,%s,%s,%s" % tuple( + tile_bbox = "%s,%s,%s,%s" % tuple( projections.from4326(projections.bbox_by_tile(z, x, y, req_proj), req_proj) ) - wms += tile_bbox + "&width=%s&height=%s&srs=%s" % (width, height, req_proj) + wms = _format_wms_url(this_layer["remote_url"], tile_bbox, width, height, req_proj) if this_layer.get("cached", True) and not cache.needs_fetch(): return cache.open_image() locked = False From 794940bd0094f5b9fabf3c4dfcb12a637f0fb6b4 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:21:59 +0400 Subject: [PATCH 29/51] fix: bound upstream fetch timeouts Keep the stdlib threaded server tiny, but avoid waiting forever on a stalled upstream tile or WMS source. Layer configs may override the timeout when an old deployment deliberately needs different behavior. Co-authored-by: Eugene Dvoretsky --- README.md | 6 ++++++ tests/test_legacy_smoke.py | 43 ++++++++++++++++++++++++++++++++++---- twms/fetchers.py | 21 +++++++++++++++++-- twms/twms.conf | 1 + 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3d590cb..4c4a9da 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,9 @@ Important settings: - `tiles_cache`: root directory for the filesystem tile cache - `gpx_cache`: cache for downloaded OSM GPX traces - `service_url`: externally visible base URL used in generated capabilities +- `upstream_timeout`: default upstream HTTP timeout in seconds; the example + config uses 30 seconds so threaded servers do not wait forever on a stalled + tile source - `default_layers`: layer list used when a request does not name layers - `default_format`: output image MIME type, usually `image/jpeg` - `layers`: configured imagery layers and their fetchers @@ -76,6 +79,9 @@ Layer dictionaries usually define: - `headers`: optional upstream HTTP request headers, such as `Referer`, `User-Agent`, or authentication cookies required by a particular source - `fetch`: fetcher function, normally `fetchers.Tile` +- `timeout`: optional per-layer upstream HTTP timeout in seconds; set to + `None` only if an old deployment deliberately wants the historical unbounded + wait - `min_zoom` / `max_zoom`: optional zoom limits - `cache_ttl`: optional fresh-cache lifetime in seconds - `cache_layout`: optional cache path layout; the default is TWMS' diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 65f25eb..c8a2198 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -628,7 +628,7 @@ def test_wms_fetcher_caches_downloaded_image(self): self.assertEqual(image.getpixel((0, 0)), (1, 2, 3, 255)) self.assertTrue(os.path.exists(path)) - urlopen.assert_called_once_with(mock.ANY) + urlopen.assert_called_once_with(mock.ANY, timeout=30) finally: twms.fetchers.config.tiles_cache = old_cache @@ -651,6 +651,7 @@ def test_wms_fetcher_keeps_legacy_url_append(self): url = urlopen.call_args.args[0] self.assertIsInstance(url, str) + self.assertEqual(urlopen.call_args.kwargs["timeout"], 30) self.assertTrue(url.startswith("http://example.test/wms?bbox=")) self.assertIn("&width=384&height=384&srs=EPSG:3857", url) finally: @@ -678,6 +679,7 @@ def test_wms_fetcher_formats_named_url_placeholders(self): url = urlopen.call_args.args[0] self.assertIsInstance(url, str) + self.assertEqual(urlopen.call_args.kwargs["timeout"], 30) self.assertIn("WIDTH=384&HEIGHT=384&CRS=EPSG:3857&BBOX=", url) self.assertNotIn("srs=EPSG:3857", url) self.assertEqual(url.count("BBOX="), 1) @@ -751,6 +753,7 @@ def test_tile_fetcher_sends_configured_headers(self): headers = { key.lower(): value for key, value in request.header_items() } + self.assertEqual(urlopen.call_args.kwargs["timeout"], 30) self.assertEqual(request.full_url, "http://example.test/2/3/4.png") self.assertEqual(headers["referer"], "https://example.test/map/") self.assertEqual(headers["user-agent"], "twms-test") @@ -781,12 +784,37 @@ def test_wms_fetcher_sends_configured_headers(self): headers = { key.lower(): value for key, value in request.header_items() } + self.assertEqual(urlopen.call_args.kwargs["timeout"], 30) self.assertTrue(request.full_url.startswith("http://example.test/wms?")) self.assertIn("bbox=", request.full_url) self.assertEqual(headers["referer"], "https://example.test/wms-client/") finally: twms.fetchers.config.tiles_cache = old_cache + def test_tile_fetcher_uses_configured_layer_timeout(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "timeout", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "timeout": 7, + } + try: + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (1, 2, 3, 255) + ) + twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_called_once_with( + "http://example.test/2/3/4.png", + timeout=7, + ) + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_configured_http_status_tile_is_recorded_as_tne(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache @@ -833,7 +861,10 @@ def test_legacy_percent_tile_template_still_uses_transform_tuple(self): ) twms.fetchers.Tile(2, 3, 4, layer) - urlopen.assert_called_once_with("http://example.test/3/4/1.png") + urlopen.assert_called_once_with( + "http://example.test/3/4/1.png", + timeout=30, + ) finally: twms.fetchers.config.tiles_cache = old_cache @@ -854,7 +885,8 @@ def test_named_tile_template_placeholders(self): twms.fetchers.Tile(4, 9, 5, layer) urlopen.assert_called_once_with( - "http://example.test/4/9/5/10/1203.png" + "http://example.test/4/9/5/10/1203.png", + timeout=30, ) finally: twms.fetchers.config.tiles_cache = old_cache @@ -876,7 +908,10 @@ def test_named_tile_template_uses_transform_tuple(self): ) twms.fetchers.Tile(4, 5, 6, layer) - urlopen.assert_called_once_with("http://example.test/z3/x6/y8.png") + urlopen.assert_called_once_with( + "http://example.test/z3/x6/y8.png", + timeout=30, + ) finally: twms.fetchers.config.tiles_cache = old_cache diff --git a/twms/fetchers.py b/twms/fetchers.py index f2de405..00be8f7 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -51,6 +51,23 @@ def _upstream_request(url, this_layer): return url +def _upstream_timeout(this_layer): + if "timeout" in this_layer: + return this_layer["timeout"] + return getattr( + config, + "upstream_timeout", + min(getattr(config, "deadline", 30), 30), + ) + + +def _read_upstream(url, this_layer): + return urlopen( + _upstream_request(url, this_layer), + timeout=_upstream_timeout(this_layer), + ).read() + + def _outside_zoom_limits(z, this_layer): if "min_zoom" in this_layer and z < this_layer["min_zoom"]: return True @@ -238,7 +255,7 @@ def WMS(z, x, y, this_layer): return cache.wait_for_peer() try: try: - contents = urlopen(_upstream_request(wms, this_layer)).read() + contents = _read_upstream(wms, this_layer) im = _open_downloaded_image(contents) if im is None: raise OSError @@ -287,7 +304,7 @@ def Tile(z, x, y, this_layer): return cache.wait_for_peer() try: try: - contents = urlopen(_upstream_request(remote, this_layer)).read() + contents = _read_upstream(remote, this_layer) im = _open_downloaded_image(contents) if im is None: raise OSError diff --git a/twms/twms.conf b/twms/twms.conf index 9d4ab48..2d02e0c 100644 --- a/twms/twms.conf +++ b/twms/twms.conf @@ -15,6 +15,7 @@ tiles_cache = "/var/cache/twms/tiles/" # where to put cache install_path = "/usr/share/twms/" # where to look for broken tiles and other stuff gpx_cache = "/var/cache/twms/traces/" # where to store cached OSM GPX files deadline = 45 # number of seconds that are given to make up image +upstream_timeout = 30 # upstream HTTP timeout in seconds; layers may override with "timeout" default_max_zoom = 18 # can be overridden per layer geometry_color = { # default color for overlayed vectors "LINESTRING": "#ff0000", From faadb4ed1ebf1c9be49881b9307461eb98aba0d4 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:25:00 +0400 Subject: [PATCH 30/51] fix: harden stdlib route dispatch Keep malformed REST-style protocol paths from falling through to the overview or disconnecting handler threads. Also fix the legacy GetTile fast-cache layer lookup exposed by WMTS REST tile requests with query strings. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 16 ++++++++++++++++ twms/server.py | 4 +++- twms/twms.py | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index c8a2198..1010cc4 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -976,6 +976,22 @@ def test_stdlib_server_serves_wms_and_gettile(self): with Image.open(BytesIO(body)) as image: self.assertEqual(image.size, (256, 256)) self.assertEqual(image.mode, "RGBA") + + with urllib.request.urlopen( + base + "/wmts/transparent/0/0/0.png?cache=1" + ) as response: + body = response.read() + self.assertEqual(response.status, 200) + self.assertIn("image/png", response.headers["Content-Type"]) + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.size, (256, 256)) + self.assertEqual(image.mode, "RGBA") + + for path in ("/wmts", "/tilejson/.json", "/does-not-exist"): + with self.subTest(path=path): + with self.assertRaises(urllib.error.HTTPError) as error: + urllib.request.urlopen(base + path) + self.assertEqual(error.exception.code, 404) finally: httpd.shutdown() httpd.server_close() diff --git a/twms/server.py b/twms/server.py index 5197c4e..bd5d30b 100644 --- a/twms/server.py +++ b/twms/server.py @@ -15,7 +15,7 @@ tile_route = re.compile(r"/(.*)/([0-9]+)/([0-9]+)/([0-9]+)(\.[a-zA-Z]+)?(.*)") -tilejson_route = re.compile(r"/tilejson/(.*)\.json") +tilejson_route = re.compile(r"/tilejson/(.+)\.json") josm_imagery_routes = {"/josm/imagery.xml", "/maps.xml"} wmts_capabilities_route = "/wmts/1.0.0/WMTSCapabilities.xml" wmts_tile_route = re.compile( @@ -73,6 +73,8 @@ def dispatch(path, ref=None): "y": match.group(4), } else: + if not parsed.query and parsed.path not in ("", "/"): + return 404, "text/plain", "Not Found\n" data = dict(urllib.parse.parse_qsl(parsed.query)) data = dict((key.lower(), data[key]) for key in data) diff --git a/twms/twms.py b/twms/twms.py index bee2bad..84d2ece 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -234,7 +234,7 @@ def twms_main(data): + config.layers[layer[0]]["prefix"] + "/z%s/%s/x%s/%s/y%s." % (z, x / 1024, x, y / 1024, y) ) - ext = config.layers[layer]["ext"] + ext = config.layers[layer[0]]["ext"] adds = ["", "ups."] for add in adds: if os.path.exists(local + add + ext): From 2fe1b219f7ff30898d2b259ac0e74459abdd146a Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:28:04 +0400 Subject: [PATCH 31/51] fix: bound legacy canvas upstream waits Keep the preserved WmsCanvas helper from waiting forever on remote WMS tiles while retaining an explicit timeout=None escape hatch for deployments that relied on the historical unbounded wait. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 25 +++++++++++++++++++++++++ twms/canvas.py | 14 +++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 1010cc4..129fe13 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -219,6 +219,31 @@ def test_legacy_canvas_blank_tile_smoke(self): self.assertEqual(canvas.tiles[(0, 0)]["im"].size, (32, 32)) self.assertEqual(canvas.tiles[(0, 0)]["im"].mode, "RGBA") + def test_legacy_canvas_uses_default_upstream_timeout(self): + canvas = twms.canvas.WmsCanvas( + wms_url="http://example.test/wms?", + proj="EPSG:3857", + ) + + with mock.patch("twms.canvas.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes((1, 2, 3, 255)) + canvas.FetchTile(0, 0) + + urlopen.assert_called_once_with(mock.ANY, timeout=30) + + def test_legacy_canvas_can_preserve_unbounded_upstream_wait(self): + canvas = twms.canvas.WmsCanvas( + wms_url="http://example.test/wms?", + proj="EPSG:3857", + timeout=None, + ) + + with mock.patch("twms.canvas.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes((1, 2, 3, 255)) + canvas.FetchTile(0, 0) + + urlopen.assert_called_once_with(mock.ANY, timeout=None) + def test_getimg_resize_works_with_current_pillow(self): tile = Image.new("RGBA", (256, 256), (1, 2, 3, 255)) tile.is_ok = True diff --git a/twms/canvas.py b/twms/canvas.py index 7a09cae..0041daa 100644 --- a/twms/canvas.py +++ b/twms/canvas.py @@ -18,6 +18,7 @@ from urllib.request import urlopen from io import BytesIO +import config import projections from PIL import Image, ImageFilter @@ -35,12 +36,14 @@ def __init__( tile_size=None, mode="RGBA", tile_mode="WMS", + timeout="default", ): self.wms_url = wms_url self.zoom = zoom self.proj = proj self.mode = mode self.tile_mode = tile_mode + self.timeout = timeout self.tile_height = 256 self.tile_width = 256 @@ -105,7 +108,7 @@ def FetchTile(self, x, y): remote = self.ConstructTileUrl(x, y) debug(remote) ttz = datetime.datetime.now() - contents = urlopen(remote).read() + contents = urlopen(remote, timeout=self.UpstreamTimeout()).read() debug("Download took %s" % str(datetime.datetime.now() - ttz)) im = Image.open(BytesIO(contents)) if im.mode != self.mode: @@ -118,6 +121,15 @@ def FetchTile(self, x, y): self.tiles[(x, y)]["pix"] = im.load() self.tiles[(x, y)]["status"] = "RD" + def UpstreamTimeout(self): + if self.timeout != "default": + return self.timeout + return getattr( + config, + "upstream_timeout", + min(getattr(config, "deadline", 30), 30), + ) + def PreparePixel(self, x, y): tile_x = int(x / self.tile_height) x = x % self.tile_height From ad5ead43b0a6f7ec59d9ecf9a5e22a01b78c7e38 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:33:06 +0400 Subject: [PATCH 32/51] fix: restore empty-color overlay transparency Preserve the legacy overlay/filter surface by making configured empty_color values transparent again, including per-channel empty_color_delta tolerance. This repairs the behavior instead of deleting the old rendering helpers. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 62 ++++++++++++++++++++++++++++++++++++++ twms/twms.py | 16 +++++----- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 129fe13..8e42892 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -268,6 +268,68 @@ def test_getimg_resize_works_with_current_pillow(self): self.assertEqual(image.size, (32, 32)) self.assertEqual(image.getpixel((0, 0)), (1, 2, 3, 255)) + def test_legacy_empty_color_overlay_is_transparent(self): + base = Image.new("RGBA", (2, 1), (10, 20, 30, 255)) + overlay = Image.new("RGBA", (2, 1), (255, 255, 255, 255)) + overlay.putpixel((1, 0), (255, 0, 0, 255)) + base.is_ok = True + overlay.is_ok = True + layers = { + "base": {"empty_color": "#000000"}, + "overlay": {"empty_color": "#ffffff"}, + } + + with mock.patch.object(twms.twms.config, "layers", layers): + with mock.patch.object(twms.twms, "getimg", side_effect=[base, overlay]): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "base,overlay", + "format": "image/png", + "bbox": "0,0,1,1", + "width": "2", + "height": "1", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + image = image.convert("RGBA") + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + self.assertEqual(image.getpixel((1, 0)), (132, 10, 15, 255)) + + def test_legacy_empty_color_delta_applies_per_channel(self): + base = Image.new("RGBA", (1, 1), (10, 20, 30, 255)) + overlay = Image.new("RGBA", (1, 1), (255, 254, 253, 255)) + base.is_ok = True + overlay.is_ok = True + layers = { + "base": {"empty_color": "#000000"}, + "overlay": { + "empty_color": "#ffffff", + "empty_color_delta": 2, + }, + } + + with mock.patch.object(twms.twms.config, "layers", layers): + with mock.patch.object(twms.twms, "getimg", side_effect=[base, overlay]): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "base,overlay", + "format": "image/png", + "bbox": "0,0,1,1", + "width": "1", + "height": "1", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.convert("RGBA").getpixel((0, 0)), (10, 20, 30, 255)) + def test_tilejson_smoke(self): status, content_type, body = twms.twms.twms_main( { diff --git a/twms/twms.py b/twms/twms.py index 84d2ece..088cb16 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -290,19 +290,19 @@ def twms_main(data): if "empty_color" in config.layers[ll]: ec = ImageColor.getcolor(config.layers[ll]["empty_color"], "RGBA") - sec = set(ec) + sec = {ec} if "empty_color_delta" in config.layers[ll]: delta = config.layers[ll]["empty_color_delta"] - for tr in range(-delta, delta): - for tg in range(-delta, delta): - for tb in range(-delta, delta): + for tr in range(-delta, delta + 1): + for tg in range(-delta, delta + 1): + for tb in range(-delta, delta + 1): if ( (ec[0] + tr) >= 0 and (ec[0] + tr) < 256 - and (ec[1] + tr) >= 0 - and (ec[1] + tr) < 256 - and (ec[2] + tr) >= 0 - and (ec[2] + tr) < 256 + and (ec[1] + tg) >= 0 + and (ec[1] + tg) < 256 + and (ec[2] + tb) >= 0 + and (ec[2] + tb) < 256 ): sec.add((ec[0] + tr, ec[1] + tg, ec[2] + tb, ec[3])) i2l = im2.load() From 637ef8b2f439abd6b1f51724498b9b65482bc285 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:37:34 +0400 Subject: [PATCH 33/51] fix: preserve legacy cache path lookups Use integer cache shard directories under Python 3 and read cached GetTile responses as bytes, so the historical grouped cache layout remains usable alongside the new opt-in zxy layout. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 70 ++++++++++++++++++++++++++++++++++++++ twms/twms.py | 8 ++--- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 8e42892..356715c 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -488,6 +488,76 @@ def test_tile_cache_uses_fresh_file_without_network(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_legacy_tile_image_reuses_historical_cache_path(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.twms.config.tiles_cache + twms.twms.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "legacy-hit", + "proj": "EPSG:3857", + "ext": "png", + "cached": True, + "scalable": False, + "empty_color": "#000000", + "fetch": mock.Mock(return_value=None), + } + try: + path = self.cache_path(cache_root, layer, 3, 3, 3) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + + image = twms.twms.tile_image( + layer, + 3, + 3, + 3, + datetime.datetime.now(), + real=True, + ) + + layer["fetch"].assert_not_called() + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + finally: + twms.twms.config.tiles_cache = old_cache + + def test_legacy_gettile_fast_cache_reads_binary_tile(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.twms.config.tiles_cache + old_layers = twms.twms.config.layers + twms.twms.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "legacy-fast-hit", + "proj": "EPSG:3857", + "ext": "png", + } + twms.twms.config.layers = {"legacy-fast-hit": layer} + try: + path = self.cache_path(cache_root, layer, 3, 3, 4) + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "legacy-fast-hit", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + finally: + twms.twms.config.tiles_cache = old_cache + twms.twms.config.layers = old_layers + def test_tile_cache_can_use_zxy_layout(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/twms.py b/twms/twms.py index 088cb16..7ea1b76 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -232,14 +232,14 @@ def twms_main(data): local = ( config.tiles_cache + config.layers[layer[0]]["prefix"] - + "/z%s/%s/x%s/%s/y%s." % (z, x / 1024, x, y / 1024, y) + + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) ) ext = config.layers[layer[0]]["ext"] adds = ["", "ups."] for add in adds: if os.path.exists(local + add + ext): - tile_file = open(local + add + ext, "r") - resp = tile_file.read() + with open(local + add + ext, "rb") as tile_file: + resp = tile_file.read() return (OK, content_type, resp) req_bbox = projections.from4326(projections.bbox_by_tile(z, x, y, srs), srs) @@ -428,7 +428,7 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal local = ( config.tiles_cache + layer["prefix"] - + "/z%s/%s/x%s/%s/y%s." % (z, x / 1024, x, y / 1024, y) + + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) ) ext = layer["ext"] if "cache_ttl" in layer: From eb261e0814ef2336324e2772de491abc7b1b9dbd Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:40:26 +0400 Subject: [PATCH 34/51] fix: keep response cache binary-safe Preserve the legacy cache_tile_responses fast path under Python 3 by reading and writing cached image responses as bytes instead of text. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 74 ++++++++++++++++++++++++++++++++++++++ twms/twms.py | 8 ++--- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 356715c..3bf0289 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -558,6 +558,80 @@ def test_legacy_gettile_fast_cache_reads_binary_tile(self): twms.twms.config.tiles_cache = old_cache twms.twms.config.layers = old_layers + def test_legacy_response_cache_reads_binary_tile(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = getattr(twms.twms.config, "cache_tile_responses", None) + had_cache = hasattr(twms.twms.config, "cache_tile_responses") + twms.twms.config.cache_tile_responses = { + ("EPSG:3857", ("transparent",), (), 256, 256, (), "PNG"): ( + cache_root, + "png", + ), + } + try: + path = os.path.join(cache_root, "2", "3", "4.png") + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((10, 20, 30, 255))) + + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.getpixel((0, 0)), (10, 20, 30, 255)) + finally: + if had_cache: + twms.twms.config.cache_tile_responses = old_cache + else: + del twms.twms.config.cache_tile_responses + + def test_legacy_response_cache_writes_binary_tile(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = getattr(twms.twms.config, "cache_tile_responses", None) + had_cache = hasattr(twms.twms.config, "cache_tile_responses") + twms.twms.config.cache_tile_responses = { + ("EPSG:3857", ("transparent",), (), 256, 256, (), "PNG"): ( + cache_root, + "png", + ), + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + path = os.path.join(cache_root, "2", "3", "4.png") + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with open(path, "rb") as cached_tile: + self.assertEqual(cached_tile.read(), body) + with Image.open(path) as image: + self.assertEqual(image.size, (256, 256)) + finally: + if had_cache: + twms.twms.config.cache_tile_responses = old_cache + else: + del twms.twms.config.cache_tile_responses + def test_tile_cache_can_use_zxy_layout(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/twms.py b/twms/twms.py index 7ea1b76..f9ec1b5 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -218,7 +218,8 @@ def twms_main(data): resp_ext, ) if os.path.exists(resp_cache_path): - return (OK, content_type, open(resp_cache_path, "r").read()) + with open(resp_cache_path, "rb") as cached_response: + return (OK, content_type, cached_response.read()) if len(layer) == 1: if layer[0] in config.layers: if ( @@ -391,9 +392,8 @@ def twms_main(data): except OSError: pass try: - a = open(resp_cache_path, "w") - a.write(resp) - a.close() + with open(resp_cache_path, "wb") as cached_response: + cached_response.write(resp) except (OSError, IOError): print( "error saving response answer to file %s." % (resp_cache_path), From a4e0d9eca811e9b762a5b11b240397d120421e88 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:47:54 +0400 Subject: [PATCH 35/51] feat: add optional upstream retries Harvest Radioxoma's HTTP retry idea without taking the larger backend rewrite: keep urllib, default to one attempt for legacy behavior, and let global or per-layer config opt in retries for transient transport errors. HTTP errors such as 404 still flow to the existing TNE/cache handling instead of being retried as generic network failures. Co-authored-by: Eugene Dvoretsky --- README.md | 7 +++++ tests/test_legacy_smoke.py | 55 ++++++++++++++++++++++++++++++++++++++ twms/fetchers.py | 36 ++++++++++++++++++++++--- twms/twms.conf | 2 ++ 4 files changed, 96 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4c4a9da..4aab1f2 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,9 @@ Important settings: - `upstream_timeout`: default upstream HTTP timeout in seconds; the example config uses 30 seconds so threaded servers do not wait forever on a stalled tile source +- `upstream_retries` / `upstream_retry_delay`: optional retry budget for + transient upstream network errors. The default is one attempt, preserving the + historical no-retry behavior; layers can opt in with their own values. - `default_layers`: layer list used when a request does not name layers - `default_format`: output image MIME type, usually `image/jpeg` - `layers`: configured imagery layers and their fetchers @@ -82,6 +85,10 @@ Layer dictionaries usually define: - `timeout`: optional per-layer upstream HTTP timeout in seconds; set to `None` only if an old deployment deliberately wants the historical unbounded wait +- `upstream_retries` / `upstream_retry_delay`: optional per-layer retry + override for temporary network failures. HTTP errors such as 404 are still + handled by the cache/TNE rules instead of being retried as generic transport + failures. - `min_zoom` / `max_zoom`: optional zoom limits - `cache_ttl`: optional fresh-cache lifetime in seconds - `cache_layout`: optional cache path layout; the default is TWMS' diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 3bf0289..a57e6c7 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -717,6 +717,61 @@ def test_tile_cache_refetches_expired_file_and_keeps_stale_on_error(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_tile_fetch_retries_transient_upstream_error(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "retry", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "upstream_retries": 2, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + response = mock.Mock() + response.read.return_value = self.image_bytes((11, 22, 33, 255)) + with mock.patch( + "twms.fetchers.urlopen", + side_effect=[OSError("temporary"), response], + ) as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(urlopen.call_count, 2) + self.assertEqual(image.getpixel((0, 0)), (11, 22, 33, 255)) + self.assertTrue(os.path.exists(path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + + def test_tile_fetch_does_not_retry_http_tne_status(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "retry-http-tne", + "ext": "png", + "remote_url": "http://example.test/%s/%s/%s.png", + "upstream_retries": 3, + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + tne_path = path[:-3] + "tne" + error = urllib.error.HTTPError( + url="http://example.test/2/3/4.png", + code=404, + msg="Not Found", + hdrs={}, + fp=None, + ) + with mock.patch("twms.fetchers.urlopen", side_effect=error) as urlopen: + image = twms.fetchers.Tile(2, 3, 4, layer) + + urlopen.assert_called_once() + self.assertFalse(image) + self.assertTrue(os.path.exists(tne_path)) + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_tile_cache_tne_suppresses_fetch_until_ttl_expires(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/fetchers.py b/twms/fetchers.py index 00be8f7..d996922 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -14,6 +14,7 @@ import time from io import BytesIO from urllib.error import HTTPError +from urllib.error import URLError from urllib.request import Request, urlopen import config @@ -61,11 +62,38 @@ def _upstream_timeout(this_layer): ) +def _upstream_retries(this_layer): + return max( + 1, + int(this_layer.get("upstream_retries", getattr(config, "upstream_retries", 1))), + ) + + +def _upstream_retry_delay(this_layer): + return float( + this_layer.get( + "upstream_retry_delay", getattr(config, "upstream_retry_delay", 0) + ) + ) + + def _read_upstream(url, this_layer): - return urlopen( - _upstream_request(url, this_layer), - timeout=_upstream_timeout(this_layer), - ).read() + attempts = _upstream_retries(this_layer) + for attempt in range(attempts): + try: + return urlopen( + _upstream_request(url, this_layer), + timeout=_upstream_timeout(this_layer), + ).read() + except HTTPError: + raise + except (OSError, URLError): + if attempt + 1 == attempts: + raise + delay = _upstream_retry_delay(this_layer) + if delay: + time.sleep(delay) + def _outside_zoom_limits(z, this_layer): diff --git a/twms/twms.conf b/twms/twms.conf index 2d02e0c..812217d 100644 --- a/twms/twms.conf +++ b/twms/twms.conf @@ -16,6 +16,8 @@ install_path = "/usr/share/twms/" # where to look for bro gpx_cache = "/var/cache/twms/traces/" # where to store cached OSM GPX files deadline = 45 # number of seconds that are given to make up image upstream_timeout = 30 # upstream HTTP timeout in seconds; layers may override with "timeout" +upstream_retries = 1 # total upstream HTTP attempts; layers may override with "upstream_retries" +upstream_retry_delay = 0 # seconds to wait between retry attempts; layers may override default_max_zoom = 18 # can be overridden per layer geometry_color = { # default color for overlayed vectors "LINESTRING": "#ff0000", From 618a447e7c8ade36c271a201e7d142c7abf79763 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:50:33 +0400 Subject: [PATCH 36/51] fix: make RAM tile cache LRU Preserve the legacy max_ram_cached_tiles cache, but make eviction least-recently-used instead of FIFO/off-by-one history popping. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 28 +++++++++++++++++++++++++ twms/twms.py | 42 +++++++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index a57e6c7..ce097a5 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -5,6 +5,7 @@ import json import math import os +from collections import OrderedDict from io import BytesIO import tempfile import threading @@ -521,6 +522,33 @@ def test_legacy_tile_image_reuses_historical_cache_path(self): finally: twms.twms.config.tiles_cache = old_cache + def test_legacy_ram_cache_is_lru_bounded(self): + old_cached_objs = twms.twms.cached_objs + old_limit = twms.twms.config.max_ram_cached_tiles + twms.twms.cached_objs = OrderedDict() + twms.twms.config.max_ram_cached_tiles = 2 + layer = {"prefix": "ram"} + first = Image.new("RGBA", (1, 1), (1, 1, 1, 255)) + second = Image.new("RGBA", (1, 1), (2, 2, 2, 255)) + third = Image.new("RGBA", (1, 1), (3, 3, 3, 255)) + try: + first_key = twms.twms._ram_cache_key(layer, 1, 1, 1) + second_key = twms.twms._ram_cache_key(layer, 1, 2, 2) + third_key = twms.twms._ram_cache_key(layer, 1, 3, 3) + twms.twms._ram_cache_put(first_key, first) + twms.twms._ram_cache_put(second_key, second) + + self.assertIs(twms.twms._ram_cache_get(first_key), first) + + twms.twms._ram_cache_put(third_key, third) + + self.assertIn(first_key, twms.twms.cached_objs) + self.assertNotIn(second_key, twms.twms.cached_objs) + self.assertIn(third_key, twms.twms.cached_objs) + finally: + twms.twms.cached_objs = old_cached_objs + twms.twms.config.max_ram_cached_tiles = old_limit + def test_legacy_gettile_fast_cache_reads_binary_tile(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.twms.config.tiles_cache diff --git a/twms/twms.py b/twms/twms.py index f9ec1b5..5c4b526 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -12,6 +12,7 @@ import sys import time import urllib +from collections import OrderedDict from io import BytesIO sys.path.append(os.path.join(os.path.dirname(__file__))) @@ -49,8 +50,7 @@ OK = 200 ERROR = 500 -cached_objs = {} # a dict. (layer, z, x, y): PIL image -cached_hist_list = [] +cached_objs = OrderedDict() # (layer, z, x, y): PIL image, least-recent first formats = { "image/gif": "GIF", @@ -63,6 +63,27 @@ mimetypes = dict(zip(formats.values(), formats.keys())) +def _ram_cache_key(layer, z, x, y): + return (layer["prefix"], z, x, y) + + +def _ram_cache_get(key): + if key not in cached_objs: + return None + cached_objs.move_to_end(key) + return cached_objs[key] + + +def _ram_cache_put(key, image): + limit = int(getattr(config, "max_ram_cached_tiles", 1024)) + if limit <= 0: + return + cached_objs[key] = image + cached_objs.move_to_end(key) + while len(cached_objs) > limit: + cached_objs.popitem(last=False) + + def twms_main(data): """ Do main TWMS work. @@ -420,10 +441,11 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal fully=False, ): return None - global cached_objs, cached_hist_list + global cached_objs if "prefix" in layer: - if (layer["prefix"], z, x, y) in cached_objs: - return cached_objs[(layer["prefix"], z, x, y)] + cached = _ram_cache_get(_ram_cache_key(layer, z, x, y)) + if cached is not None: + return cached if layer.get("cached", True): local = ( config.tiles_cache @@ -600,16 +622,12 @@ def getimg(bbox, request_proj, size, layer, start_time, force): im1 = tile_image(layer, zoom, x, y, start_time, real=True) if im1: if "prefix" in layer: - if (layer["prefix"], zoom, x, y) not in cached_objs: + cache_key = _ram_cache_key(layer, zoom, x, y) + if cache_key not in cached_objs: if im1.is_ok: - cached_objs[(layer["prefix"], zoom, x, y)] = im1 - cached_hist_list.append((layer["prefix"], zoom, x, y)) + _ram_cache_put(cache_key, im1) # print((layer["prefix"], zoom, x, y), cached_objs[(layer["prefix"], zoom, x, y)], file=sys.stderr) # sys.stderr.flush() - if len(cached_objs) >= config.max_ram_cached_tiles: - del cached_objs[cached_hist_list.pop(0)] - # print("Removed tile from cache", file=sys.stderr) - # sys.stderr.flush() else: ec = ImageColor.getcolor( layer.get("empty_color", config.default_background), "RGBA" From 0dc7b06adf623ec083ba43b81ffd80d26d1ce579 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:53:22 +0400 Subject: [PATCH 37/51] fix: accept MIME response cache keys Keep legacy cache_tile_responses configs working when they use MIME strings such as image/png, as shown in the packaged example, while still accepting the normalized Pillow format keys used internally. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 74 ++++++++++++++++++++++++++++++++++++++ twms/twms.py | 24 +++++++++---- 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index ce097a5..73ea917 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -624,6 +624,44 @@ def test_legacy_response_cache_reads_binary_tile(self): else: del twms.twms.config.cache_tile_responses + def test_legacy_response_cache_accepts_mime_format_key(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = getattr(twms.twms.config, "cache_tile_responses", None) + had_cache = hasattr(twms.twms.config, "cache_tile_responses") + twms.twms.config.cache_tile_responses = { + ("EPSG:3857", ("transparent",), (), 256, 256, (), "image/png"): ( + cache_root, + "png", + ), + } + try: + path = os.path.join(cache_root, "2", "3", "4.png") + os.makedirs(os.path.dirname(path)) + with open(path, "wb") as cached_tile: + cached_tile.write(self.image_bytes((12, 34, 56, 255))) + + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.getpixel((0, 0)), (12, 34, 56, 255)) + finally: + if had_cache: + twms.twms.config.cache_tile_responses = old_cache + else: + del twms.twms.config.cache_tile_responses + def test_legacy_response_cache_writes_binary_tile(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = getattr(twms.twms.config, "cache_tile_responses", None) @@ -660,6 +698,42 @@ def test_legacy_response_cache_writes_binary_tile(self): else: del twms.twms.config.cache_tile_responses + def test_legacy_response_cache_writes_mime_format_key(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = getattr(twms.twms.config, "cache_tile_responses", None) + had_cache = hasattr(twms.twms.config, "cache_tile_responses") + twms.twms.config.cache_tile_responses = { + ("EPSG:3857", ("transparent",), (), 256, 256, (), "image/png"): ( + cache_root, + "png", + ), + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTile", + "layers": "transparent", + "format": "image/png", + "z": "2", + "x": "3", + "y": "4", + } + ) + + path = os.path.join(cache_root, "2", "3", "4.png") + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/png") + self.assertIsInstance(body, bytes) + with open(path, "rb") as cached_tile: + self.assertEqual(cached_tile.read(), body) + with Image.open(path) as image: + self.assertEqual(image.size, (256, 256)) + finally: + if had_cache: + twms.twms.config.cache_tile_responses = old_cache + else: + del twms.twms.config.cache_tile_responses + def test_tile_cache_can_use_zxy_layout(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/twms.py b/twms/twms.py index 5c4b526..e34dc36 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -84,6 +84,17 @@ def _ram_cache_put(key, image): cached_objs.popitem(last=False) +def _response_cache_entry( + response_cache, srs, layers, filt, width, height, force, image_format +): + key_parts = (srs, tuple(layers), filt, width, height, force) + for format_key in (image_format, mimetypes.get(image_format)): + key = key_parts + (format_key,) + if key in response_cache: + return response_cache[key] + return None + + def twms_main(data): """ Do main TWMS work. @@ -219,19 +230,18 @@ def twms_main(data): y = int(data.get("y", 0)) z = int(data.get("z", 1)) + 1 if "cache_tile_responses" in dir(config) and not wkt and (len(gpx) == 0): - if ( + response_cache = _response_cache_entry( + config.cache_tile_responses, srs, - tuple(layer), + layer, filt, width, height, force, format, - ) in config.cache_tile_responses: - - resp_cache_path, resp_ext = config.cache_tile_responses[ - (srs, tuple(layer), filt, width, height, force, format) - ] + ) + if response_cache: + resp_cache_path, resp_ext = response_cache resp_cache_path = resp_cache_path + "/%s/%s/%s.%s" % ( z - 1, x, From d99e15aa56ac64803d0e3abbcfada99738ef2b20 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:55:33 +0400 Subject: [PATCH 38/51] docs: keep cookies explicit in layer headers Document the decision not to import Radioxoma's Firefox cookie-store discovery into upstream TWMS. Private deployments can still provide cookies through per-layer headers or their own Python config code without adding a browser-profile dependency to the tiny server. Co-authored-by: Eugene Dvoretsky --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 4aab1f2..13241a6 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,11 @@ Layer dictionaries usually define: `{ "size": ..., "md5": {...} }` dictionary; dictionaries may also set `http_status` for an upstream status code that should be cached as `.tne` +TWMS intentionally does not read browser cookie stores automatically. If a +private deployment needs a short-lived cookie, copy it into the layer `headers` +or load it from your own local config code so the server does not gain a +browser-profile dependency. + ## Client URLs Assuming the server runs at `http://127.0.0.1:8080/`: From a85b8b4b65ea95350f585cff6ca2b2d0323a7ab8 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 15:58:15 +0400 Subject: [PATCH 39/51] chore: ignore generated build artifacts Keep the useful hygiene from Radioxoma's added ignore file, but reduce it to artifacts TWMS actually produces during tests, builds, and PyInstaller packaging. Co-authored-by: Eugene Dvoretsky --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c713d46 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +*.py[cod] + +build/ +dist/ +*.egg-info/ + +*.spec From 952f1ae6e02321fb1a3421432f61032c457a41bb Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 16:03:17 +0400 Subject: [PATCH 40/51] fix: accept bounds layer metadata alias Accept Radioxoma's readable bounds layer metadata key as an alias for historical data_bounding_box/bbox in tile filtering and generated TileJSON/WMTS metadata without replacing the legacy config contract. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 88 ++++++++++++++++++++++++++++++++++++++ twms/tilejson.py | 2 +- twms/twms.py | 9 +++- twms/wmts.py | 2 +- 4 files changed, 98 insertions(+), 3 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 73ea917..f652e57 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -351,6 +351,34 @@ def test_tilejson_smoke(self): self.assertEqual(doc["minzoom"], 0) self.assertEqual(doc["maxzoom"], 18) + def test_tilejson_accepts_layer_bounds_alias(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "bounded": { + "name": "Bounded", + "prefix": "bounded", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (1.0, 2.0, 3.0, 4.0), + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTileJSON", + "layers": "bounded", + "ref": "http://example.test/", + } + ) + + doc = json.loads(body) + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/json") + self.assertEqual(doc["bounds"], [1.0, 2.0, 3.0, 4.0]) + self.assertEqual(doc["center"], [2.0, 3.0, 0]) + finally: + twms.twms.config.layers = old_layers + def test_josm_imagery_xml_smoke(self): status, content_type, body = twms.twms.twms_main( { @@ -410,6 +438,43 @@ def test_wmts_capabilities_smoke(self): "http://example.test/wmts/osm/{TileMatrix}/{TileCol}/{TileRow}.png", ) + def test_wmts_capabilities_accepts_layer_bounds_alias(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "bounded": { + "name": "Bounded", + "prefix": "bounded", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (1.0, 2.0, 3.0, 4.0), + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "service": "WMTS", + "request": "GetCapabilities", + "ref": "http://example.test/", + } + ) + + root = ET.fromstring(body) + namespaces = { + "wmts": "http://www.opengis.net/wmts/1.0", + "ows": "http://www.opengis.net/ows/1.1", + } + bounds = root.find( + "./wmts:Contents/wmts:Layer[ows:Identifier='bounded']/ows:WGS84BoundingBox", + namespaces, + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(bounds.find("ows:LowerCorner", namespaces).text, "1.0 2.0") + self.assertEqual(bounds.find("ows:UpperCorner", namespaces).text, "3.0 4.0") + finally: + twms.twms.config.layers = old_layers + def test_wmts_kvp_gettile_smoke(self): status, content_type, body = twms.twms.twms_main( { @@ -1088,6 +1153,29 @@ def test_tile_fetcher_respects_min_zoom(self): self.assertIsNone(image) urlopen.assert_not_called() + def test_legacy_tile_image_accepts_layer_bounds_alias(self): + layer = { + "prefix": "bounded", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (170.0, -80.0, 171.0, -79.0), + "scalable": False, + "fetch": twms.fetchers.Tile, + "remote_url": "http://example.test/%s/%s/%s.png", + } + + with mock.patch("twms.fetchers.fetch") as fetch: + image = twms.twms.tile_image( + layer, + 2, + 0, + 0, + datetime.datetime.now(), + ) + + self.assertIsNone(image) + fetch.assert_not_called() + def test_wms_fetcher_respects_min_zoom(self): layer = { "prefix": "wms-minzoom", diff --git a/twms/tilejson.py b/twms/tilejson.py index 29bc1bb..f37bf3c 100644 --- a/twms/tilejson.py +++ b/twms/tilejson.py @@ -25,7 +25,7 @@ def _layer_names(config, layers): def _layer_bounds(config, layer): return layer.get( "data_bounding_box", - layer.get("bbox", config.default_bbox), + layer.get("bounds", layer.get("bbox", config.default_bbox)), ) diff --git a/twms/twms.py b/twms/twms.py index e34dc36..8ae831a 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -95,6 +95,13 @@ def _response_cache_entry( return None +def _layer_bounds(layer): + return layer.get( + "data_bounding_box", + layer.get("bounds", layer.get("bbox", config.default_bbox)), + ) + + def twms_main(data): """ Do main TWMS work. @@ -447,7 +454,7 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal return None if not bbox.bbox_is_in( projections.bbox_by_tile(z, x, y, layer["proj"]), - layer.get("data_bounding_box", config.default_bbox), + _layer_bounds(layer), fully=False, ): return None diff --git a/twms/wmts.py b/twms/wmts.py index 289f79a..3d1a796 100644 --- a/twms/wmts.py +++ b/twms/wmts.py @@ -34,7 +34,7 @@ def _mime_type(layer): def _layer_bounds(config, layer): return layer.get( "data_bounding_box", - layer.get("bbox", config.default_bbox), + layer.get("bounds", layer.get("bbox", config.default_bbox)), ) From f168719242fd977cb67c545565439819e01326d6 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 16:07:39 +0400 Subject: [PATCH 41/51] feat: enrich JOSM imagery metadata Harvest Radioxoma's useful JOSM imagery metadata idea without importing the larger API/config rewrite or launching JOSM remote-control side effects. The generated maps-1.0 XML now carries layer bounds, overlays, attribution URLs, valid-georeference/default hints, and fixed per-MD5 no-tile checksums while preserving TWMS' existing URL and zoom semantics. Co-authored-by: Eugene Dvoretsky --- README.md | 3 ++ tests/test_legacy_smoke.py | 69 ++++++++++++++++++++++++++++++++++++++ twms/josm.py | 48 +++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 13241a6..7749f79 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,9 @@ JOSM can also consume the generated imagery list: http://127.0.0.1:8080/josm/imagery.xml ``` +The generated list includes configured layer bounds, zoom limits, overlays, +attribution URLs, and known no-tile MD5 checksums when those are present. + For TWMS-specific parameters, use the WMS-style `GetTile` URL instead: ```text diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index f652e57..be23c41 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -395,14 +395,83 @@ def test_josm_imagery_xml_smoke(self): self.assertEqual(status, 200) self.assertEqual(content_type, "text/xml") self.assertEqual(root.tag, "{http://josm.openstreetmap.de/maps-1.0}imagery") + self.assertEqual(osm.find("josm:default", namespaces).text, "true") self.assertEqual(osm.find("josm:name", namespaces).text, "OpenStreetMap mapnik") self.assertEqual(osm.find("josm:type", namespaces).text, "tms") self.assertEqual( osm.find("josm:url", namespaces).text, "http://example.test/osm/{zoom}/{x}/{y}.png", ) + self.assertEqual( + osm.find("josm:description", namespaces).text, + "OpenStreetMap mapnik", + ) + self.assertEqual(osm.find("josm:valid-georeference", namespaces).text, "true") self.assertEqual(landsat.find("josm:max-zoom", namespaces).text, "11") + def test_josm_imagery_xml_layer_metadata(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "metadata": { + "name": "Metadata", + "prefix": "metadata", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (1.0, 2.0, 3.0, 4.0), + "overlay": True, + "provider_url": "http://provider.example/", + "dead_tile": { + "md5": { + "11111111111111111111111111111111", + "22222222222222222222222222222222", + }, + }, + "min_zoom": 3, + "max_zoom": 7, + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetJOSMImagery", + "ref": "http://example.test/", + } + ) + + namespaces = {"josm": "http://josm.openstreetmap.de/maps-1.0"} + root = ET.fromstring(body) + entry = root.find("./josm:entry[josm:id='twms-metadata']", namespaces) + bounds = entry.find("josm:bounds", namespaces) + checksums = entry.findall("josm:no-tile-checksum", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(entry.attrib["overlay"], "true") + self.assertEqual( + entry.find("josm:attribution-url", namespaces).text, + "http://provider.example/", + ) + self.assertEqual( + bounds.attrib, + { + "min-lon": "1.0", + "min-lat": "2.0", + "max-lon": "3.0", + "max-lat": "4.0", + }, + ) + self.assertEqual( + [(item.attrib["type"], item.attrib["value"]) for item in checksums], + [ + ("MD5", "11111111111111111111111111111111"), + ("MD5", "22222222222222222222222222222222"), + ], + ) + self.assertEqual(entry.find("josm:min-zoom", namespaces).text, "3") + self.assertEqual(entry.find("josm:max-zoom", namespaces).text, "6") + finally: + twms.twms.config.layers = old_layers + def test_wmts_capabilities_smoke(self): status, content_type, body = twms.twms.twms_main( { diff --git a/twms/josm.py b/twms/josm.py index e21dd94..a6caad7 100644 --- a/twms/josm.py +++ b/twms/josm.py @@ -28,17 +28,63 @@ def _layer_url(ref, layer_name, layer): ) +def _layer_bounds(config, layer): + return layer.get( + "data_bounding_box", + layer.get("bounds", layer.get("bbox", config.default_bbox)), + ) + + +def _dead_tile_md5_values(layer): + dead_tile = layer.get("dead_tile") + if not isinstance(dead_tile, dict) or "md5" not in dead_tile: + return () + md5 = dead_tile["md5"] + if isinstance(md5, str): + return (md5,) + try: + return tuple(sorted(md5)) + except TypeError: + return () + + def document(config, ref): root = ET.Element(_tag("imagery")) for layer_name in sorted(config.layers): layer = config.layers[layer_name] - entry = ET.SubElement(root, _tag("entry")) + attrs = {} + if layer.get("overlay"): + attrs["overlay"] = "true" + entry = ET.SubElement(root, _tag("entry"), attrs) + ET.SubElement(entry, _tag("default")).text = "true" ET.SubElement(entry, _tag("name")).text = layer.get("name", layer_name) ET.SubElement(entry, _tag("id")).text = "twms-%s" % layer_name ET.SubElement(entry, _tag("type")).text = "tms" ET.SubElement(entry, _tag("url")).text = _layer_url(ref, layer_name, layer) + ET.SubElement(entry, _tag("description")).text = layer.get("name", layer_name) + bounds = _layer_bounds(config, layer) + ET.SubElement( + entry, + _tag("bounds"), + { + "min-lon": str(bounds[0]), + "min-lat": str(bounds[1]), + "max-lon": str(bounds[2]), + "max-lat": str(bounds[3]), + }, + ) + ET.SubElement(entry, _tag("valid-georeference")).text = "true" if "provider_url" in layer: ET.SubElement(entry, _tag("attribution-url")).text = layer["provider_url"] + for md5 in _dead_tile_md5_values(layer): + ET.SubElement( + entry, + _tag("no-tile-checksum"), + { + "type": "MD5", + "value": md5, + }, + ) if "max_zoom" in layer: ET.SubElement(entry, _tag("max-zoom")).text = str(layer["max_zoom"] - 1) if "min_zoom" in layer: From b29d80d4917a909bd233b428a18aa96d27d29148 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 16:10:07 +0400 Subject: [PATCH 42/51] fix: align overview layer metadata Keep the legacy overview page but harvest Radioxoma's useful layer metadata behavior: accept the readable bounds alias for preview/bbox links and link configured provider URLs without reshaping the overview UI. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 32 ++++++++++++++++++++++++++++++++ twms/overview.py | 20 ++++++++++++++++---- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index be23c41..7b88fee 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -152,6 +152,38 @@ def test_overview_smoke(self): self.assertIn("
", body) self.assertIn("Yandex Satellite", body) + def test_overview_accepts_bounds_alias_and_provider_link(self): + old_config_layers = twms.twms.config.layers + old_overview_layers = twms.twms.overview.layers + layers = { + "bounded": { + "name": "Bounded", + "prefix": "bounded", + "ext": "png", + "proj": "EPSG:3857", + "bounds": (1.0, 2.0, 3.0, 4.0), + "provider_url": "http://provider.example/", + } + } + twms.twms.config.layers = layers + twms.twms.overview.layers = layers + try: + status, content_type, body = twms.twms.twms_main( + {"ref": "http://example.test/"} + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/html") + self.assertIn("bbox=1.0,2.0,3.0,4.0", body) + self.assertIn( + '' + "Bounded", + body, + ) + finally: + twms.twms.config.layers = old_config_layers + twms.twms.overview.layers = old_overview_layers + def test_gettile_transparent_layer_smoke(self): status, content_type, body = twms.twms.twms_main( { diff --git a/twms/overview.py b/twms/overview.py index f2f646f..67d35b4 100644 --- a/twms/overview.py +++ b/twms/overview.py @@ -9,6 +9,13 @@ from config import * +def _layer_bounds(layer): + return layer.get( + "data_bounding_box", + layer.get("bounds", projections.projs[layer["proj"]]["bounds"]), + ) + + def html(ref): """ Gives overall information about twms server and its layers in HTML format. @@ -20,9 +27,7 @@ def html(ref): resp += wms_name resp += "
" for i in layers: - bbox = layers[i].get( - "data_bounding_box", projections.projs[layers[i]["proj"]]["bounds"] - ) + bbox = _layer_bounds(layers[i]) resp += '" diff --git a/twms/tilejson.py b/twms/tilejson.py index f37bf3c..c9b3b1e 100644 --- a/twms/tilejson.py +++ b/twms/tilejson.py @@ -33,7 +33,14 @@ def _tile_extension(config, layer_names, format_name): if format_name: return format_name.lower().replace("image/", "").replace("jpeg", "jpg") if len(layer_names) == 1: - return config.layers[layer_names[0]].get("ext", "jpg") + layer = config.layers[layer_names[0]] + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg") + .lower() + .replace("image/", "") + .replace("jpeg", "jpg"), + ) return config.default_format.lower().replace("image/", "").replace("jpeg", "jpg") diff --git a/twms/twms.py b/twms/twms.py index 8ae831a..cab0802 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -102,6 +102,13 @@ def _layer_bounds(layer): ) +def _layer_extension(layer): + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg").lower().replace("image/", ""), + ).lower().replace("jpeg", "jpg") + + def twms_main(data): """ Do main TWMS work. @@ -273,7 +280,7 @@ def twms_main(data): + config.layers[layer[0]]["prefix"] + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) ) - ext = config.layers[layer[0]]["ext"] + ext = _layer_extension(config.layers[layer[0]]) adds = ["", "ups."] for add in adds: if os.path.exists(local + add + ext): @@ -469,7 +476,7 @@ def tile_image(layer, z, x, y, start_time, again=False, trybetter=True, real=Fal + layer["prefix"] + "/z%s/%s/x%s/%s/y%s." % (z, x // 1024, x, y // 1024, y) ) - ext = layer["ext"] + ext = _layer_extension(layer) if "cache_ttl" in layer: for ex in [ext, "dsc." + ext, "ups." + ext, "tne"]: f = local + ex diff --git a/twms/wmts.py b/twms/wmts.py index 3d1a796..c4b4822 100644 --- a/twms/wmts.py +++ b/twms/wmts.py @@ -21,10 +21,15 @@ def _tag(namespace, name): def _extension(layer): - return layer.get("ext", "jpg").lower().replace("jpeg", "jpg") + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg").lower().replace("image/", ""), + ).lower().replace("jpeg", "jpg") def _mime_type(layer): + if "mimetype" in layer: + return layer["mimetype"] ext = _extension(layer) if ext == "jpg": return "image/jpeg" From f2522cce0f416cf8fb0b7e7fe101ada32424f4e2 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 16:40:20 +0400 Subject: [PATCH 46/51] feat: support WebP WMS output Accept image/webp as a normal WMS GetMap output format and advertise it in the legacy WMS 1.1.1 capabilities response. Keep the change tiny: it reuses Pillow's WEBP encoder through the existing format map instead of adding a new protocol or rendering path. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 20 ++++++++++++++++++++ twms/capabilities.py | 9 ++++++++- twms/twms.py | 1 + 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 9109593..90c3e14 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -112,6 +112,7 @@ def test_wms_capabilities_smoke(self): self.assertEqual(content_type, "application/vnd.ogc.wms_xml") self.assertIn("image/webp", body) self.assertNotIn("CRS:84", body) def test_wms_130_capabilities_smoke(self): @@ -161,6 +162,25 @@ def test_wms_getmap_accepts_crs_parameter(self): with Image.open(BytesIO(body)) as image: self.assertEqual(image.size, (32, 32)) + def test_wms_getmap_accepts_webp_format(self): + status, content_type, body = twms.twms.twms_main( + { + "request": "GetMap", + "layers": "transparent", + "format": "image/webp", + "width": "32", + "height": "32", + "srs": "EPSG:3857", + "bbox": "-1,-1,1,1", + } + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "image/webp") + with Image.open(BytesIO(body)) as image: + self.assertEqual(image.format, "WEBP") + self.assertEqual(image.size, (32, 32)) + def test_overview_smoke(self): status, content_type, body = twms.twms.twms_main({"ref": "http://example.test/"}) diff --git a/twms/capabilities.py b/twms/capabilities.py index 033f091..a5f8b01 100644 --- a/twms/capabilities.py +++ b/twms/capabilities.py @@ -72,7 +72,13 @@ def _wms130(ref): ) get_map = ET.SubElement(request, "GetMap") - for image_format in ("image/png", "image/jpeg", "image/gif", "image/bmp"): + for image_format in ( + "image/png", + "image/jpeg", + "image/gif", + "image/bmp", + "image/webp", + ): ET.SubElement(get_map, "Format").text = image_format ET.SubElement( ET.SubElement(ET.SubElement(ET.SubElement(get_map, "DCPType"), "HTTP"), "Get"), @@ -312,6 +318,7 @@ def get(version, ref): image/jpeg image/gif image/bmp + image/webp diff --git a/twms/twms.py b/twms/twms.py index cab0802..d8adf16 100644 --- a/twms/twms.py +++ b/twms/twms.py @@ -58,6 +58,7 @@ "image/jpg": "JPEG", "image/png": "PNG", "image/bmp": "BMP", + "image/webp": "WEBP", } mimetypes = dict(zip(formats.values(), formats.keys())) From d552b8a265fb89e4e63554fb0f00bda66581ddd2 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 16:43:38 +0400 Subject: [PATCH 47/51] feat: support layer defaults Accept an optional layer_defaults mapping from Python configs and expose it through a tiny dict wrapper rather than importing the fork's full config rewrite. Defaults participate in get() and [] lookups while explicit layer keys still remain distinguishable, and format metadata is normalized for defaults as well as per-layer values. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 71 ++++++++++++++++++++++++++++++++++++++ twms/config_loader.py | 53 ++++++++++++++++++++-------- 2 files changed, 110 insertions(+), 14 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 90c3e14..da7df30 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -79,6 +79,31 @@ def test_layer_metadata_normalizes_ext_and_mimetype(self): self.assertEqual(module.layers["default-format"]["mimetype"], "image/png") self.assertEqual(module.layers["default-format"]["ext"], "png") + def test_layer_metadata_supports_layer_defaults(self): + module = type("Config", (), {})() + module.layer_defaults = { + "mimetype": "image/png", + "proj": "EPSG:3857", + "cached": False, + } + module.layers = { + "defaulted": {"name": "Defaulted", "prefix": "defaulted"}, + "override": {"name": "Override", "prefix": "override", "ext": "jpg"}, + } + + twms.config_loader.normalize_layer_metadata(module) + + defaulted = module.layers["defaulted"] + override = module.layers["override"] + self.assertNotIn("ext", defaulted) + self.assertEqual(defaulted["proj"], "EPSG:3857") + self.assertEqual(defaulted.get("mimetype"), "image/png") + self.assertEqual(defaulted.get("ext"), "png") + self.assertEqual(defaulted.get("cached"), False) + self.assertEqual(override["proj"], "EPSG:3857") + self.assertEqual(override.get("mimetype"), "image/jpeg") + self.assertEqual(override.get("ext"), "jpg") + def test_legacy_modules_import_as_package_modules(self): modules = [ "twms.bbox", @@ -246,6 +271,27 @@ def test_overview_accepts_mimetype_only_layer(self): twms.twms.config.layers = old_config_layers twms.twms.overview.layers = old_overview_layers + def test_overview_accepts_layer_defaults(self): + old_config_layers = twms.twms.config.layers + old_overview_layers = twms.twms.overview.layers + module = type("Config", (), {})() + module.layer_defaults = {"mimetype": "image/png", "proj": "EPSG:3857"} + module.layers = {"defaulted": {"name": "Defaulted", "prefix": "defaulted"}} + twms.config_loader.normalize_layer_metadata(module) + twms.twms.config.layers = module.layers + twms.twms.overview.layers = module.layers + try: + status, content_type, body = twms.twms.twms_main( + {"ref": "http://example.test/"} + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/html") + self.assertIn("http://example.test/defaulted/!/!/!.png", body) + finally: + twms.twms.config.layers = old_config_layers + twms.twms.overview.layers = old_overview_layers + def test_gettile_transparent_layer_smoke(self): status, content_type, body = twms.twms.twms_main( { @@ -499,6 +545,31 @@ def test_tilejson_accepts_mimetype_only_layer(self): finally: twms.twms.config.layers = old_layers + def test_tilejson_accepts_layer_defaults(self): + old_layers = twms.twms.config.layers + module = type("Config", (), {})() + module.layer_defaults = {"mimetype": "image/png", "proj": "EPSG:3857"} + module.layers = {"defaulted": {"name": "Defaulted", "prefix": "defaulted"}} + twms.config_loader.normalize_layer_metadata(module) + twms.twms.config.layers = module.layers + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTileJSON", + "layers": "defaulted", + "ref": "http://example.test/", + } + ) + + doc = json.loads(body) + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/json") + self.assertEqual( + doc["tiles"], ["http://example.test/defaulted/{z}/{x}/{y}.png"] + ) + finally: + twms.twms.config.layers = old_layers + def test_josm_imagery_xml_smoke(self): status, content_type, body = twms.twms.twms_main( { diff --git a/twms/config_loader.py b/twms/config_loader.py index 1421b33..630ca6c 100644 --- a/twms/config_loader.py +++ b/twms/config_loader.py @@ -7,6 +7,22 @@ import sys +class LayerConfig(dict): + def __init__(self, defaults, values): + super().__init__(values) + self.defaults = defaults + + def __missing__(self, key): + if key in self.defaults: + return self.defaults[key] + raise KeyError(key) + + def get(self, key, default=None): + if key in self: + return super().get(key) + return self.defaults.get(key, default) + + def _extension_from_mimetype(mimetype): extension = mimetypes.guess_extension(mimetype or "") if not extension: @@ -23,22 +39,31 @@ def _mimetype_from_extension(extension): return None +def _normalize_format_metadata(layer, default_mimetype=None): + if "mimetype" in layer and "ext" not in layer: + extension = _extension_from_mimetype(layer["mimetype"]) + if extension: + layer["ext"] = extension + if "ext" in layer and "mimetype" not in layer: + mimetype = _mimetype_from_extension(layer["ext"]) + if mimetype: + layer["mimetype"] = mimetype + if "ext" not in layer and "mimetype" not in layer and default_mimetype: + layer["mimetype"] = default_mimetype + extension = _extension_from_mimetype(default_mimetype) + if extension: + layer["ext"] = extension + + def normalize_layer_metadata(module): default_mimetype = getattr(module, "default_format", None) - for layer in getattr(module, "layers", {}).values(): - if "mimetype" in layer and "ext" not in layer: - extension = _extension_from_mimetype(layer["mimetype"]) - if extension: - layer["ext"] = extension - if "ext" in layer and "mimetype" not in layer: - mimetype = _mimetype_from_extension(layer["ext"]) - if mimetype: - layer["mimetype"] = mimetype - if "ext" not in layer and "mimetype" not in layer and default_mimetype: - layer["mimetype"] = default_mimetype - extension = _extension_from_mimetype(default_mimetype) - if extension: - layer["ext"] = extension + layer_defaults = getattr(module, "layer_defaults", None) + if isinstance(layer_defaults, dict): + _normalize_format_metadata(layer_defaults, default_mimetype) + for name, layer in list(getattr(module, "layers", {}).items()): + _normalize_format_metadata(layer, default_mimetype) + if isinstance(layer_defaults, dict): + module.layers[name] = LayerConfig(layer_defaults, layer) def load_config(path): From d0b1ed1266e373dbd1ec9a55ae10c9b1ae29463d Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 16:48:35 +0400 Subject: [PATCH 48/51] feat: add WMS and JOSM route aliases Keep the stdlib server compatible with the useful route shape from the fork by accepting /wms//// tile URLs, /wms query requests with a /wms service URL, and /josm/maps.xml. This keeps the route layer tiny and delegates behavior to the existing twms_main requests instead of importing the fork's larger api.py split. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 27 +++++++++++++++++++ twms/server.py | 55 +++++++++++++++++++++----------------- 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index da7df30..dc67c37 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -1766,6 +1766,14 @@ def test_stdlib_server_serves_wms_and_gettile(self): self.assertIn("application/vnd.ogc.wms_xml", response.headers["Content-Type"]) self.assertIn(" Date: Sun, 26 Jul 2026 16:53:17 +0400 Subject: [PATCH 49/51] chore: print server startup URLs List the useful local overview, WMS, WMTS, and JOSM imagery URLs when the stdlib server starts, and expose the TWMS version in the Server header. This harvests the helpful operator-facing part of the fork's server startup changes without importing its logging/color-output rewrite. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 18 ++++++++++++++++++ twms/server.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index dc67c37..4863270 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -1751,6 +1751,23 @@ def test_named_tile_template_uses_transform_tuple(self): def test_wsgi_application_imports(self): self.assertTrue(callable(twms.daemon.application)) + def test_stdlib_server_startup_banner_lists_client_urls(self): + banner = twms.server.startup_banner("", 8080) + + self.assertIn("TWMS server 0.07z listening on 0.0.0.0:8080", banner) + self.assertIn("Overview: http://127.0.0.1:8080/", banner) + self.assertIn( + "WMS: http://127.0.0.1:8080/wms?SERVICE=WMS&REQUEST=GetCapabilities", + banner, + ) + self.assertIn( + "WMTS: http://127.0.0.1:8080/wmts/1.0.0/WMTSCapabilities.xml", + banner, + ) + self.assertIn("JOSM imagery: http://127.0.0.1:8080/josm/maps.xml", banner) + self.assertNotIn("Cookie", banner) + self.assertNotIn("headers", banner.lower()) + def test_stdlib_server_serves_wms_and_gettile(self): httpd = ThreadingHTTPServer(("127.0.0.1", 0), twms.server.TWMSRequestHandler) thread = threading.Thread(target=httpd.serve_forever) @@ -1763,6 +1780,7 @@ def test_stdlib_server_serves_wms_and_gettile(self): ) as response: body = response.read().decode("utf-8") self.assertEqual(response.status, 200) + self.assertIn("twms/0.07z", response.headers["Server"]) self.assertIn("application/vnd.ogc.wms_xml", response.headers["Content-Type"]) self.assertIn(" Date: Sun, 26 Jul 2026 16:58:26 +0400 Subject: [PATCH 50/51] feat: accept string fetcher aliases Allow readable config fetch values such as tms, tile, and wms to resolve to the existing legacy fetcher functions. This harvests the useful config readability part of the fork without importing its TileFetcher class rewrite or provider-specific dynamic URL discovery. Co-authored-by: Eugene Dvoretsky --- README.md | 3 ++- tests/test_legacy_smoke.py | 30 ++++++++++++++++++++++++++++++ twms/config_loader.py | 20 ++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a39b26a..01f5fa8 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,8 @@ Layer dictionaries usually define: templates may use `{bbox}`, `{width}`, `{height}`, and `{proj}` - `headers`: optional upstream HTTP request headers, such as `Referer`, `User-Agent`, or authentication cookies required by a particular source -- `fetch`: fetcher function, normally `fetchers.Tile` +- `fetch`: fetcher function, normally `fetchers.Tile`; readable aliases + `"tms"` / `"tile"` and `"wms"` are also accepted - `timeout`: optional per-layer upstream HTTP timeout in seconds; set to `None` only if an old deployment deliberately wants the historical unbounded wait diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 4863270..415b1e2 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -104,6 +104,36 @@ def test_layer_metadata_supports_layer_defaults(self): self.assertEqual(override.get("mimetype"), "image/jpeg") self.assertEqual(override.get("ext"), "jpg") + def test_layer_metadata_accepts_string_fetch_aliases(self): + module = type("Config", (), {})() + module.layers = { + "tiles": {"fetch": "tms"}, + "wms": {"fetch": "wms"}, + } + + twms.config_loader.normalize_layer_metadata(module) + + self.assertIs(module.layers["tiles"]["fetch"], twms.fetchers.Tile) + self.assertIs(module.layers["wms"]["fetch"], twms.fetchers.WMS) + + def test_layer_metadata_accepts_default_string_fetch_alias(self): + module = type("Config", (), {})() + module.layer_defaults = {"fetch": "wms"} + module.layers = { + "defaulted": {"name": "Defaulted", "prefix": "defaulted"}, + } + + twms.config_loader.normalize_layer_metadata(module) + + self.assertIs(module.layers["defaulted"]["fetch"], twms.fetchers.WMS) + + def test_layer_metadata_rejects_unknown_string_fetch_alias(self): + module = type("Config", (), {})() + module.layers = {"bad": {"fetch": "factory-factory"}} + + with self.assertRaisesRegex(ValueError, "factory-factory"): + twms.config_loader.normalize_layer_metadata(module) + def test_legacy_modules_import_as_package_modules(self): modules = [ "twms.bbox", diff --git a/twms/config_loader.py b/twms/config_loader.py index 630ca6c..3c5770c 100644 --- a/twms/config_loader.py +++ b/twms/config_loader.py @@ -55,13 +55,33 @@ def _normalize_format_metadata(layer, default_mimetype=None): layer["ext"] = extension +def _normalize_fetch_metadata(layer): + fetch = layer.get("fetch") + if not isinstance(fetch, str): + return + + from twms import fetchers + + fetchers_by_name = { + "tile": fetchers.Tile, + "tms": fetchers.Tile, + "wms": fetchers.WMS, + } + try: + layer["fetch"] = fetchers_by_name[fetch.lower()] + except KeyError: + raise ValueError("Unknown fetcher alias: %s" % fetch) + + def normalize_layer_metadata(module): default_mimetype = getattr(module, "default_format", None) layer_defaults = getattr(module, "layer_defaults", None) if isinstance(layer_defaults, dict): _normalize_format_metadata(layer_defaults, default_mimetype) + _normalize_fetch_metadata(layer_defaults) for name, layer in list(getattr(module, "layers", {}).items()): _normalize_format_metadata(layer, default_mimetype) + _normalize_fetch_metadata(layer) if isinstance(layer_defaults, dict): module.layers[name] = LayerConfig(layer_defaults, layer) From 7e4d2ed2bf7ed793d040335f0a2af672a7150154 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 17:58:39 +0400 Subject: [PATCH 51/51] ci: make import smoke shell portable Use python -c for the import smoke check so the same workflow step works under both Bash on Linux and PowerShell on Windows. --- .github/workflows/ci.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eaaa2d3..69cd23d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,16 +48,7 @@ jobs: run: python -m compileall -q setup.py index.py tools twms - name: Import WSGI application - run: | - python - <<'PY' - import importlib.metadata - import twms - import twms.daemon - - assert twms.__version__ == "0.07z" - assert importlib.metadata.version("twms") == "0.7+z" - print(type(twms.daemon.application).__name__) - PY + run: python -c "import importlib.metadata; import twms; import twms.daemon; assert twms.__version__ == '0.07z'; assert importlib.metadata.version('twms') == '0.7+z'; print(type(twms.daemon.application).__name__)" - name: Run legacy smoke tests run: python -m unittest discover -s tests

' ) - resp += layers[i]["name"] + if "provider_url" in layers[i]: + resp += 'Bounding box: " + str(bbox) From edabf43a27e4a98db6efa728c29f1eea19004a94 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 16:13:28 +0400 Subject: [PATCH 43/51] chore: ship optional launcher templates Preserve the small launcher deployment idea from Radioxoma's branch without installing desktop integration by default. Package Windows batch and Linux desktop-entry templates under share/twms/contrib and document them as optional manual deployment helpers. Co-authored-by: Eugene Dvoretsky --- README.md | 7 +++++++ contrib/twms.bat | 3 +++ contrib/twms.desktop | 11 +++++++++++ setup.py | 3 ++- 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 contrib/twms.bat create mode 100644 contrib/twms.desktop diff --git a/README.md b/README.md index 7749f79..a39b26a 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,13 @@ On GitHub, CI builds Windows executable artifacts for both entry points: - `twms.exe` - `twms-webpy.exe` +Optional launcher templates are shipped under `share/twms/contrib`: + +- `twms.bat` starts `python -m twms` minimized for small Windows/JOSM proxy + deployments. +- `twms.desktop` is a simple terminal desktop-entry template for Linux + desktops or downstream packages. + ## Configuration twms loads Python configuration from: diff --git a/contrib/twms.bat b/contrib/twms.bat new file mode 100644 index 0000000..c555bec --- /dev/null +++ b/contrib/twms.bat @@ -0,0 +1,3 @@ +@echo off +rem Start TWMS minimized for small Windows/JOSM proxy deployments. +start "twms" /MIN python -m twms %* diff --git a/contrib/twms.desktop b/contrib/twms.desktop new file mode 100644 index 0000000..cb6f868 --- /dev/null +++ b/contrib/twms.desktop @@ -0,0 +1,11 @@ +[Desktop Entry] +Version=1.0 +Type=Application +Name=TWMS +GenericName=Tiny WMS/TMS server +Comment=Start the tiny WMS/TMS tile proxy +Categories=Geoscience;Geography;Network; +Exec=twms +Terminal=true +StartupNotify=false +Icon=utilities-terminal diff --git a/setup.py b/setup.py index b86b7ce..d22973a 100755 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ def initialize_options(self): (os.path.join('share', 'doc', __name__), ['COPYING']), (os.path.join('share', 'doc', __name__), glob('*.md')), (os.path.join('share', __name__), glob('*.jpg')), - (os.path.join('share', __name__, 'tools'), glob(os.path.join('tools', '*.py'))) + (os.path.join('share', __name__, 'tools'), glob(os.path.join('tools', '*.py'))), + (os.path.join('share', __name__, 'contrib'), glob(os.path.join('contrib', '*'))), ] + man_files('*.1') + config_files(), ) From b8b3d18cf99eda242fdeb918859b90a9308ecec0 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 16:22:20 +0400 Subject: [PATCH 44/51] ci: cancel stale branch runs Keep the CI queue useful while the PR is being rebuilt commit-by-commit. Cancel older runs for the same workflow event and branch, but keep push and pull_request groups separate so the latest checks remain visible. --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a983747..eaaa2d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,10 @@ on: - published workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + jobs: package-smoke: name: Package smoke (${{ matrix.os }}, Python ${{ matrix.python-version }}) From a86e8566ab92ebd33b4b86cf7cd60bee348ca505 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 26 Jul 2026 16:30:44 +0400 Subject: [PATCH 45/51] feat: accept layer MIME type metadata Treat a layer mimetype as the source for its tile extension when ext is not configured, and normalize legacy ext-only configs back to MIME types at config load time. Keep existing ext configs working while letting TileJSON, WMTS, JOSM imagery, overview HTML, GetTile cache paths, and downloaded cache writes use mimetype-only layers. Co-authored-by: Eugene Dvoretsky --- tests/test_legacy_smoke.py | 169 ++++++++++++++++++++++++++++++++++++- twms/config_loader.py | 36 ++++++++ twms/fetchers.py | 13 ++- twms/josm.py | 5 +- twms/overview.py | 9 +- twms/tilejson.py | 9 +- twms/twms.py | 11 ++- twms/wmts.py | 7 +- 8 files changed, 248 insertions(+), 11 deletions(-) diff --git a/tests/test_legacy_smoke.py b/tests/test_legacy_smoke.py index 7b88fee..9109593 100644 --- a/tests/test_legacy_smoke.py +++ b/tests/test_legacy_smoke.py @@ -21,6 +21,7 @@ import twms import twms.canvas +import twms.config_loader import twms.daemon import twms.fetchers import twms.filter @@ -46,7 +47,7 @@ def cache_path(self, cache_root, layer, z, x, y): "%s" % (x // 1024), "x%s" % x, "%s" % (y // 1024), - "y%s.%s" % (y, layer["ext"]), + "y%s.%s" % (y, twms.fetchers._layer_extension(layer)), ) def zxy_cache_path(self, cache_root, layer, z, x, y): @@ -55,13 +56,29 @@ def zxy_cache_path(self, cache_root, layer, z, x, y): layer["prefix"], "%s" % z, "%s" % x, - "%s.%s" % (y, layer["ext"]), + "%s.%s" % (y, twms.fetchers._layer_extension(layer)), ) def test_public_version_keeps_keyboard_suffix(self): self.assertEqual(twms.__version__, "0.07z") self.assertEqual(importlib.metadata.version("twms"), "0.7+z") + def test_layer_metadata_normalizes_ext_and_mimetype(self): + module = type("Config", (), {})() + module.default_format = "image/png" + module.layers = { + "mimetype-only": {"mimetype": "image/png"}, + "ext-only": {"ext": "jpg"}, + "default-format": {}, + } + + twms.config_loader.normalize_layer_metadata(module) + + self.assertEqual(module.layers["mimetype-only"]["ext"], "png") + self.assertEqual(module.layers["ext-only"]["mimetype"], "image/jpeg") + self.assertEqual(module.layers["default-format"]["mimetype"], "image/png") + self.assertEqual(module.layers["default-format"]["ext"], "png") + def test_legacy_modules_import_as_package_modules(self): modules = [ "twms.bbox", @@ -184,6 +201,31 @@ def test_overview_accepts_bounds_alias_and_provider_link(self): twms.twms.config.layers = old_config_layers twms.twms.overview.layers = old_overview_layers + def test_overview_accepts_mimetype_only_layer(self): + old_config_layers = twms.twms.config.layers + old_overview_layers = twms.twms.overview.layers + layers = { + "typed": { + "name": "Typed", + "prefix": "typed", + "mimetype": "image/png", + "proj": "EPSG:3857", + } + } + twms.twms.config.layers = layers + twms.twms.overview.layers = layers + try: + status, content_type, body = twms.twms.twms_main( + {"ref": "http://example.test/"} + ) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/html") + self.assertIn("http://example.test/typed/!/!/!.png", body) + finally: + twms.twms.config.layers = old_config_layers + twms.twms.overview.layers = old_overview_layers + def test_gettile_transparent_layer_smoke(self): status, content_type, body = twms.twms.twms_main( { @@ -411,6 +453,32 @@ def test_tilejson_accepts_layer_bounds_alias(self): finally: twms.twms.config.layers = old_layers + def test_tilejson_accepts_mimetype_only_layer(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "typed": { + "name": "Typed", + "prefix": "typed", + "mimetype": "image/png", + "proj": "EPSG:3857", + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetTileJSON", + "layers": "typed", + "ref": "http://example.test/", + } + ) + + doc = json.loads(body) + self.assertEqual(status, 200) + self.assertEqual(content_type, "application/json") + self.assertEqual(doc["tiles"], ["http://example.test/typed/{z}/{x}/{y}.png"]) + finally: + twms.twms.config.layers = old_layers + def test_josm_imagery_xml_smoke(self): status, content_type, body = twms.twms.twms_main( { @@ -441,6 +509,37 @@ def test_josm_imagery_xml_smoke(self): self.assertEqual(osm.find("josm:valid-georeference", namespaces).text, "true") self.assertEqual(landsat.find("josm:max-zoom", namespaces).text, "11") + def test_josm_imagery_xml_accepts_mimetype_only_layer(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "typed": { + "name": "Typed", + "prefix": "typed", + "mimetype": "image/png", + "proj": "EPSG:3857", + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "request": "GetJOSMImagery", + "ref": "http://example.test/", + } + ) + + namespaces = {"josm": "http://josm.openstreetmap.de/maps-1.0"} + root = ET.fromstring(body) + entry = root.find("./josm:entry[josm:id='twms-typed']", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual( + entry.find("josm:url", namespaces).text, + "http://example.test/typed/{zoom}/{x}/{y}.png", + ) + finally: + twms.twms.config.layers = old_layers + def test_josm_imagery_xml_layer_metadata(self): old_layers = twms.twms.config.layers twms.twms.config.layers = { @@ -576,6 +675,48 @@ def test_wmts_capabilities_accepts_layer_bounds_alias(self): finally: twms.twms.config.layers = old_layers + def test_wmts_capabilities_accepts_mimetype_only_layer(self): + old_layers = twms.twms.config.layers + twms.twms.config.layers = { + "typed": { + "name": "Typed", + "prefix": "typed", + "mimetype": "image/png", + "proj": "EPSG:3857", + } + } + try: + status, content_type, body = twms.twms.twms_main( + { + "service": "WMTS", + "request": "GetCapabilities", + "ref": "http://example.test/", + } + ) + + root = ET.fromstring(body) + namespaces = { + "wmts": "http://www.opengis.net/wmts/1.0", + "ows": "http://www.opengis.net/ows/1.1", + } + layer = root.find( + "./wmts:Contents/wmts:Layer[ows:Identifier='typed']", + namespaces, + ) + resource = layer.find("wmts:ResourceURL", namespaces) + format_node = layer.find("wmts:Format", namespaces) + + self.assertEqual(status, 200) + self.assertEqual(content_type, "text/xml") + self.assertEqual(resource.attrib["format"], "image/png") + self.assertEqual( + resource.attrib["template"], + "http://example.test/wmts/typed/{TileMatrix}/{TileCol}/{TileRow}.png", + ) + self.assertEqual(format_node.text, "image/png") + finally: + twms.twms.config.layers = old_layers + def test_wmts_kvp_gettile_smoke(self): status, content_type, body = twms.twms.twms_main( { @@ -1134,6 +1275,30 @@ def test_tile_cache_converts_download_to_layer_extension(self): finally: twms.fetchers.config.tiles_cache = old_cache + def test_tile_cache_accepts_mimetype_only_layer(self): + with tempfile.TemporaryDirectory() as cache_root: + old_cache = twms.fetchers.config.tiles_cache + twms.fetchers.config.tiles_cache = cache_root + os.sep + layer = { + "prefix": "mimetype", + "mimetype": "image/png", + "remote_url": "http://example.test/%s/%s/%s.jpg", + } + try: + path = self.cache_path(cache_root, layer, 2, 3, 4) + with mock.patch("twms.fetchers.urlopen") as urlopen: + urlopen.return_value.read.return_value = self.image_bytes( + (20, 30, 40), image_format="JPEG" + ) + image = twms.fetchers.Tile(2, 3, 4, layer) + + self.assertEqual(image.format, "JPEG") + self.assertTrue(path.endswith(".png")) + with Image.open(path) as cached_image: + self.assertEqual(cached_image.format, "PNG") + finally: + twms.fetchers.config.tiles_cache = old_cache + def test_http_404_tile_is_recorded_as_tne(self): with tempfile.TemporaryDirectory() as cache_root: old_cache = twms.fetchers.config.tiles_cache diff --git a/twms/config_loader.py b/twms/config_loader.py index 039281f..1421b33 100644 --- a/twms/config_loader.py +++ b/twms/config_loader.py @@ -2,10 +2,45 @@ import importlib.machinery import importlib.util +import mimetypes import os import sys +def _extension_from_mimetype(mimetype): + extension = mimetypes.guess_extension(mimetype or "") + if not extension: + return None + return extension.strip(".").lower().replace("jpeg", "jpg") + + +def _mimetype_from_extension(extension): + extension = (extension or "").lower().strip(".").replace("jpeg", "jpg") + if extension == "jpg": + return "image/jpeg" + if extension: + return mimetypes.types_map.get("." + extension, "image/" + extension) + return None + + +def normalize_layer_metadata(module): + default_mimetype = getattr(module, "default_format", None) + for layer in getattr(module, "layers", {}).values(): + if "mimetype" in layer and "ext" not in layer: + extension = _extension_from_mimetype(layer["mimetype"]) + if extension: + layer["ext"] = extension + if "ext" in layer and "mimetype" not in layer: + mimetype = _mimetype_from_extension(layer["ext"]) + if mimetype: + layer["mimetype"] = mimetype + if "ext" not in layer and "mimetype" not in layer and default_mimetype: + layer["mimetype"] = default_mimetype + extension = _extension_from_mimetype(default_mimetype) + if extension: + layer["ext"] = extension + + def load_config(path): loader = importlib.machinery.SourceFileLoader("twms.config", path) spec = importlib.util.spec_from_loader("twms.config", loader) @@ -13,6 +48,7 @@ def load_config(path): sys.modules["twms.config"] = module sys.modules["config"] = module loader.exec_module(module) + normalize_layer_metadata(module) return module diff --git a/twms/fetchers.py b/twms/fetchers.py index d996922..98d3225 100644 --- a/twms/fetchers.py +++ b/twms/fetchers.py @@ -36,6 +36,13 @@ } +def _layer_extension(this_layer): + if "ext" in this_layer: + return this_layer["ext"].lower().strip(".").replace("jpeg", "jpg") + mimetype = this_layer.get("mimetype", "image/jpeg") + return mimetype.lower().replace("image/", "").replace("jpeg", "jpg") + + def _cache_stem(z, x, y, this_layer): cache_prefix = config.tiles_cache + this_layer["prefix"] if this_layer.get("cache_layout") in ("zxy", "slippy", "mobac", "tms"): @@ -137,7 +144,7 @@ def __init__(self, z, x, y, this_layer): self.cached = this_layer.get("cached", True) if self.cached: self.stem = _cache_stem(z, x, y, this_layer) - self.path = self.stem + this_layer["ext"] + self.path = self.stem + _layer_extension(this_layer) self.tne_path = self.stem + "tne" self.lock_path = self.stem + "lock" else: @@ -209,7 +216,7 @@ def save_image(self, image): if not self.cached: return tmp_path = self.path + ".tmp.%s" % os.getpid() - image.save(tmp_path, _EXTENSION_FORMATS.get(self.layer["ext"].lower())) + image.save(tmp_path, _EXTENSION_FORMATS.get(_layer_extension(self.layer))) os.replace(tmp_path, self.path) if os.path.exists(self.tne_path): os.remove(self.tne_path) @@ -353,7 +360,7 @@ def Tile(z, x, y, this_layer): cache.mark_tne() return False if this_layer.get("cached", True): - cache.write_bytes(_cache_image_bytes(contents, im, this_layer["ext"])) + cache.write_bytes(_cache_image_bytes(contents, im, _layer_extension(this_layer))) return im finally: if locked: diff --git a/twms/josm.py b/twms/josm.py index a6caad7..27736e1 100644 --- a/twms/josm.py +++ b/twms/josm.py @@ -17,7 +17,10 @@ def _tag(name): def _layer_extension(layer): - return layer.get("ext", "jpg").lower().replace("jpeg", "jpg") + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg").lower().replace("image/", ""), + ).lower().replace("jpeg", "jpg") def _layer_url(ref, layer_name, layer): diff --git a/twms/overview.py b/twms/overview.py index 67d35b4..2cc91ba 100644 --- a/twms/overview.py +++ b/twms/overview.py @@ -16,6 +16,13 @@ def _layer_bounds(layer): ) +def _layer_extension(layer): + return layer.get( + "ext", + layer.get("mimetype", "image/jpeg").lower().replace("image/", ""), + ).lower().replace("jpeg", "jpg") + + def html(ref): """ Gives overall information about twms server and its layers in HTML format. @@ -59,7 +66,7 @@ def html(ref): + "" + i + "/!/!/!." - + layers[i].get("ext", "jpg") + + _layer_extension(layers[i]) + "
" ) resp += "