From 47442e506bfa5908e569bb76cae10ec1584d4089 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Sat, 31 Jan 2026 15:06:24 +0100 Subject: [PATCH 1/2] Fix: Delete to be expired tiles at the same time as reading them The TRUNCATE didn't work, because there can be tiles from more than one zoom level in the same table. Using DELETE ... RETURNING we are ensured to always delete exactly what we are getting back. --- src/gen/osm2pgsql-gen.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/gen/osm2pgsql-gen.cpp b/src/gen/osm2pgsql-gen.cpp index eba14215f..ce3cc42ef 100644 --- a/src/gen/osm2pgsql-gen.cpp +++ b/src/gen/osm2pgsql-gen.cpp @@ -175,7 +175,7 @@ void get_tiles_from_table(pg_conn_t const &connection, std::string const &table, std::vector> *tiles) { auto const result = connection.exec( - R"(SELECT x, y FROM "{}" WHERE zoom = {})", table, zoom); + R"(DELETE FROM "{}" WHERE zoom = {} RETURNING x, y)", table, zoom); tiles->reserve(result.num_tuples()); @@ -476,8 +476,6 @@ class genproc_t log_debug("Running generalizer for expire list from table '{}'...", table); get_tiles_from_table(db_connection, table, zoom, &tile_list); - log_debug("Truncating table '{}'...", table); - db_connection.exec("TRUNCATE {}", table); } else { auto const extent = get_extent_from_db(db_connection, m_dbschema, params, zoom); From e6c0986cce8ecbf9f4602d097bd58296e07ef45b Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Sat, 31 Jan 2026 16:14:58 +0100 Subject: [PATCH 2/2] Add max_tiles_per_run setting for tile generalizers Limits the number of tiles for which the generalizer is run. Used when generalization is too expensive to always run for all expired tiles. There is a good chance that later changes in the data affect the same tile again, so in the long run this saves effort. Tiles are generalized oldest first when this is used. Don't set max_tiles_per_run or set to 0 to run generalization for all tiles in the expire table for that zoom level. --- src/gen/osm2pgsql-gen.cpp | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/gen/osm2pgsql-gen.cpp b/src/gen/osm2pgsql-gen.cpp index ce3cc42ef..10f2d5a27 100644 --- a/src/gen/osm2pgsql-gen.cpp +++ b/src/gen/osm2pgsql-gen.cpp @@ -171,11 +171,30 @@ tile_extent get_extent_from_db(pg_conn_t const &db_connection, } void get_tiles_from_table(pg_conn_t const &connection, std::string const &table, - uint32_t zoom, + uint32_t zoom, int64_t max_tiles_per_run, std::vector> *tiles) { - auto const result = connection.exec( - R"(DELETE FROM "{}" WHERE zoom = {} RETURNING x, y)", table, zoom); + std::string query; + if (max_tiles_per_run == 0) { + query = fmt::format( + R"(DELETE FROM "{}" WHERE zoom = {} RETURNING x, y)", table, zoom); + } else { + query = fmt::format(R"( +WITH to_delete AS ( + SELECT t.ctid FROM "{0}" AS t + WHERE zoom = {1} + ORDER BY first + FOR UPDATE + LIMIT {2} +) +DELETE FROM "{0}" AS et + USING to_delete AS del + WHERE et.ctid = del.ctid + RETURNING x, y;)", + table, zoom, max_tiles_per_run); + } + + auto const result = connection.exec(query); tiles->reserve(result.num_tuples()); @@ -473,9 +492,12 @@ class genproc_t std::vector> tile_list; if (m_append) { auto const table = params.get_string("expire_list"); + auto const max_tiles_per_run = + params.get_int64("max_tiles_per_run", 0); log_debug("Running generalizer for expire list from table '{}'...", table); - get_tiles_from_table(db_connection, table, zoom, &tile_list); + get_tiles_from_table(db_connection, table, zoom, max_tiles_per_run, + &tile_list); } else { auto const extent = get_extent_from_db(db_connection, m_dbschema, params, zoom);