From 0e4344377a0e538f94967393f140e50207ef4140 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Tue, 5 Nov 2024 10:54:25 +0100
Subject: [PATCH 01/17] automatically create a country index before import
---
src/main/java/de/komoot/photon/App.java | 7 +++++--
.../photon/nominatim/NominatimConnector.java | 19 ++++++++++++++++++-
2 files changed, 23 insertions(+), 3 deletions(-)
diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java
index b4bf50e0..3f77e810 100644
--- a/src/main/java/de/komoot/photon/App.java
+++ b/src/main/java/de/komoot/photon/App.java
@@ -121,14 +121,17 @@ private static void startJsonDump(CommandLineArgs args) {
* Read all data from a Nominatim database and import it into a Photon database.
*/
private static void startNominatimImport(CommandLineArgs args, Server esServer) {
- DatabaseProperties dbProperties;
- NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
+ final var nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
Date importDate = nominatimConnector.getLastImportDate();
+
+ DatabaseProperties dbProperties;
try {
dbProperties = esServer.recreateIndex(args.getLanguages(), importDate, args.getSupportStructuredQueries()); // clear out previous data
} catch (IOException e) {
throw new UsageException("Cannot setup index, elastic search config files not readable");
}
+ LOGGER.info("Preparing Nominatim database for export.");
+ nominatimConnector.prepareDatabase();
LOGGER.info("Starting import from nominatim to photon with languages: {}", String.join(",", dbProperties.getLanguages()));
nominatimConnector.setImporter(esServer.createImporter(dbProperties.getLanguages(), args.getExtraTags()));
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index eebe9ab3..9d0c7608 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -92,7 +92,7 @@ public NominatimConnector(String host, int port, String database, String usernam
}
public NominatimConnector(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) {
- BasicDataSource dataSource = buildDataSource(host, port, database, username, password, false);
+ BasicDataSource dataSource = buildDataSource(host, port, database, username, password, true);
template = new JdbcTemplate(dataSource);
template.setFetchSize(100000);
@@ -333,4 +333,21 @@ private void completePlace(PhotonDoc doc) {
public DBDataAdapter getDataAdaptor() {
return dbutils;
}
+
+ /**
+ * Prepare the database for export.
+ *
+ * This function ensures that the proper index are available and if
+ * not will create them. This may take a while.
+ */
+ public void prepareDatabase() {
+ Integer indexRowNum = template.queryForObject(
+ "SELECT count(*) FROM pg_indexes WHERE tablename = 'placex' AND indexdef LIKE '%(country_code)'",
+ Integer.class);
+
+ if (indexRowNum == null || indexRowNum == 0) {
+ LOGGER.info("Creating index over countries.");
+ template.execute("CREATE INDEX ON placex (country_code)");
+ }
+ }
}
From 6d6ae5b02b2dd331b732e6ccac6edc6e11bdf813 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Tue, 5 Nov 2024 16:53:19 +0100
Subject: [PATCH 02/17] run import for each country separately
---
src/main/java/de/komoot/photon/App.java | 20 ++++-
.../photon/nominatim/NominatimConnector.java | 89 +++++++++++--------
.../photon/nominatim/NominatimUpdater.java | 1 +
.../nominatim/NominatimConnectorDBTest.java | 4 +-
.../nominatim/NominatimConnectorTest.java | 15 ----
src/test/resources/test-schema.sql | 6 ++
6 files changed, 79 insertions(+), 56 deletions(-)
delete mode 100644 src/test/java/de/komoot/photon/nominatim/NominatimConnectorTest.java
diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java
index 3f77e810..1893e320 100644
--- a/src/main/java/de/komoot/photon/App.java
+++ b/src/main/java/de/komoot/photon/App.java
@@ -109,7 +109,15 @@ private static void startJsonDump(CommandLineArgs args) {
final JsonDumper jsonDumper = new JsonDumper(filename, args.getLanguages(), args.getExtraTags());
NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
nominatimConnector.setImporter(jsonDumper);
- nominatimConnector.readEntireDatabase(args.getCountryCodes());
+ if (args.getCountryCodes().length > 0) {
+ for (var countryCode: args.getCountryCodes()) {
+ if (!countryCode.isBlank()) {
+ nominatimConnector.readCountry(countryCode.strip());
+ }
+ }
+ } else {
+ nominatimConnector.readEntireDatabase();
+ }
LOGGER.info("Json dump was created: {}", filename);
} catch (FileNotFoundException e) {
throw new UsageException("Cannot create dump: " + e.getMessage());
@@ -135,7 +143,15 @@ private static void startNominatimImport(CommandLineArgs args, Server esServer)
LOGGER.info("Starting import from nominatim to photon with languages: {}", String.join(",", dbProperties.getLanguages()));
nominatimConnector.setImporter(esServer.createImporter(dbProperties.getLanguages(), args.getExtraTags()));
- nominatimConnector.readEntireDatabase(args.getCountryCodes());
+ if (args.getCountryCodes().length > 0) {
+ for (var countryCode: args.getCountryCodes()) {
+ if (!countryCode.isBlank()) {
+ nominatimConnector.readCountry(countryCode.strip());
+ }
+ }
+ } else {
+ nominatimConnector.readEntireDatabase();
+ }
LOGGER.info("Imported data from nominatim to photon with languages: {}", String.join(",", dbProperties.getLanguages()));
}
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 9d0c7608..8164a0c0 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -8,7 +8,10 @@
import org.apache.commons.dbcp2.BasicDataSource;
import org.slf4j.Logger;
import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -17,6 +20,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
/**
* Importer for data from a Mominatim database.
@@ -69,7 +73,7 @@ public NominatimResult mapRow(ResultSet rs, int rowNum) throws SQLException {
// Add address last, so it takes precedence.
doc.address(address);
- doc.setCountry(getCountryNames(rs.getString("country_code")));
+ doc.setCountry(countryNames.get(rs.getString("country_code")));
NominatimResult result = new NominatimResult(doc);
result.addHousenumbersFromAddress(address);
@@ -114,7 +118,7 @@ public NominatimConnector(String host, int port, String database, String usernam
completePlace(doc);
- doc.setCountry(getCountryNames(rs.getString("country_code")));
+ doc.setCountry(countryNames.get(rs.getString("country_code")));
NominatimResult result = new NominatimResult(doc);
result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"),
@@ -136,7 +140,7 @@ public NominatimConnector(String host, int port, String database, String usernam
completePlace(doc);
- doc.setCountry(getCountryNames(rs.getString("country_code")));
+ doc.setCountry(countryNames.get(rs.getString("country_code")));
NominatimResult result = new NominatimResult(doc);
result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"),
@@ -160,17 +164,16 @@ static BasicDataSource buildDataSource(String host, int port, String database, S
return dataSource;
}
- private Map<String, String> getCountryNames(String countrycode) {
+ public void loadCountryNames() {
if (countryNames == null) {
countryNames = new HashMap<>();
template.query("SELECT country_code, name FROM country_name", rs -> {
countryNames.put(rs.getString("country_code"), dbutils.getMap(rs, "name"));
});
}
-
- return countryNames.get(countrycode);
}
+
public void setImporter(Importer importer) {
this.importer = importer;
}
@@ -239,58 +242,68 @@ List<AddressRow> getAddresses(PhotonDoc doc) {
return terms;
}
- static String convertCountryCode(String... countryCodes) {
- String countryCodeStr = "";
- for (String cc : countryCodes) {
- if (cc.isEmpty())
- continue;
- if (cc.length() != 2)
- throw new IllegalArgumentException("country code invalid " + cc);
- if (!countryCodeStr.isEmpty())
- countryCodeStr += ",";
- countryCodeStr += "'" + cc.toLowerCase() + "'";
- }
- return countryCodeStr;
- }
-
/**
* Parse every relevant row in placex, create a corresponding document and call the {@link #importer} for each document.
*/
- public void readEntireDatabase(String... countryCodes) {
- String andCountryCodeStr = "";
- String countryCodeStr = convertCountryCode(countryCodes);
- if (!countryCodeStr.isEmpty()) {
- andCountryCodeStr = "AND country_code in (" + countryCodeStr + ")";
+ public void readEntireDatabase() {
+ // Make sure, country names are available.
+ loadCountryNames();
+ for (var countryCode: countryNames.keySet()) {
+ readCountry(countryCode);
+ }
+ // Import all places not connected to a country.
+ readCountry(null);
+ }
+
+ public void readCountry(String countryCode) {
+ // Make sure, country names are available.
+ loadCountryNames();
+ if (countryCode != null && !countryNames.containsKey(countryCode)) {
+ LOGGER.warn("Unknown country code {}. Skipping.", countryCode);
+ return;
}
- LOGGER.info("Start importing documents from nominatim ({})", countryCodeStr.isEmpty() ? "global" : countryCodeStr);
+ LOGGER.info("Importing places for country {}.", countryCode);
- ImportThread importThread = new ImportThread(importer);
+ final ImportThread importThread = new ImportThread(importer);
- try {
- template.query(SELECT_COLS_PLACEX + " FROM placex " +
- " WHERE linked_place_id IS NULL AND centroid IS NOT NULL " + andCountryCodeStr +
- " ORDER BY geometry_sector, parent_place_id; ", rs -> {
- // turns a placex row into a photon document that gathers all de-normalised information
+ final RowCallbackHandler placeMapper = rs -> {
NominatimResult docs = placeRowMapper.mapRow(rs, 0);
assert (docs != null);
if (docs.isUsefulForIndex()) {
importThread.addDocument(docs);
}
- });
+ };
- template.query(selectOsmlineSql + " FROM location_property_osmline " +
- "WHERE startnumber is not null " +
- andCountryCodeStr +
- " ORDER BY geometry_sector, parent_place_id; ", rs -> {
+ final RowCallbackHandler osmlineMapper = rs -> {
NominatimResult docs = osmlineRowMapper.mapRow(rs, 0);
assert (docs != null);
if (docs.isUsefulForIndex()) {
importThread.addDocument(docs);
}
- });
+ };
+
+ try {
+ if (countryCode == null) {
+ template.query(SELECT_COLS_PLACEX + " FROM placex " +
+ " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code is null" +
+ " ORDER BY geometry_sector, parent_place_id; ", placeMapper);
+
+ template.query(selectOsmlineSql + " FROM location_property_osmline " +
+ "WHERE startnumber is not null AND country_code is null " +
+ " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper);
+ } else {
+ template.query(SELECT_COLS_PLACEX + " FROM placex " +
+ " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code = ?" +
+ " ORDER BY geometry_sector, parent_place_id; ", placeMapper, countryCode);
+
+ template.query(selectOsmlineSql + " FROM location_property_osmline " +
+ "WHERE startnumber is not null AND country_code = ?" +
+ " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper, countryCode);
+
+ }
} finally {
importThread.finish();
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
index 3e3637aa..01455b0c 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
@@ -81,6 +81,7 @@ public void initUpdates(String updateUser) {
public void update() {
if (updateLock.tryLock()) {
try {
+ exporter.loadCountryNames();
updateFromPlacex();
updateFromInterpolations();
updater.finish();
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
index 7edc325e..819175c2 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
@@ -57,7 +57,9 @@ void testImportForSelectedCountries() throws ParseException {
PlacexTestRow place = new PlacexTestRow("amenity", "cafe").name("SpotHU").country("hu").add(jdbc);
new PlacexTestRow("amenity", "cafe").name("SpotDE").country("de").add(jdbc);
new PlacexTestRow("amenity", "cafe").name("SpotUS").country("us").add(jdbc);
- connector.readEntireDatabase("uk", "hu", "nl");
+ connector.readCountry("uk");
+ connector.readCountry("hu");
+ connector.readCountry("nl");
assertEquals(1, importer.size());
importer.assertContains(place);
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorTest.java
deleted file mode 100644
index 5cd8cf08..00000000
--- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorTest.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package de.komoot.photon.nominatim;
-
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-class NominatimConnectorTest {
-
- @Test
- void testConvertCountryCode() {
- assertEquals("", NominatimConnector.convertCountryCode("".split(",")));
- assertEquals("'uk'", NominatimConnector.convertCountryCode("uk".split(",")));
- assertEquals("'uk','de'", NominatimConnector.convertCountryCode("uk,de".split(",")));
- }
-}
\ No newline at end of file
diff --git a/src/test/resources/test-schema.sql b/src/test/resources/test-schema.sql
index 652a9c6a..0da63df8 100644
--- a/src/test/resources/test-schema.sql
+++ b/src/test/resources/test-schema.sql
@@ -63,6 +63,12 @@ CREATE TABLE country_name (
INSERT INTO country_name
VALUES ('de', JSON '{"name" : "Deutschland", "name:en" : "Germany"}', 'de', 2);
+INSERT INTO country_name
+ VALUES ('us', JSON '{"name" : "USA", "name:en" : "United States"}', 'en', 1);
+INSERT INTO country_name
+ VALUES ('hu', JSON '{"name" : "Magyarország", "name:en" : "Hungary"}', 'hu', 12);
+INSERT INTO country_name
+ VALUES ('nl', JSON '{"name" : "Nederland", "name:en" : "Netherlands"}', null, 2);
CREATE TABLE photon_updates (
From f5c0d25fd8f54921f76ce89ded2a88588ffb2e45 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Tue, 5 Nov 2024 17:33:45 +0100
Subject: [PATCH 03/17] move handling of importer thread to App
This way, the thread is only created once.
---
src/main/java/de/komoot/photon/App.java | 60 ++++++++------
.../komoot/photon/nominatim/ImportThread.java | 2 +-
.../photon/nominatim/NominatimConnector.java | 79 +++++++------------
.../nominatim/NominatimConnectorDBTest.java | 55 ++++++++-----
4 files changed, 99 insertions(+), 97 deletions(-)
diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java
index 1893e320..13663313 100644
--- a/src/main/java/de/komoot/photon/App.java
+++ b/src/main/java/de/komoot/photon/App.java
@@ -2,6 +2,7 @@
import com.beust.jcommander.JCommander;
import com.beust.jcommander.ParameterException;
+import de.komoot.photon.nominatim.ImportThread;
import de.komoot.photon.nominatim.NominatimConnector;
import de.komoot.photon.nominatim.NominatimUpdater;
import de.komoot.photon.searcher.ReverseHandler;
@@ -107,17 +108,9 @@ private static void startJsonDump(CommandLineArgs args) {
try {
final String filename = args.getJsonDump();
final JsonDumper jsonDumper = new JsonDumper(filename, args.getLanguages(), args.getExtraTags());
- NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
- nominatimConnector.setImporter(jsonDumper);
- if (args.getCountryCodes().length > 0) {
- for (var countryCode: args.getCountryCodes()) {
- if (!countryCode.isBlank()) {
- nominatimConnector.readCountry(countryCode.strip());
- }
- }
- } else {
- nominatimConnector.readEntireDatabase();
- }
+ final NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
+
+ importFromDatabase(nominatimConnector, jsonDumper, args.getCountryCodes());
LOGGER.info("Json dump was created: {}", filename);
} catch (FileNotFoundException e) {
throw new UsageException("Cannot create dump: " + e.getMessage());
@@ -130,30 +123,45 @@ private static void startJsonDump(CommandLineArgs args) {
*/
private static void startNominatimImport(CommandLineArgs args, Server esServer) {
final var nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
- Date importDate = nominatimConnector.getLastImportDate();
+ final Date importDate = nominatimConnector.getLastImportDate();
- DatabaseProperties dbProperties;
+ String[] languages;
try {
- dbProperties = esServer.recreateIndex(args.getLanguages(), importDate, args.getSupportStructuredQueries()); // clear out previous data
+ // Clear out previous data.
+ var dbProperties = esServer.recreateIndex(args.getLanguages(), importDate, args.getSupportStructuredQueries());
+ languages = dbProperties.getLanguages();
} catch (IOException e) {
throw new UsageException("Cannot setup index, elastic search config files not readable");
}
- LOGGER.info("Preparing Nominatim database for export.");
- nominatimConnector.prepareDatabase();
-
- LOGGER.info("Starting import from nominatim to photon with languages: {}", String.join(",", dbProperties.getLanguages()));
- nominatimConnector.setImporter(esServer.createImporter(dbProperties.getLanguages(), args.getExtraTags()));
- if (args.getCountryCodes().length > 0) {
- for (var countryCode: args.getCountryCodes()) {
- if (!countryCode.isBlank()) {
- nominatimConnector.readCountry(countryCode.strip());
+
+ LOGGER.info("Starting import from nominatim to photon with languages: {}", String.join(",", languages));
+ importFromDatabase(
+ nominatimConnector,
+ esServer.createImporter(languages, args.getExtraTags()),
+ args.getCountryCodes());
+
+ LOGGER.info("Imported data from nominatim to photon with languages: {}", String.join(",", languages));
+ }
+
+ private static void importFromDatabase(NominatimConnector connector, Importer importer, String[] countries) {
+ connector.prepareDatabase();
+
+ ImportThread importThread = new ImportThread(importer);
+
+ try {
+ if (countries != null && countries.length > 0) {
+ for (var countryCode : countries) {
+ if (!countryCode.isBlank()) {
+ connector.readCountry(countryCode.strip(), importThread);
+ }
}
+ } else {
+ connector.readEntireDatabase(importThread);
}
- } else {
- nominatimConnector.readEntireDatabase();
+ } finally {
+ importThread.finish();
}
- LOGGER.info("Imported data from nominatim to photon with languages: {}", String.join(",", dbProperties.getLanguages()));
}
private static void startNominatimUpdateInit(CommandLineArgs args) {
diff --git a/src/main/java/de/komoot/photon/nominatim/ImportThread.java b/src/main/java/de/komoot/photon/nominatim/ImportThread.java
index 86e941e5..f83cf497 100644
--- a/src/main/java/de/komoot/photon/nominatim/ImportThread.java
+++ b/src/main/java/de/komoot/photon/nominatim/ImportThread.java
@@ -11,7 +11,7 @@
/**
* Worker thread for bulk importing data from a Nominatim database.
*/
-class ImportThread {
+public class ImportThread {
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(ImportThread.class);
private static final int PROGRESS_INTERVAL = 50000;
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 8164a0c0..01076e4b 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -1,29 +1,21 @@
package de.komoot.photon.nominatim;
-import org.locationtech.jts.geom.Geometry;
-import de.komoot.photon.Importer;
import de.komoot.photon.PhotonDoc;
import de.komoot.photon.nominatim.model.AddressRow;
import de.komoot.photon.nominatim.model.AddressType;
import org.apache.commons.dbcp2.BasicDataSource;
+import org.locationtech.jts.geom.Geometry;
import org.slf4j.Logger;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.RowMapper;
-import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
-import org.springframework.jdbc.core.namedparam.SqlParameterSource;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
+import java.util.*;
/**
- * Importer for data from a Mominatim database.
+ * Importer for data from a Nominatim database.
*/
public class NominatimConnector {
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimConnector.class);
@@ -42,7 +34,6 @@ public class NominatimConnector {
*/
private final RowMapper<NominatimResult> osmlineRowMapper;
private final String selectOsmlineSql;
- private Importer importer;
/**
@@ -174,10 +165,6 @@ public void loadCountryNames() {
}
- public void setImporter(Importer importer) {
- this.importer = importer;
- }
-
public List<PhotonDoc> getByPlaceId(long placeId) {
List<NominatimResult> result = template.query(SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ? and indexed_status = 0",
placeRowMapper, placeId);
@@ -243,19 +230,20 @@ List<AddressRow> getAddresses(PhotonDoc doc) {
}
/**
- * Parse every relevant row in placex, create a corresponding document and call the {@link #importer} for each document.
+ * Parse every relevant row in placex and location_osmline
+ * country by country. Also imports place from county-less places.
*/
- public void readEntireDatabase() {
+ public void readEntireDatabase(ImportThread importThread) {
// Make sure, country names are available.
loadCountryNames();
for (var countryCode: countryNames.keySet()) {
- readCountry(countryCode);
+ readCountry(countryCode, importThread);
}
// Import all places not connected to a country.
- readCountry(null);
+ readCountry(null, importThread);
}
- public void readCountry(String countryCode) {
+ public void readCountry(String countryCode, ImportThread importThread) {
// Make sure, country names are available.
loadCountryNames();
if (countryCode != null && !countryNames.containsKey(countryCode)) {
@@ -263,10 +251,6 @@ public void readCountry(String countryCode) {
return;
}
- LOGGER.info("Importing places for country {}.", countryCode);
-
- final ImportThread importThread = new ImportThread(importer);
-
final RowCallbackHandler placeMapper = rs -> {
NominatimResult docs = placeRowMapper.mapRow(rs, 0);
assert (docs != null);
@@ -277,36 +261,31 @@ public void readCountry(String countryCode) {
};
final RowCallbackHandler osmlineMapper = rs -> {
- NominatimResult docs = osmlineRowMapper.mapRow(rs, 0);
- assert (docs != null);
+ NominatimResult docs = osmlineRowMapper.mapRow(rs, 0);
+ assert (docs != null);
- if (docs.isUsefulForIndex()) {
- importThread.addDocument(docs);
- }
- };
-
- try {
- if (countryCode == null) {
- template.query(SELECT_COLS_PLACEX + " FROM placex " +
- " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code is null" +
- " ORDER BY geometry_sector, parent_place_id; ", placeMapper);
+ if (docs.isUsefulForIndex()) {
+ importThread.addDocument(docs);
+ }
+ };
- template.query(selectOsmlineSql + " FROM location_property_osmline " +
- "WHERE startnumber is not null AND country_code is null " +
- " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper);
- } else {
- template.query(SELECT_COLS_PLACEX + " FROM placex " +
- " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code = ?" +
- " ORDER BY geometry_sector, parent_place_id; ", placeMapper, countryCode);
+ if (countryCode == null) {
+ template.query(SELECT_COLS_PLACEX + " FROM placex " +
+ " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code is null" +
+ " ORDER BY geometry_sector, parent_place_id; ", placeMapper);
- template.query(selectOsmlineSql + " FROM location_property_osmline " +
- "WHERE startnumber is not null AND country_code = ?" +
- " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper, countryCode);
+ template.query(selectOsmlineSql + " FROM location_property_osmline " +
+ "WHERE startnumber is not null AND country_code is null " +
+ " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper);
+ } else {
+ template.query(SELECT_COLS_PLACEX + " FROM placex " +
+ " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code = ?" +
+ " ORDER BY geometry_sector, parent_place_id; ", placeMapper, countryCode);
- }
+ template.query(selectOsmlineSql + " FROM location_property_osmline " +
+ "WHERE startnumber is not null AND country_code = ?" +
+ " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper, countryCode);
- } finally {
- importThread.finish();
}
}
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
index 819175c2..c155b192 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
@@ -37,16 +37,25 @@ void setup() {
connector = new NominatimConnector(null, 0, null, null, null, new H2DataAdapter());
importer = new CollectingImporter();
- connector.setImporter(importer);
jdbc = new JdbcTemplate(db);
ReflectionTestUtil.setFieldValue(connector, "template", jdbc);
}
+ private void readEntireDatabase() {
+ ImportThread importThread = new ImportThread(importer);
+ try {
+ connector.readEntireDatabase(importThread);
+ } finally {
+ importThread.finish();
+ }
+
+ }
+
@Test
void testSimpleNodeImport() throws ParseException {
PlacexTestRow place = new PlacexTestRow("amenity", "cafe").name("Spot").add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(1, importer.size());
importer.assertContains(place);
@@ -57,9 +66,15 @@ void testImportForSelectedCountries() throws ParseException {
PlacexTestRow place = new PlacexTestRow("amenity", "cafe").name("SpotHU").country("hu").add(jdbc);
new PlacexTestRow("amenity", "cafe").name("SpotDE").country("de").add(jdbc);
new PlacexTestRow("amenity", "cafe").name("SpotUS").country("us").add(jdbc);
- connector.readCountry("uk");
- connector.readCountry("hu");
- connector.readCountry("nl");
+
+ ImportThread importThread = new ImportThread(importer);
+ try {
+ connector.readCountry("uk", importThread);
+ connector.readCountry("hu", importThread);
+ connector.readCountry("nl", importThread);
+ } finally {
+ importThread.finish();
+ }
assertEquals(1, importer.size());
importer.assertContains(place);
@@ -70,7 +85,7 @@ void testImportance() {
PlacexTestRow place1 = new PlacexTestRow("amenity", "cafe").name("Spot").rankSearch(10).add(jdbc);
PlacexTestRow place2 = new PlacexTestRow("amenity", "cafe").name("Spot").importance(0.3).add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(0.5, importer.get(place1.getPlaceId()).getImportance(), 0.00001);
assertEquals(0.3, importer.get(place2.getPlaceId()).getImportance(), 0.00001);
@@ -87,7 +102,7 @@ void testPlaceAddress() throws ParseException {
new PlacexTestRow("place", "county").name("Lost County").rankAddress(12).add(jdbc),
new PlacexTestRow("place", "state").name("Le Havre").rankAddress(8).add(jdbc));
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(6, importer.size());
importer.assertContains(place);
@@ -109,7 +124,7 @@ void testPlaceAddressAddressRank0() throws ParseException {
new PlacexTestRow("place", "county").name("Lost County").rankAddress(12).add(jdbc),
new PlacexTestRow("place", "state").name("Le Havre").rankAddress(8).add(jdbc));
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(3, importer.size());
importer.assertContains(place);
@@ -129,7 +144,7 @@ void testPoiAddress() throws ParseException {
PlacexTestRow place = new PlacexTestRow("place", "house").name("House").parent(parent).add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(3, importer.size());
importer.assertContains(place);
@@ -151,7 +166,7 @@ void testInterpolationPoint() throws ParseException {
OsmlineTestRow osmline =
new OsmlineTestRow().number(45, 45, 1).parent(street).geom("POINT(45 23)").add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(2, importer.size());
@@ -171,7 +186,7 @@ void testInterpolationAny() throws ParseException {
OsmlineTestRow osmline =
new OsmlineTestRow().number(1, 11, 1).parent(street).geom("LINESTRING(0 0, 0 1)").add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(12, importer.size());
@@ -189,7 +204,7 @@ void testInterpolationWithSteps() throws ParseException {
OsmlineTestRow osmline =
new OsmlineTestRow().number(10, 20, 2).parent(street).geom("LINESTRING(0 0, 0 1)").add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(7, importer.size());
@@ -213,7 +228,7 @@ void testAddressMappingDuplicate() {
munip,
new PlacexTestRow("place", "village").name("Dorf").rankAddress(16).add(jdbc));
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(3, importer.size());
@@ -233,7 +248,7 @@ void testAddressMappingAvoidSameTypeAsPlace() {
village.addAddresslines(jdbc, munip);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(2, importer.size());
@@ -251,7 +266,7 @@ void testUnnamedObjectWithHousenumber() {
PlacexTestRow parent = PlacexTestRow.make_street("Main St").add(jdbc);
PlacexTestRow place = new PlacexTestRow("building", "yes").addr("housenumber", "123").parent(parent).add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(2, importer.size());
@@ -267,7 +282,7 @@ void testObjectWithHousenumberList() throws ParseException {
PlacexTestRow parent = PlacexTestRow.make_street("Main St").add(jdbc);
PlacexTestRow place = new PlacexTestRow("building", "yes").addr("housenumber", "1;2a;3").parent(parent).add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(4, importer.size());
@@ -287,7 +302,7 @@ void testObjectWithconscriptionNumber() throws ParseException {
.addr("conscriptionnumber", "99521")
.parent(parent).add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(3, importer.size());
@@ -302,7 +317,7 @@ void testUnnamedObjectWithOutHousenumber() {
PlacexTestRow parent = PlacexTestRow.make_street("Main St").add(jdbc);
new PlacexTestRow("building", "yes").parent(parent).add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(1, importer.size());
@@ -317,7 +332,7 @@ void testInterpolationLines() {
PlacexTestRow parent = PlacexTestRow.make_street("Main St").add(jdbc);
new PlacexTestRow("place", "houses").name("something").parent(parent).add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(1, importer.size());
@@ -331,7 +346,7 @@ void testInterpolationLines() {
void testNoCountry() {
PlacexTestRow place = new PlacexTestRow("building", "yes").name("Building").country(null).add(jdbc);
- connector.readEntireDatabase();
+ readEntireDatabase();
assertEquals(1, importer.size());
From 1887afdcf5d02e3d6b5ec0ff24a4774c201facbb Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Tue, 5 Nov 2024 17:51:49 +0100
Subject: [PATCH 04/17] get rid of readEntireDatabase function
---
src/main/java/de/komoot/photon/App.java | 17 +++++++------
.../photon/nominatim/NominatimConnector.java | 25 +++++++++++--------
.../nominatim/NominatimConnectorDBTest.java | 4 ++-
3 files changed, 26 insertions(+), 20 deletions(-)
diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java
index 13663313..0ca415b0 100644
--- a/src/main/java/de/komoot/photon/App.java
+++ b/src/main/java/de/komoot/photon/App.java
@@ -15,6 +15,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Date;
import static spark.Spark.*;
@@ -146,17 +147,17 @@ private static void startNominatimImport(CommandLineArgs args, Server esServer)
private static void importFromDatabase(NominatimConnector connector, Importer importer, String[] countries) {
connector.prepareDatabase();
+ if (countries == null || countries.length == 0) {
+ countries = connector.getCountriesFromDatabase();
+ } else {
+ countries = Arrays.stream(countries).map(String::trim).filter(s -> !s.isBlank()).toArray(String[]::new);
+ }
+
ImportThread importThread = new ImportThread(importer);
try {
- if (countries != null && countries.length > 0) {
- for (var countryCode : countries) {
- if (!countryCode.isBlank()) {
- connector.readCountry(countryCode.strip(), importThread);
- }
- }
- } else {
- connector.readEntireDatabase(importThread);
+ for (var countryCode : countries) {
+ connector.readCountry(countryCode, importThread);
}
} finally {
importThread.finish();
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 01076e4b..548205c5 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -231,18 +231,8 @@ List<AddressRow> getAddresses(PhotonDoc doc) {
/**
* Parse every relevant row in placex and location_osmline
- * country by country. Also imports place from county-less places.
+ * for the given country. Also imports place from county-less places.
*/
- public void readEntireDatabase(ImportThread importThread) {
- // Make sure, country names are available.
- loadCountryNames();
- for (var countryCode: countryNames.keySet()) {
- readCountry(countryCode, importThread);
- }
- // Import all places not connected to a country.
- readCountry(null, importThread);
- }
-
public void readCountry(String countryCode, ImportThread importThread) {
// Make sure, country names are available.
loadCountryNames();
@@ -342,4 +332,17 @@ public void prepareDatabase() {
template.execute("CREATE INDEX ON placex (country_code)");
}
}
+
+ public String[] getCountriesFromDatabase() {
+ loadCountryNames();
+ String[] countries = new String[countryNames.keySet().size() + 1];
+ countries[0] = null;
+
+ int i = 1;
+ for (var country: countryNames.keySet()) {
+ countries[i++] = country;
+ }
+
+ return countries;
+ }
}
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
index c155b192..2e2ac28a 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
@@ -45,7 +45,9 @@ void setup() {
private void readEntireDatabase() {
ImportThread importThread = new ImportThread(importer);
try {
- connector.readEntireDatabase(importThread);
+ for (var country: connector.getCountriesFromDatabase()) {
+ connector.readCountry(country, importThread);
+ }
} finally {
importThread.finish();
}
From fc3f5a34b071280bc2df1d1c7488d9c2e3f3a297 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Tue, 5 Nov 2024 23:00:13 +0100
Subject: [PATCH 05/17] enable multi-threading for reading from postgresql
---
src/main/java/de/komoot/photon/App.java | 80 +++++++++++++++----
.../de/komoot/photon/CommandLineArgs.java | 7 ++
.../komoot/photon/nominatim/ImportThread.java | 5 +-
.../photon/nominatim/NominatimConnector.java | 6 +-
4 files changed, 76 insertions(+), 22 deletions(-)
diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java
index 0ca415b0..c30d5db6 100644
--- a/src/main/java/de/komoot/photon/App.java
+++ b/src/main/java/de/komoot/photon/App.java
@@ -15,8 +15,8 @@
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Date;
+import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
import static spark.Spark.*;
@@ -109,9 +109,8 @@ private static void startJsonDump(CommandLineArgs args) {
try {
final String filename = args.getJsonDump();
final JsonDumper jsonDumper = new JsonDumper(filename, args.getLanguages(), args.getExtraTags());
- final NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
- importFromDatabase(nominatimConnector, jsonDumper, args.getCountryCodes());
+ importFromDatabase(args, jsonDumper);
LOGGER.info("Json dump was created: {}", filename);
} catch (FileNotFoundException e) {
throw new UsageException("Cannot create dump: " + e.getMessage());
@@ -123,29 +122,33 @@ private static void startJsonDump(CommandLineArgs args) {
* Read all data from a Nominatim database and import it into a Photon database.
*/
private static void startNominatimImport(CommandLineArgs args, Server esServer) {
+ final var languages = initDatabase(args, esServer);
+
+ LOGGER.info("Starting import from nominatim to photon with languages: {}", String.join(",", languages));
+ importFromDatabase(args, esServer.createImporter(languages, args.getExtraTags()));
+
+ LOGGER.info("Imported data from nominatim to photon with languages: {}", String.join(",", languages));
+ }
+
+ private static String[] initDatabase(CommandLineArgs args, Server esServer) {
final var nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
final Date importDate = nominatimConnector.getLastImportDate();
- String[] languages;
try {
// Clear out previous data.
var dbProperties = esServer.recreateIndex(args.getLanguages(), importDate, args.getSupportStructuredQueries());
- languages = dbProperties.getLanguages();
+ return dbProperties.getLanguages();
} catch (IOException e) {
throw new UsageException("Cannot setup index, elastic search config files not readable");
}
-
- LOGGER.info("Starting import from nominatim to photon with languages: {}", String.join(",", languages));
- importFromDatabase(
- nominatimConnector,
- esServer.createImporter(languages, args.getExtraTags()),
- args.getCountryCodes());
-
- LOGGER.info("Imported data from nominatim to photon with languages: {}", String.join(",", languages));
}
- private static void importFromDatabase(NominatimConnector connector, Importer importer, String[] countries) {
+ private static void importFromDatabase(CommandLineArgs args, Importer importer) {
+ final var connector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
connector.prepareDatabase();
+ connector.loadCountryNames();
+
+ String[] countries = args.getCountryCodes();
if (countries == null || countries.length == 0) {
countries = connector.getCountriesFromDatabase();
@@ -153,11 +156,53 @@ private static void importFromDatabase(NominatimConnector connector, Importer im
countries = Arrays.stream(countries).map(String::trim).filter(s -> !s.isBlank()).toArray(String[]::new);
}
+ final int numThreads = args.getThreads();
ImportThread importThread = new ImportThread(importer);
try {
- for (var countryCode : countries) {
- connector.readCountry(countryCode, importThread);
+
+ if (numThreads == 1) {
+ for (var country : countries) {
+ connector.readCountry(country, importThread);
+ }
+ } else {
+ final Queue<String> todolist = new ConcurrentLinkedQueue<>(List.of(countries));
+
+ final List<Thread> readerThreads = new ArrayList<>(numThreads);
+
+ for (int i = 0; i < numThreads; ++i) {
+ final NominatimConnector threadConnector;
+ if (i > 0) {
+ threadConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
+ threadConnector.loadCountryNames();
+ } else {
+ threadConnector = connector;
+ }
+ final int threadno = i;
+ Runnable runner = () -> {
+ String nextCc = todolist.poll();
+ while (nextCc != null) {
+ LOGGER.info("Thread {}: reading country '{}'", threadno, nextCc);
+ threadConnector.readCountry(nextCc, importThread);
+ nextCc = todolist.poll();
+ }
+ };
+ Thread thread = new Thread(runner);
+ thread.start();
+ readerThreads.add(thread);
+ }
+ readerThreads.forEach(t -> {
+ while (true) {
+ try {
+ t.join();
+ break;
+ } catch (InterruptedException e) {
+ LOGGER.warn("Thread interrupted:", e);
+ // Restore interrupted state.
+ Thread.currentThread().interrupt();
+ }
+ }
+ });
}
} finally {
importThread.finish();
@@ -165,6 +210,7 @@ private static void importFromDatabase(NominatimConnector connector, Importer im
}
+
private static void startNominatimUpdateInit(CommandLineArgs args) {
NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
nominatimUpdater.initUpdates(args.getNominatimUpdateInit());
diff --git a/src/main/java/de/komoot/photon/CommandLineArgs.java b/src/main/java/de/komoot/photon/CommandLineArgs.java
index 724f4835..cd496a2b 100644
--- a/src/main/java/de/komoot/photon/CommandLineArgs.java
+++ b/src/main/java/de/komoot/photon/CommandLineArgs.java
@@ -14,6 +14,9 @@
@Parameters(parametersValidators = CorsMutuallyExclusiveValidator.class)
public class CommandLineArgs {
+ @Parameter(names = "-j", description = "Number of threads to use for import.")
+ private int threads = 1;
+
@Parameter(names = "-structured", description = "Enable support for structured queries.")
private boolean supportStructuredQueries = false;
@@ -107,6 +110,10 @@ public String[] getLanguages() {
return getLanguages(true);
}
+ public int getThreads() {
+ return Integer.min(10, Integer.max(0, threads));
+ }
+
public String getCluster() {
return this.cluster;
}
diff --git a/src/main/java/de/komoot/photon/nominatim/ImportThread.java b/src/main/java/de/komoot/photon/nominatim/ImportThread.java
index f83cf497..0344f488 100644
--- a/src/main/java/de/komoot/photon/nominatim/ImportThread.java
+++ b/src/main/java/de/komoot/photon/nominatim/ImportThread.java
@@ -16,7 +16,7 @@ public class ImportThread {
private static final int PROGRESS_INTERVAL = 50000;
private static final NominatimResult FINAL_DOCUMENT = new NominatimResult(new PhotonDoc(0, null, 0, null, null));
- private final BlockingQueue<NominatimResult> documents = new LinkedBlockingDeque<>(20);
+ private final BlockingQueue<NominatimResult> documents = new LinkedBlockingDeque<>(100);
private final AtomicLong counter = new AtomicLong();
private final Importer importer;
private final Thread thread;
@@ -70,7 +70,8 @@ public void finish() {
Thread.currentThread().interrupt();
}
}
- LOGGER.info("Finished import of {} photon documents.", counter.longValue());
+ LOGGER.info("Finished import of {} photon documents. (Total processing time: {}s)",
+ counter.longValue(), (System.currentTimeMillis() - startMillis)/1000);
}
private class ImportRunnable implements Runnable {
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 548205c5..6aede0da 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -236,7 +236,7 @@ List<AddressRow> getAddresses(PhotonDoc doc) {
public void readCountry(String countryCode, ImportThread importThread) {
// Make sure, country names are available.
loadCountryNames();
- if (countryCode != null && !countryNames.containsKey(countryCode)) {
+ if ("".equals(countryCode) && !countryNames.containsKey(countryCode)) {
LOGGER.warn("Unknown country code {}. Skipping.", countryCode);
return;
}
@@ -259,7 +259,7 @@ public void readCountry(String countryCode, ImportThread importThread) {
}
};
- if (countryCode == null) {
+ if ("".equals(countryCode)) {
template.query(SELECT_COLS_PLACEX + " FROM placex " +
" WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code is null" +
" ORDER BY geometry_sector, parent_place_id; ", placeMapper);
@@ -336,7 +336,7 @@ public void prepareDatabase() {
public String[] getCountriesFromDatabase() {
loadCountryNames();
String[] countries = new String[countryNames.keySet().size() + 1];
- countries[0] = null;
+ countries[0] = "";
int i = 1;
for (var country: countryNames.keySet()) {
From ccb9245654ef3e759f1d9aa3ae133c51aec6cc2c Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 8 Nov 2024 11:33:18 +0100
Subject: [PATCH 06/17] make PlaceRowMapper a full class
---
.../photon/nominatim/NominatimConnector.java | 93 +++++++++----------
.../nominatim/model/PlaceRowMapper.java | 44 +++++++++
2 files changed, 90 insertions(+), 47 deletions(-)
create mode 100644 src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 6aede0da..0a3243f6 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -3,6 +3,7 @@
import de.komoot.photon.PhotonDoc;
import de.komoot.photon.nominatim.model.AddressRow;
import de.komoot.photon.nominatim.model.AddressType;
+import de.komoot.photon.nominatim.model.PlaceRowMapper;
import org.apache.commons.dbcp2.BasicDataSource;
import org.locationtech.jts.geom.Geometry;
import org.slf4j.Logger;
@@ -40,38 +41,7 @@ public class NominatimConnector {
* Maps a placex row in nominatim to a photon doc.
* Some attributes are still missing and can be derived by connected address items.
*/
- private final RowMapper<NominatimResult> placeRowMapper = new RowMapper<>() {
- @Override
- public NominatimResult mapRow(ResultSet rs, int rowNum) throws SQLException {
- Map<String, String> address = dbutils.getMap(rs, "address");
- PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"),
- rs.getString("osm_type"), rs.getLong("osm_id"),
- rs.getString("class"), rs.getString("type"))
- .names(dbutils.getMap(rs, "name"))
- .extraTags(dbutils.getMap(rs, "extratags"))
- .bbox(dbutils.extractGeometry(rs, "bbox"))
- .parentPlaceId(rs.getLong("parent_place_id"))
- .countryCode(rs.getString("country_code"))
- .centroid(dbutils.extractGeometry(rs, "centroid"))
- .linkedPlaceId(rs.getLong("linked_place_id"))
- .rankAddress(rs.getInt("rank_address"))
- .postcode(rs.getString("postcode"));
-
- double importance = rs.getDouble("importance");
- doc.importance(rs.wasNull() ? (0.75 - rs.getInt("rank_search") / 40d) : importance);
-
- completePlace(doc);
- // Add address last, so it takes precedence.
- doc.address(address);
-
- doc.setCountry(countryNames.get(rs.getString("country_code")));
-
- NominatimResult result = new NominatimResult(doc);
- result.addHousenumbersFromAddress(address);
-
- return result;
- }
- };
+ private final RowMapper<NominatimResult> placeToNominatimResult;
/**
* Construct a new importer.
@@ -94,6 +64,25 @@ public NominatimConnector(String host, int port, String database, String usernam
dbutils = dataAdapter;
+ final var placeRowMapper = new PlaceRowMapper(dbutils);
+ placeToNominatimResult = (rs, rowNum) -> {
+ PhotonDoc doc = placeRowMapper.mapRow(rs, rowNum);
+ assert (doc != null);
+
+ Map<String, String> address = dbutils.getMap(rs, "address");
+
+ completePlace(doc);
+ // Add address last, so it takes precedence.
+ doc.address(address);
+
+ doc.setCountry(countryNames.get(rs.getString("country_code")));
+
+ NominatimResult result = new NominatimResult(doc);
+ result.addHousenumbersFromAddress(address);
+
+ return result;
+ };
+
// Setup handling of interpolation table. There are two different formats depending on the Nominatim version.
if (dbutils.hasColumn(template, "location_property_osmline", "step")) {
// new-style interpolations
@@ -158,6 +147,8 @@ static BasicDataSource buildDataSource(String host, int port, String database, S
public void loadCountryNames() {
if (countryNames == null) {
countryNames = new HashMap<>();
+ // Default for places outside any country.
+ countryNames.put("", new HashMap<>());
template.query("SELECT country_code, name FROM country_name", rs -> {
countryNames.put(rs.getString("country_code"), dbutils.getMap(rs, "name"));
});
@@ -166,8 +157,9 @@ public void loadCountryNames() {
public List<PhotonDoc> getByPlaceId(long placeId) {
- List<NominatimResult> result = template.query(SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ? and indexed_status = 0",
- placeRowMapper, placeId);
+ List<NominatimResult> result = template.query(
+ SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ? and indexed_status = 0",
+ placeToNominatimResult, placeId);
return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber();
}
@@ -236,17 +228,31 @@ List<AddressRow> getAddresses(PhotonDoc doc) {
public void readCountry(String countryCode, ImportThread importThread) {
// Make sure, country names are available.
loadCountryNames();
- if ("".equals(countryCode) && !countryNames.containsKey(countryCode)) {
+ final var cnames = countryNames.get(countryCode);
+ if (cnames == null) {
LOGGER.warn("Unknown country code {}. Skipping.", countryCode);
return;
}
+ final PlaceRowMapper placeRowMapper = new PlaceRowMapper(dbutils);
final RowCallbackHandler placeMapper = rs -> {
- NominatimResult docs = placeRowMapper.mapRow(rs, 0);
- assert (docs != null);
+ final PhotonDoc doc = placeRowMapper.mapRow(rs, 0);
+ assert (doc != null);
- if (docs.isUsefulForIndex()) {
- importThread.addDocument(docs);
+ final Map<String, String> address = dbutils.getMap(rs, "address");
+
+
+ completePlace(doc);
+ // Add address last, so it takes precedence.
+ doc.address(address);
+
+ doc.setCountry(cnames);
+
+ NominatimResult result = new NominatimResult(doc);
+ result.addHousenumbersFromAddress(address);
+
+ if (result.isUsefulForIndex()) {
+ importThread.addDocument(result);
}
};
@@ -335,14 +341,7 @@ public void prepareDatabase() {
public String[] getCountriesFromDatabase() {
loadCountryNames();
- String[] countries = new String[countryNames.keySet().size() + 1];
- countries[0] = "";
-
- int i = 1;
- for (var country: countryNames.keySet()) {
- countries[i++] = country;
- }
- return countries;
+ return countryNames.keySet().toArray(new String[0]);
}
}
diff --git a/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java b/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java
new file mode 100644
index 00000000..4944f00a
--- /dev/null
+++ b/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java
@@ -0,0 +1,44 @@
+package de.komoot.photon.nominatim.model;
+
+import de.komoot.photon.PhotonDoc;
+import de.komoot.photon.nominatim.DBDataAdapter;
+import org.springframework.jdbc.core.RowMapper;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Maps the basic attributes of a placex table row to a PhotonDoc.
+ *
+ * This class does not complete address information (neither country information)
+ * for the place.
+ */
+public class PlaceRowMapper implements RowMapper<PhotonDoc> {
+
+ private final DBDataAdapter dbutils;
+
+ public PlaceRowMapper(DBDataAdapter dbutils) {
+ this.dbutils = dbutils;
+ }
+
+ @Override
+ public PhotonDoc mapRow(ResultSet rs, int rowNum) throws SQLException {
+ PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"),
+ rs.getString("osm_type"), rs.getLong("osm_id"),
+ rs.getString("class"), rs.getString("type"))
+ .names(dbutils.getMap(rs, "name"))
+ .extraTags(dbutils.getMap(rs, "extratags"))
+ .bbox(dbutils.extractGeometry(rs, "bbox"))
+ .parentPlaceId(rs.getLong("parent_place_id"))
+ .countryCode(rs.getString("country_code"))
+ .centroid(dbutils.extractGeometry(rs, "centroid"))
+ .linkedPlaceId(rs.getLong("linked_place_id"))
+ .rankAddress(rs.getInt("rank_address"))
+ .postcode(rs.getString("postcode"));
+
+ double importance = rs.getDouble("importance");
+ doc.importance(rs.wasNull() ? (0.75 - rs.getInt("rank_search") / 40d) : importance);
+
+ return doc;
+ }
+}
From a24c2c3c3d832eca9c5fa3c416c4f4586d5c1ea2 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 8 Nov 2024 14:10:00 +0100
Subject: [PATCH 07/17] initialise NominatimResult through static functions
---
.../komoot/photon/nominatim/ImportThread.java | 2 +-
.../photon/nominatim/NominatimConnector.java | 54 ++++----
.../photon/nominatim/NominatimResult.java | 116 +++++++++---------
.../nominatim/model/OsmlineRowMapper.java | 20 +++
.../photon/nominatim/NominatimResultTest.java | 68 +++++-----
5 files changed, 136 insertions(+), 124 deletions(-)
create mode 100644 src/main/java/de/komoot/photon/nominatim/model/OsmlineRowMapper.java
diff --git a/src/main/java/de/komoot/photon/nominatim/ImportThread.java b/src/main/java/de/komoot/photon/nominatim/ImportThread.java
index 0344f488..7d9b8dd4 100644
--- a/src/main/java/de/komoot/photon/nominatim/ImportThread.java
+++ b/src/main/java/de/komoot/photon/nominatim/ImportThread.java
@@ -15,7 +15,7 @@ public class ImportThread {
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(ImportThread.class);
private static final int PROGRESS_INTERVAL = 50000;
- private static final NominatimResult FINAL_DOCUMENT = new NominatimResult(new PhotonDoc(0, null, 0, null, null));
+ private static final NominatimResult FINAL_DOCUMENT = NominatimResult.fromAddress(new PhotonDoc(0, null, 0, null, null), null);
private final BlockingQueue<NominatimResult> documents = new LinkedBlockingDeque<>(100);
private final AtomicLong counter = new AtomicLong();
private final Importer importer;
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 0a3243f6..427d2e06 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -23,6 +23,8 @@ public class NominatimConnector {
private static final String SELECT_COLS_PLACEX = "SELECT place_id, osm_type, osm_id, class, type, name, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_address, rank_search, importance, country_code, centroid";
private static final String SELECT_COLS_ADDRESS = "SELECT p.name, p.class, p.type, p.rank_address";
+ private static final String SELECT_OSMLINE_OLD_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
+ private static final String SELECT_OSMLINE_NEW_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo";
private final DBDataAdapter dbutils;
private final JdbcTemplate template;
@@ -33,8 +35,8 @@ public class NominatimConnector {
* This may be old-style interpolation (using interpolationtype) or
* new-style interpolation (using step).
*/
- private final RowMapper<NominatimResult> osmlineRowMapper;
- private final String selectOsmlineSql;
+ private final RowMapper<NominatimResult> osmlineToNominatimResult;
+ private final boolean hasNewStyleInterpolation;
/**
@@ -77,17 +79,14 @@ public NominatimConnector(String host, int port, String database, String usernam
doc.setCountry(countryNames.get(rs.getString("country_code")));
- NominatimResult result = new NominatimResult(doc);
- result.addHousenumbersFromAddress(address);
-
- return result;
+ return NominatimResult.fromAddress(doc, address);
};
+ hasNewStyleInterpolation = dbutils.hasColumn(template, "location_property_osmline", "step");
// Setup handling of interpolation table. There are two different formats depending on the Nominatim version.
- if (dbutils.hasColumn(template, "location_property_osmline", "step")) {
+ if (hasNewStyleInterpolation) {
// new-style interpolations
- selectOsmlineSql = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo";
- osmlineRowMapper = (rs, rownum) -> {
+ osmlineToNominatimResult = (rs, rownum) -> {
Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"),
@@ -100,16 +99,13 @@ public NominatimConnector(String host, int port, String database, String usernam
doc.setCountry(countryNames.get(rs.getString("country_code")));
- NominatimResult result = new NominatimResult(doc);
- result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"),
+ return NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
rs.getLong("step"), geometry);
-
- return result;
};
} else {
// old-style interpolations
- selectOsmlineSql = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
- osmlineRowMapper = (rs, rownum) -> {
+ osmlineToNominatimResult = (rs, rownum) -> {
Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"),
@@ -122,11 +118,9 @@ public NominatimConnector(String host, int port, String database, String usernam
doc.setCountry(countryNames.get(rs.getString("country_code")));
- NominatimResult result = new NominatimResult(doc);
- result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"),
+ return NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
rs.getString("interpolationtype"), geometry);
-
- return result;
};
}
}
@@ -165,9 +159,10 @@ public List<PhotonDoc> getByPlaceId(long placeId) {
}
public List<PhotonDoc> getInterpolationsByPlaceId(long placeId) {
- List<NominatimResult> result = template.query(selectOsmlineSql
- + " FROM location_property_osmline WHERE place_id = ? and indexed_status = 0",
- osmlineRowMapper, placeId);
+ List<NominatimResult> result = template.query(
+ (hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE)
+ + " FROM location_property_osmline WHERE place_id = ? and indexed_status = 0",
+ osmlineToNominatimResult, placeId);
return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber();
}
@@ -248,8 +243,7 @@ public void readCountry(String countryCode, ImportThread importThread) {
doc.setCountry(cnames);
- NominatimResult result = new NominatimResult(doc);
- result.addHousenumbersFromAddress(address);
+ var result = NominatimResult.fromAddress(doc, address);
if (result.isUsefulForIndex()) {
importThread.addDocument(result);
@@ -257,7 +251,7 @@ public void readCountry(String countryCode, ImportThread importThread) {
};
final RowCallbackHandler osmlineMapper = rs -> {
- NominatimResult docs = osmlineRowMapper.mapRow(rs, 0);
+ NominatimResult docs = osmlineToNominatimResult.mapRow(rs, 0);
assert (docs != null);
if (docs.isUsefulForIndex()) {
@@ -270,16 +264,18 @@ public void readCountry(String countryCode, ImportThread importThread) {
" WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code is null" +
" ORDER BY geometry_sector, parent_place_id; ", placeMapper);
- template.query(selectOsmlineSql + " FROM location_property_osmline " +
- "WHERE startnumber is not null AND country_code is null " +
+ template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
+ " FROM location_property_osmline" +
+ " WHERE startnumber is not null AND country_code is null" +
" ORDER BY geometry_sector, parent_place_id; ", osmlineMapper);
} else {
template.query(SELECT_COLS_PLACEX + " FROM placex " +
" WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code = ?" +
" ORDER BY geometry_sector, parent_place_id; ", placeMapper, countryCode);
- template.query(selectOsmlineSql + " FROM location_property_osmline " +
- "WHERE startnumber is not null AND country_code = ?" +
+ template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
+ " FROM location_property_osmline" +
+ " WHERE startnumber is not null AND country_code = ?" +
" ORDER BY geometry_sector, parent_place_id; ", osmlineMapper, countryCode);
}
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimResult.java b/src/main/java/de/komoot/photon/nominatim/NominatimResult.java
index 75a13cee..9f120408 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimResult.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimResult.java
@@ -1,10 +1,11 @@
package de.komoot.photon.nominatim;
+import de.komoot.photon.PhotonDoc;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryFactory;
import org.locationtech.jts.geom.Point;
import org.locationtech.jts.linearref.LengthIndexedLine;
-import de.komoot.photon.PhotonDoc;
+import org.slf4j.Logger;
import java.util.*;
import java.util.regex.Pattern;
@@ -20,7 +21,7 @@ class NominatimResult {
private static final Pattern HOUSENUMBER_CHECK = Pattern.compile("(\\A|.*,)[^\\d,]{3,}(,.*|\\Z)");
private static final Pattern HOUSENUMBER_SPLIT = Pattern.compile("\\s*[;,]\\s*");
- public NominatimResult(PhotonDoc baseobj) {
+ private NominatimResult(PhotonDoc baseobj) {
doc = baseobj;
housenumbers = null;
}
@@ -58,7 +59,7 @@ List<PhotonDoc> getDocsWithHousenumber() {
*
* @param str House number string. May be null, in which case nothing is added.
*/
- public void addHousenumbersFromString(String str) {
+ private void addHousenumbersFromString(String str) {
if (str == null || str.isEmpty())
return;
@@ -68,9 +69,6 @@ public void addHousenumbersFromString(String str) {
return;
}
- if (housenumbers == null)
- housenumbers = new HashMap<>();
-
String[] parts = HOUSENUMBER_SPLIT.split(str);
for (String part : parts) {
String h = part.trim();
@@ -79,14 +77,17 @@ public void addHousenumbersFromString(String str) {
}
}
- public void addHousenumbersFromAddress(Map<String, String> address) {
- if (address == null) {
- return;
+ public static NominatimResult fromAddress(PhotonDoc doc, Map<String, String> address) {
+ NominatimResult result = new NominatimResult(doc);
+
+ if (address != null) {
+ result.housenumbers = new HashMap<>();
+ result.addHousenumbersFromString(address.get("housenumber"));
+ result.addHousenumbersFromString(address.get("streetnumber"));
+ result.addHousenumbersFromString(address.get("conscriptionnumber"));
}
- addHousenumbersFromString(address.get("housenumber"));
- addHousenumbersFromString(address.get("streetnumber"));
- addHousenumbersFromString(address.get("conscriptionnumber"));
+ return result;
}
/**
@@ -101,35 +102,36 @@ public void addHousenumbersFromAddress(Map<String, String> address) {
* @param interpoltype Kind of interpolation (odd, even or all).
* @param geom Geometry of the interpolation line.
*/
- public void addHouseNumbersFromInterpolation(long first, long last, String interpoltype, Geometry geom) {
- if (last <= first || (last - first) > 1000)
- return;
+ public static NominatimResult fromInterpolation(PhotonDoc doc, long first, long last, String interpoltype, Geometry geom) {
+ NominatimResult result = new NominatimResult(doc);
+ if (last > first && (last - first) < 1000) {
+ result.housenumbers = new HashMap<>();
- if (housenumbers == null)
- housenumbers = new HashMap<>();
-
- LengthIndexedLine line = new LengthIndexedLine(geom);
- double si = line.getStartIndex();
- double ei = line.getEndIndex();
- double lstep = (ei - si) / (last - first);
-
- // leave out first and last, they have a distinct OSM node that is already indexed
- long step = 2;
- long num = 1;
- if (interpoltype.equals("odd")) {
- if (first % 2 == 1)
- ++num;
- } else if (interpoltype.equals("even")) {
- if (first % 2 == 0)
- ++num;
- } else {
- step = 1;
- }
+ LengthIndexedLine line = new LengthIndexedLine(geom);
+ double si = line.getStartIndex();
+ double ei = line.getEndIndex();
+ double lstep = (ei - si) / (last - first);
- GeometryFactory fac = geom.getFactory();
- for (; first + num < last; num += step) {
- housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num)));
+ // leave out first and last, they have a distinct OSM node that is already indexed
+ long step = 2;
+ long num = 1;
+ if (interpoltype.equals("odd")) {
+ if (first % 2 == 1)
+ ++num;
+ } else if (interpoltype.equals("even")) {
+ if (first % 2 == 0)
+ ++num;
+ } else {
+ step = 1;
+ }
+
+ GeometryFactory fac = geom.getFactory();
+ for (; first + num < last; num += step) {
+ result.housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num)));
+ }
}
+
+ return result;
}
/**
@@ -143,25 +145,27 @@ public void addHouseNumbersFromInterpolation(long first, long last, String inter
* @param step Gap to leave between each interpolated house number.
* @param geom Geometry of the interpolation line.
*/
- public void addHouseNumbersFromInterpolation(long first, long last, long step, Geometry geom) {
- if (last < first || (last - first) > 1000)
- return;
-
- if (housenumbers == null)
- housenumbers = new HashMap<>();
-
- if (last == first) {
- housenumbers.put(String.valueOf(first), geom.getCentroid());
- } else {
- LengthIndexedLine line = new LengthIndexedLine(geom);
- double si = line.getStartIndex();
- double ei = line.getEndIndex();
- double lstep = (ei - si) / (last - first);
-
- GeometryFactory fac = geom.getFactory();
- for (long num = 0; first + num <= last; num += step) {
- housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num)));
+ public static NominatimResult fromInterpolation(PhotonDoc doc, long first, long last, long step, Geometry geom) {
+ NominatimResult result = new NominatimResult(doc);
+ if (last >= first && (last - first) < 1000) {
+ result.housenumbers = new HashMap<>();
+
+ if (last == first) {
+ result.housenumbers.put(String.valueOf(first), geom.getCentroid());
+ } else {
+ LengthIndexedLine line = new LengthIndexedLine(geom);
+ double si = line.getStartIndex();
+ double ei = line.getEndIndex();
+ double lstep = (ei - si) / (last - first);
+
+ GeometryFactory fac = geom.getFactory();
+ for (long num = 0; first + num <= last; num += step) {
+ result.housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num)));
+ }
}
+
}
+
+ return result;
}
}
diff --git a/src/main/java/de/komoot/photon/nominatim/model/OsmlineRowMapper.java b/src/main/java/de/komoot/photon/nominatim/model/OsmlineRowMapper.java
new file mode 100644
index 00000000..5c6a2c6a
--- /dev/null
+++ b/src/main/java/de/komoot/photon/nominatim/model/OsmlineRowMapper.java
@@ -0,0 +1,20 @@
+package de.komoot.photon.nominatim.model;
+
+import de.komoot.photon.PhotonDoc;
+import org.springframework.jdbc.core.RowMapper;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class OsmlineRowMapper implements RowMapper<PhotonDoc> {
+ @Override
+ public PhotonDoc mapRow(ResultSet rs, int rowNum) throws SQLException {
+ return new PhotonDoc(
+ rs.getLong("place_id"),
+ "W", rs.getLong("osm_id"),
+ "place", "house_number")
+ .parentPlaceId(rs.getLong("parent_place_id"))
+ .countryCode(rs.getString("country_code"))
+ .postcode(rs.getString("postcode"));
+ }
+}
\ No newline at end of file
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimResultTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimResultTest.java
index 3c3eebc4..37de39f7 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimResultTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimResultTest.java
@@ -47,35 +47,39 @@ private void assertSimpleOnly(List<PhotonDoc> docs) {
assertSame(simpleDoc, docs.get(0));
}
+ private Map<String, String> housenumberAddress(String housenumber) {
+ Map<String, String> address = new HashMap<>(1);
+ address.put("housenumber", housenumber);
+ return address;
+ }
+
@Test
void testIsUsefulForIndex() {
assertFalse(simpleDoc.isUsefulForIndex());
- assertFalse(new NominatimResult(simpleDoc).isUsefulForIndex());
+ assertFalse(NominatimResult.fromAddress(simpleDoc, null).isUsefulForIndex());
}
@Test
void testGetDocsWithHousenumber() {
- List<PhotonDoc> docs = new NominatimResult(simpleDoc).getDocsWithHousenumber();
+ List<PhotonDoc> docs = NominatimResult.fromAddress(simpleDoc, null).getDocsWithHousenumber();
assertSimpleOnly(docs);
}
@Test
void testAddHousenumbersFromStringSimple() {
- NominatimResult res = new NominatimResult(simpleDoc);
- res.addHousenumbersFromString("34");
+ NominatimResult res = NominatimResult.fromAddress(simpleDoc, housenumberAddress("34"));
assertDocWithHousenumbers(Arrays.asList("34"), res.getDocsWithHousenumber());
}
@Test
void testAddHousenumbersFromStringList() {
- NominatimResult res = new NominatimResult(simpleDoc);
- res.addHousenumbersFromString("34; 50b");
+ NominatimResult res = NominatimResult.fromAddress(simpleDoc, housenumberAddress("34; 50b"));
assertDocWithHousenumbers(Arrays.asList("34", "50b"), res.getDocsWithHousenumber());
- res.addHousenumbersFromString("4;");
- assertDocWithHousenumbers(Arrays.asList("34", "50b", "4"), res.getDocsWithHousenumber());
+ res = NominatimResult.fromAddress(simpleDoc, housenumberAddress("4;"));
+ assertDocWithHousenumbers(Arrays.asList("4"), res.getDocsWithHousenumber());
}
@ParameterizedTest
@@ -85,81 +89,69 @@ void testAddHousenumbersFromStringList() {
"14, portsmith"
})
void testLongHousenumber(String houseNumber) {
- NominatimResult res = new NominatimResult(simpleDoc);
+ NominatimResult res = NominatimResult.fromAddress(simpleDoc, housenumberAddress(houseNumber));
- res.addHousenumbersFromString(houseNumber);
assertNoHousenumber(res.getDocsWithHousenumber());
}
@Test
void testAddHouseNumbersFromInterpolationBad() throws ParseException {
- NominatimResult res = new NominatimResult(simpleDoc);
-
WKTReader reader = new WKTReader();
- res.addHouseNumbersFromInterpolation(34, 33, "odd",
+ NominatimResult res = NominatimResult.fromInterpolation(simpleDoc, 34, 33, "odd",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
assertSimpleOnly(res.getDocsWithHousenumber());
- res.addHouseNumbersFromInterpolation(1, 10000, "odd",
+ res = NominatimResult.fromInterpolation(simpleDoc, 1, 10000, "odd",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
assertSimpleOnly(res.getDocsWithHousenumber());
}
@Test
void testAddHouseNumbersFromInterpolationOdd() throws ParseException {
- NominatimResult res = new NominatimResult(simpleDoc);
-
WKTReader reader = new WKTReader();
-
- res.addHouseNumbersFromInterpolation(1, 5, "odd",
+ NominatimResult res = NominatimResult.fromInterpolation(simpleDoc, 1, 5, "odd",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
assertDocWithHousenumbers(Arrays.asList("3"), res.getDocsWithHousenumber());
- res.addHouseNumbersFromInterpolation(10, 13, "odd",
+ res = NominatimResult.fromInterpolation(simpleDoc, 10, 13, "odd",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
- assertDocWithHousenumbers(Arrays.asList("3", "11"), res.getDocsWithHousenumber());
+ assertDocWithHousenumbers(Arrays.asList("11"), res.getDocsWithHousenumber());
- res.addHouseNumbersFromInterpolation(101, 106, "odd",
+ res = NominatimResult.fromInterpolation(simpleDoc, 101, 106, "odd",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
- assertDocWithHousenumbers(Arrays.asList("3", "11", "103", "105"), res.getDocsWithHousenumber());
+ assertDocWithHousenumbers(Arrays.asList("103", "105"), res.getDocsWithHousenumber());
}
@Test
void testAddHouseNumbersFromInterpolationEven() throws ParseException {
- NominatimResult res = new NominatimResult(simpleDoc);
-
WKTReader reader = new WKTReader();
-
- res.addHouseNumbersFromInterpolation(1, 5, "even",
+ NominatimResult res = NominatimResult.fromInterpolation(simpleDoc, 1, 5, "even",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
assertDocWithHousenumbers(Arrays.asList("2", "4"), res.getDocsWithHousenumber());
- res.addHouseNumbersFromInterpolation(10, 16, "even",
+ res= NominatimResult.fromInterpolation(simpleDoc, 10, 16, "even",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
- assertDocWithHousenumbers(Arrays.asList("2", "4", "12", "14"), res.getDocsWithHousenumber());
+ assertDocWithHousenumbers(Arrays.asList("12", "14"), res.getDocsWithHousenumber());
- res.addHouseNumbersFromInterpolation(51, 52, "even",
+ res= NominatimResult.fromInterpolation(simpleDoc, 51, 52, "even",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
- assertDocWithHousenumbers(Arrays.asList("2", "4", "12", "14"), res.getDocsWithHousenumber());
+ assertSimpleOnly(res.getDocsWithHousenumber());
}
@Test
void testAddHouseNumbersFromInterpolationAll() throws ParseException {
- NominatimResult res = new NominatimResult(simpleDoc);
-
WKTReader reader = new WKTReader();
-
- res.addHouseNumbersFromInterpolation(1, 3, "",
+ NominatimResult res = NominatimResult.fromInterpolation(simpleDoc, 1, 3, "",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
assertDocWithHousenumbers(Arrays.asList("2"), res.getDocsWithHousenumber());
- res.addHouseNumbersFromInterpolation(22, 22, null,
+ res = NominatimResult.fromInterpolation(simpleDoc, 22, 22, null,
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
- assertDocWithHousenumbers(Arrays.asList("2"), res.getDocsWithHousenumber());
+ assertSimpleOnly(res.getDocsWithHousenumber());
- res.addHouseNumbersFromInterpolation(100, 106, "all",
+ res = NominatimResult.fromInterpolation(simpleDoc, 100, 106, "all",
reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)"));
- assertDocWithHousenumbers(Arrays.asList("2", "101", "102", "103", "104", "105"), res.getDocsWithHousenumber());
+ assertDocWithHousenumbers(Arrays.asList("101", "102", "103", "104", "105"), res.getDocsWithHousenumber());
}
}
\ No newline at end of file
From 30449fea96748e1e69d4370845ddcfd64abbcff1 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 8 Nov 2024 14:39:05 +0100
Subject: [PATCH 08/17] make OsmlineRowMapper a full class
---
.../photon/nominatim/NominatimConnector.java | 67 +++++++++----------
.../photon/nominatim/NominatimResult.java | 1 -
2 files changed, 32 insertions(+), 36 deletions(-)
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 427d2e06..90c3f6b7 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -3,6 +3,7 @@
import de.komoot.photon.PhotonDoc;
import de.komoot.photon.nominatim.model.AddressRow;
import de.komoot.photon.nominatim.model.AddressType;
+import de.komoot.photon.nominatim.model.OsmlineRowMapper;
import de.komoot.photon.nominatim.model.PlaceRowMapper;
import org.apache.commons.dbcp2.BasicDataSource;
import org.locationtech.jts.geom.Geometry;
@@ -82,47 +83,28 @@ public NominatimConnector(String host, int port, String database, String usernam
return NominatimResult.fromAddress(doc, address);
};
- hasNewStyleInterpolation = dbutils.hasColumn(template, "location_property_osmline", "step");
// Setup handling of interpolation table. There are two different formats depending on the Nominatim version.
- if (hasNewStyleInterpolation) {
- // new-style interpolations
- osmlineToNominatimResult = (rs, rownum) -> {
- Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
-
- PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"),
- "place", "house_number")
- .parentPlaceId(rs.getLong("parent_place_id"))
- .countryCode(rs.getString("country_code"))
- .postcode(rs.getString("postcode"));
+ // new-style interpolations
+ hasNewStyleInterpolation = dbutils.hasColumn(template, "location_property_osmline", "step");
+ final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper();
+ osmlineToNominatimResult = (rs, rownum) -> {
+ PhotonDoc doc = osmlineRowMapper.mapRow(rs, rownum);
- completePlace(doc);
+ completePlace(doc);
+ doc.setCountry(countryNames.get(rs.getString("country_code")));
- doc.setCountry(countryNames.get(rs.getString("country_code")));
+ Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
+ if (hasNewStyleInterpolation) {
return NominatimResult.fromInterpolation(
doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
rs.getLong("step"), geometry);
- };
- } else {
- // old-style interpolations
- osmlineToNominatimResult = (rs, rownum) -> {
- Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
-
- PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"),
- "place", "house_number")
- .parentPlaceId(rs.getLong("parent_place_id"))
- .countryCode(rs.getString("country_code"))
- .postcode(rs.getString("postcode"));
-
- completePlace(doc);
-
- doc.setCountry(countryNames.get(rs.getString("country_code")));
+ }
- return NominatimResult.fromInterpolation(
- doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
- rs.getString("interpolationtype"), geometry);
- };
- }
+ return NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
+ rs.getString("interpolationtype"), geometry);
+ };
}
@@ -250,9 +232,24 @@ public void readCountry(String countryCode, ImportThread importThread) {
}
};
+ final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper();
final RowCallbackHandler osmlineMapper = rs -> {
- NominatimResult docs = osmlineToNominatimResult.mapRow(rs, 0);
- assert (docs != null);
+ final PhotonDoc doc = osmlineRowMapper.mapRow(rs, 0);
+
+ completePlace(doc);
+ doc.setCountry(cnames);
+
+ final Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
+ final NominatimResult docs;
+ if (hasNewStyleInterpolation) {
+ docs = NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
+ rs.getLong("step"), geometry);
+ } else {
+ docs = NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
+ rs.getString("interpolationtype"), geometry);
+ }
if (docs.isUsefulForIndex()) {
importThread.addDocument(docs);
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimResult.java b/src/main/java/de/komoot/photon/nominatim/NominatimResult.java
index 9f120408..938c6d37 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimResult.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimResult.java
@@ -5,7 +5,6 @@
import org.locationtech.jts.geom.GeometryFactory;
import org.locationtech.jts.geom.Point;
import org.locationtech.jts.linearref.LengthIndexedLine;
-import org.slf4j.Logger;
import java.util.*;
import java.util.regex.Pattern;
From 8f692a264270c1dfe518480f9d0ff61ecb5e4df7 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 8 Nov 2024 15:16:23 +0100
Subject: [PATCH 09/17] decouple NominatimConnector and NominatimUpdater class
The actual importer is now named NominatimImporter, while
NominatimConnector becomes the class for the common code shared
between the two classes.
---
src/main/java/de/komoot/photon/App.java | 10 +-
.../photon/nominatim/NominatimConnector.java | 325 ++----------------
.../photon/nominatim/NominatimImporter.java | 206 +++++++++++
.../photon/nominatim/NominatimUpdater.java | 184 ++++++++--
.../nominatim/NominatimConnectorDBTest.java | 6 +-
.../nominatim/NominatimUpdaterDBTest.java | 3 +-
6 files changed, 395 insertions(+), 339 deletions(-)
create mode 100644 src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java
index c30d5db6..3f57c9c1 100644
--- a/src/main/java/de/komoot/photon/App.java
+++ b/src/main/java/de/komoot/photon/App.java
@@ -3,7 +3,7 @@
import com.beust.jcommander.JCommander;
import com.beust.jcommander.ParameterException;
import de.komoot.photon.nominatim.ImportThread;
-import de.komoot.photon.nominatim.NominatimConnector;
+import de.komoot.photon.nominatim.NominatimImporter;
import de.komoot.photon.nominatim.NominatimUpdater;
import de.komoot.photon.searcher.ReverseHandler;
import de.komoot.photon.searcher.SearchHandler;
@@ -131,7 +131,7 @@ private static void startNominatimImport(CommandLineArgs args, Server esServer)
}
private static String[] initDatabase(CommandLineArgs args, Server esServer) {
- final var nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
+ final var nominatimConnector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
final Date importDate = nominatimConnector.getLastImportDate();
try {
@@ -144,7 +144,7 @@ private static String[] initDatabase(CommandLineArgs args, Server esServer) {
}
private static void importFromDatabase(CommandLineArgs args, Importer importer) {
- final var connector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
+ final var connector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
connector.prepareDatabase();
connector.loadCountryNames();
@@ -171,9 +171,9 @@ private static void importFromDatabase(CommandLineArgs args, Importer importer)
final List<Thread> readerThreads = new ArrayList<>(numThreads);
for (int i = 0; i < numThreads; ++i) {
- final NominatimConnector threadConnector;
+ final NominatimImporter threadConnector;
if (i > 0) {
- threadConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
+ threadConnector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
threadConnector.loadCountryNames();
} else {
threadConnector = connector;
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 90c3f6b7..9c1b89af 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -1,114 +1,31 @@
package de.komoot.photon.nominatim;
-import de.komoot.photon.PhotonDoc;
-import de.komoot.photon.nominatim.model.AddressRow;
-import de.komoot.photon.nominatim.model.AddressType;
-import de.komoot.photon.nominatim.model.OsmlineRowMapper;
-import de.komoot.photon.nominatim.model.PlaceRowMapper;
import org.apache.commons.dbcp2.BasicDataSource;
-import org.locationtech.jts.geom.Geometry;
-import org.slf4j.Logger;
import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.RowMapper;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.*;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
- * Importer for data from a Nominatim database.
+ * Base class for workers connecting to a Nominatim database
*/
public class NominatimConnector {
- private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimConnector.class);
+ protected static final String SELECT_COLS_PLACEX = "SELECT place_id, osm_type, osm_id, class, type, name, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_address, rank_search, importance, country_code, centroid";
+ protected static final String SELECT_COLS_ADDRESS = "SELECT p.name, p.class, p.type, p.rank_address";
+ protected static final String SELECT_OSMLINE_OLD_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
+ protected static final String SELECT_OSMLINE_NEW_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo";
- private static final String SELECT_COLS_PLACEX = "SELECT place_id, osm_type, osm_id, class, type, name, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_address, rank_search, importance, country_code, centroid";
- private static final String SELECT_COLS_ADDRESS = "SELECT p.name, p.class, p.type, p.rank_address";
- private static final String SELECT_OSMLINE_OLD_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
- private static final String SELECT_OSMLINE_NEW_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo";
+ protected final DBDataAdapter dbutils;
+ protected final JdbcTemplate template;
+ protected Map<String, Map<String, String>> countryNames;
+ protected final boolean hasNewStyleInterpolation;
- private final DBDataAdapter dbutils;
- private final JdbcTemplate template;
- private Map<String, Map<String, String>> countryNames;
-
- /**
- * Map a row from location_property_osmline (address interpolation lines) to a photon doc.
- * This may be old-style interpolation (using interpolationtype) or
- * new-style interpolation (using step).
- */
- private final RowMapper<NominatimResult> osmlineToNominatimResult;
- private final boolean hasNewStyleInterpolation;
-
-
- /**
- * Maps a placex row in nominatim to a photon doc.
- * Some attributes are still missing and can be derived by connected address items.
- */
- private final RowMapper<NominatimResult> placeToNominatimResult;
-
- /**
- * Construct a new importer.
- *
- * @param host database host
- * @param port database port
- * @param database database name
- * @param username db username
- * @param password db username's password
- */
- public NominatimConnector(String host, int port, String database, String username, String password) {
- this(host, port, database, username, password, new PostgisDataAdapter());
- }
-
- public NominatimConnector(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) {
- BasicDataSource dataSource = buildDataSource(host, port, database, username, password, true);
-
- template = new JdbcTemplate(dataSource);
- template.setFetchSize(100000);
-
- dbutils = dataAdapter;
-
- final var placeRowMapper = new PlaceRowMapper(dbutils);
- placeToNominatimResult = (rs, rowNum) -> {
- PhotonDoc doc = placeRowMapper.mapRow(rs, rowNum);
- assert (doc != null);
-
- Map<String, String> address = dbutils.getMap(rs, "address");
-
- completePlace(doc);
- // Add address last, so it takes precedence.
- doc.address(address);
-
- doc.setCountry(countryNames.get(rs.getString("country_code")));
-
- return NominatimResult.fromAddress(doc, address);
- };
-
- // Setup handling of interpolation table. There are two different formats depending on the Nominatim version.
- // new-style interpolations
- hasNewStyleInterpolation = dbutils.hasColumn(template, "location_property_osmline", "step");
- final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper();
- osmlineToNominatimResult = (rs, rownum) -> {
- PhotonDoc doc = osmlineRowMapper.mapRow(rs, rownum);
-
- completePlace(doc);
- doc.setCountry(countryNames.get(rs.getString("country_code")));
-
- Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
-
- if (hasNewStyleInterpolation) {
- return NominatimResult.fromInterpolation(
- doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
- rs.getLong("step"), geometry);
- }
-
- return NominatimResult.fromInterpolation(
- doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
- rs.getString("interpolationtype"), geometry);
- };
- }
-
-
- static BasicDataSource buildDataSource(String host, int port, String database, String username, String password, boolean autocommit) {
+ protected NominatimConnector(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) {
BasicDataSource dataSource = new BasicDataSource();
dataSource.setUrl(String.format("jdbc:postgresql://%s:%d/%s", host, port, database));
@@ -116,166 +33,13 @@ static BasicDataSource buildDataSource(String host, int port, String database, S
if (password != null) {
dataSource.setPassword(password);
}
- dataSource.setDefaultAutoCommit(autocommit);
- return dataSource;
- }
-
- public void loadCountryNames() {
- if (countryNames == null) {
- countryNames = new HashMap<>();
- // Default for places outside any country.
- countryNames.put("", new HashMap<>());
- template.query("SELECT country_code, name FROM country_name", rs -> {
- countryNames.put(rs.getString("country_code"), dbutils.getMap(rs, "name"));
- });
- }
- }
-
-
- public List<PhotonDoc> getByPlaceId(long placeId) {
- List<NominatimResult> result = template.query(
- SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ? and indexed_status = 0",
- placeToNominatimResult, placeId);
-
- return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber();
- }
-
- public List<PhotonDoc> getInterpolationsByPlaceId(long placeId) {
- List<NominatimResult> result = template.query(
- (hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE)
- + " FROM location_property_osmline WHERE place_id = ? and indexed_status = 0",
- osmlineToNominatimResult, placeId);
-
- return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber();
- }
-
- private long parentPlaceId = -1;
- private List<AddressRow> parentTerms = null;
-
- List<AddressRow> getAddresses(PhotonDoc doc) {
- RowMapper<AddressRow> rowMapper = (rs, rowNum) -> new AddressRow(
- dbutils.getMap(rs, "name"),
- rs.getString("class"),
- rs.getString("type"),
- rs.getInt("rank_address")
- );
-
- AddressType atype = doc.getAddressType();
-
- if (atype == null || atype == AddressType.COUNTRY) {
- return Collections.emptyList();
- }
-
- List<AddressRow> terms = null;
-
- if (atype == AddressType.HOUSE) {
- long placeId = doc.getParentPlaceId();
- if (placeId != parentPlaceId) {
- parentTerms = template.query(SELECT_COLS_ADDRESS
- + " FROM placex p, place_addressline pa"
- + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?"
- + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress"
- + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc",
- rowMapper, placeId, placeId);
-
- // need to add the term for the parent place ID itself
- parentTerms.addAll(0, template.query(SELECT_COLS_ADDRESS + " FROM placex p WHERE p.place_id = ?",
- rowMapper, placeId));
- parentPlaceId = placeId;
- }
- terms = parentTerms;
-
- } else {
- long placeId = doc.getPlaceId();
- terms = template.query(SELECT_COLS_ADDRESS
- + " FROM placex p, place_addressline pa"
- + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?"
- + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress"
- + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc",
- rowMapper, placeId, placeId);
- }
-
- return terms;
- }
-
- /**
- * Parse every relevant row in placex and location_osmline
- * for the given country. Also imports place from county-less places.
- */
- public void readCountry(String countryCode, ImportThread importThread) {
- // Make sure, country names are available.
- loadCountryNames();
- final var cnames = countryNames.get(countryCode);
- if (cnames == null) {
- LOGGER.warn("Unknown country code {}. Skipping.", countryCode);
- return;
- }
-
- final PlaceRowMapper placeRowMapper = new PlaceRowMapper(dbutils);
- final RowCallbackHandler placeMapper = rs -> {
- final PhotonDoc doc = placeRowMapper.mapRow(rs, 0);
- assert (doc != null);
-
- final Map<String, String> address = dbutils.getMap(rs, "address");
-
-
- completePlace(doc);
- // Add address last, so it takes precedence.
- doc.address(address);
-
- doc.setCountry(cnames);
-
- var result = NominatimResult.fromAddress(doc, address);
-
- if (result.isUsefulForIndex()) {
- importThread.addDocument(result);
- }
- };
-
- final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper();
- final RowCallbackHandler osmlineMapper = rs -> {
- final PhotonDoc doc = osmlineRowMapper.mapRow(rs, 0);
-
- completePlace(doc);
- doc.setCountry(cnames);
-
- final Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
- final NominatimResult docs;
- if (hasNewStyleInterpolation) {
- docs = NominatimResult.fromInterpolation(
- doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
- rs.getLong("step"), geometry);
- } else {
- docs = NominatimResult.fromInterpolation(
- doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
- rs.getString("interpolationtype"), geometry);
- }
-
- if (docs.isUsefulForIndex()) {
- importThread.addDocument(docs);
- }
- };
-
- if ("".equals(countryCode)) {
- template.query(SELECT_COLS_PLACEX + " FROM placex " +
- " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code is null" +
- " ORDER BY geometry_sector, parent_place_id; ", placeMapper);
+ dataSource.setDefaultAutoCommit(true);
- template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
- " FROM location_property_osmline" +
- " WHERE startnumber is not null AND country_code is null" +
- " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper);
- } else {
- template.query(SELECT_COLS_PLACEX + " FROM placex " +
- " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code = ?" +
- " ORDER BY geometry_sector, parent_place_id; ", placeMapper, countryCode);
-
- template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
- " FROM location_property_osmline" +
- " WHERE startnumber is not null AND country_code = ?" +
- " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper, countryCode);
+ template = new JdbcTemplate(dataSource);
+ template.setFetchSize(100000);
- }
+ dbutils = dataAdapter;
+ hasNewStyleInterpolation = dbutils.hasColumn(template, "location_property_osmline", "step");
}
public Date getLastImportDate() {
@@ -291,50 +55,15 @@ public Date mapRow(ResultSet rs, int rowNum) throws SQLException {
return importDates.get(0);
}
- /**
- * Query Nominatim's address hierarchy to complete photon doc with missing data (like country, city, street, ...)
- *
- * @param doc
- */
- private void completePlace(PhotonDoc doc) {
- final List<AddressRow> addresses = getAddresses(doc);
- final AddressType doctype = doc.getAddressType();
- for (AddressRow address : addresses) {
- AddressType atype = address.getAddressType();
-
- if (atype != null
- && (atype == doctype || !doc.setAddressPartIfNew(atype, address.getName()))
- && address.isUsefulForContext()) {
- // no specifically handled item, check if useful for context
- doc.getContext().add(address.getName());
- }
- }
- }
-
- public DBDataAdapter getDataAdaptor() {
- return dbutils;
- }
-
- /**
- * Prepare the database for export.
- *
- * This function ensures that the proper index are available and if
- * not will create them. This may take a while.
- */
- public void prepareDatabase() {
- Integer indexRowNum = template.queryForObject(
- "SELECT count(*) FROM pg_indexes WHERE tablename = 'placex' AND indexdef LIKE '%(country_code)'",
- Integer.class);
-
- if (indexRowNum == null || indexRowNum == 0) {
- LOGGER.info("Creating index over countries.");
- template.execute("CREATE INDEX ON placex (country_code)");
+ public void loadCountryNames() {
+ if (countryNames == null) {
+ countryNames = new HashMap<>();
+ // Default for places outside any country.
+ countryNames.put("", new HashMap<>());
+ template.query("SELECT country_code, name FROM country_name", rs -> {
+ countryNames.put(rs.getString("country_code"), dbutils.getMap(rs, "name"));
+ });
}
}
- public String[] getCountriesFromDatabase() {
- loadCountryNames();
-
- return countryNames.keySet().toArray(new String[0]);
- }
}
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
new file mode 100644
index 00000000..9f4020c7
--- /dev/null
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
@@ -0,0 +1,206 @@
+package de.komoot.photon.nominatim;
+
+import de.komoot.photon.PhotonDoc;
+import de.komoot.photon.nominatim.model.AddressRow;
+import de.komoot.photon.nominatim.model.AddressType;
+import de.komoot.photon.nominatim.model.OsmlineRowMapper;
+import de.komoot.photon.nominatim.model.PlaceRowMapper;
+import org.apache.commons.dbcp2.BasicDataSource;
+import org.locationtech.jts.geom.Geometry;
+import org.slf4j.Logger;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.RowCallbackHandler;
+import org.springframework.jdbc.core.RowMapper;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.*;
+
+/**
+ * Importer for data from a Nominatim database.
+ */
+public class NominatimImporter extends NominatimConnector {
+ private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimImporter.class);
+
+ // One-item cache for address lookup. Speeds up rank 30 processing.
+ private long parentPlaceId = -1;
+ private List<AddressRow> parentTerms = null;
+
+ public NominatimImporter(String host, int port, String database, String username, String password) {
+ this(host, port, database, username, password, new PostgisDataAdapter());
+ }
+
+ public NominatimImporter(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) {
+ super(host, port, database, username, password, dataAdapter);
+ }
+
+
+ List<AddressRow> getAddresses(PhotonDoc doc) {
+ RowMapper<AddressRow> rowMapper = (rs, rowNum) -> new AddressRow(
+ dbutils.getMap(rs, "name"),
+ rs.getString("class"),
+ rs.getString("type"),
+ rs.getInt("rank_address")
+ );
+
+ AddressType atype = doc.getAddressType();
+
+ if (atype == null || atype == AddressType.COUNTRY) {
+ return Collections.emptyList();
+ }
+
+ List<AddressRow> terms = null;
+
+ if (atype == AddressType.HOUSE) {
+ long placeId = doc.getParentPlaceId();
+ if (placeId != parentPlaceId) {
+ parentTerms = template.query(SELECT_COLS_ADDRESS
+ + " FROM placex p, place_addressline pa"
+ + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?"
+ + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress"
+ + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc",
+ rowMapper, placeId, placeId);
+
+ // need to add the term for the parent place ID itself
+ parentTerms.addAll(0, template.query(SELECT_COLS_ADDRESS + " FROM placex p WHERE p.place_id = ?",
+ rowMapper, placeId));
+ parentPlaceId = placeId;
+ }
+ terms = parentTerms;
+
+ } else {
+ long placeId = doc.getPlaceId();
+ terms = template.query(SELECT_COLS_ADDRESS
+ + " FROM placex p, place_addressline pa"
+ + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?"
+ + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress"
+ + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc",
+ rowMapper, placeId, placeId);
+ }
+
+ return terms;
+ }
+
+ /**
+ * Parse every relevant row in placex and location_osmline
+ * for the given country. Also imports place from county-less places.
+ */
+ public void readCountry(String countryCode, ImportThread importThread) {
+ // Make sure, country names are available.
+ loadCountryNames();
+ final var cnames = countryNames.get(countryCode);
+ if (cnames == null) {
+ LOGGER.warn("Unknown country code {}. Skipping.", countryCode);
+ return;
+ }
+
+ final PlaceRowMapper placeRowMapper = new PlaceRowMapper(dbutils);
+ final RowCallbackHandler placeMapper = rs -> {
+ final PhotonDoc doc = placeRowMapper.mapRow(rs, 0);
+ assert (doc != null);
+
+ final Map<String, String> address = dbutils.getMap(rs, "address");
+
+
+ completePlace(doc);
+ // Add address last, so it takes precedence.
+ doc.address(address);
+
+ doc.setCountry(cnames);
+
+ var result = NominatimResult.fromAddress(doc, address);
+
+ if (result.isUsefulForIndex()) {
+ importThread.addDocument(result);
+ }
+ };
+
+ final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper();
+ final RowCallbackHandler osmlineMapper = rs -> {
+ final PhotonDoc doc = osmlineRowMapper.mapRow(rs, 0);
+
+ completePlace(doc);
+ doc.setCountry(cnames);
+
+ final Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
+ final NominatimResult docs;
+ if (hasNewStyleInterpolation) {
+ docs = NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
+ rs.getLong("step"), geometry);
+ } else {
+ docs = NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
+ rs.getString("interpolationtype"), geometry);
+ }
+
+ if (docs.isUsefulForIndex()) {
+ importThread.addDocument(docs);
+ }
+ };
+
+ if ("".equals(countryCode)) {
+ template.query(SELECT_COLS_PLACEX + " FROM placex " +
+ " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code is null" +
+ " ORDER BY geometry_sector, parent_place_id; ", placeMapper);
+
+ template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
+ " FROM location_property_osmline" +
+ " WHERE startnumber is not null AND country_code is null" +
+ " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper);
+ } else {
+ template.query(SELECT_COLS_PLACEX + " FROM placex " +
+ " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code = ?" +
+ " ORDER BY geometry_sector, parent_place_id; ", placeMapper, countryCode);
+
+ template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
+ " FROM location_property_osmline" +
+ " WHERE startnumber is not null AND country_code = ?" +
+ " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper, countryCode);
+
+ }
+ }
+
+ /**
+ * Query Nominatim's address hierarchy to complete photon doc with missing data (like country, city, street, ...)
+ *
+ * @param doc
+ */
+ private void completePlace(PhotonDoc doc) {
+ final List<AddressRow> addresses = getAddresses(doc);
+ final AddressType doctype = doc.getAddressType();
+ for (AddressRow address : addresses) {
+ AddressType atype = address.getAddressType();
+
+ if (atype != null
+ && (atype == doctype || !doc.setAddressPartIfNew(atype, address.getName()))
+ && address.isUsefulForContext()) {
+ // no specifically handled item, check if useful for context
+ doc.getContext().add(address.getName());
+ }
+ }
+ }
+
+ /**
+ * Prepare the database for export.
+ *
+ * This function ensures that the proper index are available and if
+ * not will create them. This may take a while.
+ */
+ public void prepareDatabase() {
+ Integer indexRowNum = template.queryForObject(
+ "SELECT count(*) FROM pg_indexes WHERE tablename = 'placex' AND indexdef LIKE '%(country_code)'",
+ Integer.class);
+
+ if (indexRowNum == null || indexRowNum == 0) {
+ LOGGER.info("Creating index over countries.");
+ template.execute("CREATE INDEX ON placex (country_code)");
+ }
+ }
+
+ public String[] getCountriesFromDatabase() {
+ loadCountryNames();
+
+ return countryNames.keySet().toArray(new String[0]);
+ }
+}
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
index 01455b0c..1a0f5033 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
@@ -2,20 +2,17 @@
import de.komoot.photon.PhotonDoc;
import de.komoot.photon.Updater;
-import de.komoot.photon.nominatim.model.UpdateRow;
-import org.apache.commons.dbcp2.BasicDataSource;
-import org.springframework.jdbc.core.JdbcTemplate;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.List;
+import de.komoot.photon.nominatim.model.*;
+import org.locationtech.jts.geom.Geometry;
+import org.springframework.jdbc.core.RowMapper;
+
+import java.util.*;
import java.util.concurrent.locks.ReentrantLock;
/**
* Importer for updates from a Nominatim database.
*/
-public class NominatimUpdater {
+public class NominatimUpdater extends NominatimConnector {
private static final org.slf4j.Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimUpdater.class);
private static final String TRIGGER_SQL =
@@ -45,20 +42,82 @@ public class NominatimUpdater {
+ " AFTER DELETE ON location_property_osmline FOR EACH ROW"
+ " EXECUTE FUNCTION photon_update_func()";
- private final JdbcTemplate template;
- private final NominatimConnector exporter;
-
private Updater updater;
+ /**
+ * Map a row from location_property_osmline (address interpolation lines) to a photon doc.
+ * This may be old-style interpolation (using interpolationtype) or
+ * new-style interpolation (using step).
+ */
+ private final RowMapper<NominatimResult> osmlineToNominatimResult;
+
+
+ /**
+ * Maps a placex row in nominatim to a photon doc.
+ * Some attributes are still missing and can be derived by connected address items.
+ */
+ private final RowMapper<NominatimResult> placeToNominatimResult;
+
+
/**
* Lock to prevent thread from updating concurrently.
*/
private ReentrantLock updateLock = new ReentrantLock();
- public Date getLastImportDate() {
- return exporter.getLastImportDate();
+
+ // One-item cache for address terms. Speeds up processing of rank 30 objects.
+ private long parentPlaceId = -1;
+ private List<AddressRow> parentTerms = null;
+
+
+ public NominatimUpdater(String host, int port, String database, String username, String password) {
+ this(host, port, database, username, password, new PostgisDataAdapter());
}
+ public NominatimUpdater(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) {
+ super(host, port, database, username, password, dataAdapter);
+
+ final var placeRowMapper = new PlaceRowMapper(dbutils);
+ placeToNominatimResult = (rs, rowNum) -> {
+ PhotonDoc doc = placeRowMapper.mapRow(rs, rowNum);
+ assert (doc != null);
+
+ Map<String, String> address = dbutils.getMap(rs, "address");
+
+ completePlace(doc);
+ // Add address last, so it takes precedence.
+ doc.address(address);
+
+ doc.setCountry(countryNames.get(rs.getString("country_code")));
+
+ return NominatimResult.fromAddress(doc, address);
+ };
+
+ // Setup handling of interpolation table. There are two different formats depending on the Nominatim version.
+ // new-style interpolations
+ final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper();
+ osmlineToNominatimResult = (rs, rownum) -> {
+ PhotonDoc doc = osmlineRowMapper.mapRow(rs, rownum);
+
+ completePlace(doc);
+ doc.setCountry(countryNames.get(rs.getString("country_code")));
+
+ Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
+
+ if (hasNewStyleInterpolation) {
+ return NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
+ rs.getLong("step"), geometry);
+ }
+
+ return NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
+ rs.getString("interpolationtype"), geometry);
+ };
+ }
+
+
+
public boolean isBusy() {
return updateLock.isLocked();
}
@@ -81,7 +140,7 @@ public void initUpdates(String updateUser) {
public void update() {
if (updateLock.tryLock()) {
try {
- exporter.loadCountryNames();
+ loadCountryNames();
updateFromPlacex();
updateFromInterpolations();
updater.finish();
@@ -104,7 +163,7 @@ private void updateFromPlacex() {
boolean checkForMultidoc = true;
if (!place.isToDelete()) {
- final List<PhotonDoc> updatedDocs = exporter.getByPlaceId(placeId);
+ final List<PhotonDoc> updatedDocs = getByPlaceId(placeId);
if (updatedDocs != null && !updatedDocs.isEmpty() && updatedDocs.get(0).isUsefulForIndex()) {
checkForMultidoc = updatedDocs.get(0).getRankAddress() == 30;
++updatedPlaces;
@@ -144,7 +203,7 @@ private void updateFromInterpolations() {
int objectId = -1;
if (!place.isToDelete()) {
- final List<PhotonDoc> updatedDocs = exporter.getInterpolationsByPlaceId(placeId);
+ final List<PhotonDoc> updatedDocs = getInterpolationsByPlaceId(placeId);
if (updatedDocs != null) {
++updatedInterpolations;
for (PhotonDoc updatedDoc : updatedDocs) {
@@ -166,7 +225,7 @@ private void updateFromInterpolations() {
}
private List<UpdateRow> getPlaces(String table) {
- List<UpdateRow> results = template.query(exporter.getDataAdaptor().deleteReturning(
+ List<UpdateRow> results = template.query(dbutils.deleteReturning(
"DELETE FROM photon_updates WHERE rel = ?", "place_id, operation, indexed_date"),
(rs, rowNum) -> {
boolean isDelete = "DELETE".equals(rs.getString("operation"));
@@ -191,23 +250,86 @@ private List<UpdateRow> getPlaces(String table) {
}
+ public List<PhotonDoc> getByPlaceId(long placeId) {
+ List<NominatimResult> result = template.query(
+ SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ? and indexed_status = 0",
+ placeToNominatimResult, placeId);
+
+ return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber();
+ }
+
+ public List<PhotonDoc> getInterpolationsByPlaceId(long placeId) {
+ List<NominatimResult> result = template.query(
+ (hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE)
+ + " FROM location_property_osmline WHERE place_id = ? and indexed_status = 0",
+ osmlineToNominatimResult, placeId);
+
+ return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber();
+ }
+
/**
- * Create a new instance.
- *
- * @param host Nominatim database host
- * @param port Nominatim database port
- * @param database Nominatim database name
- * @param username Nominatim database username
- * @param password Nominatim database password
+ * Query Nominatim's address hierarchy to complete photon doc with missing data (like country, city, street, ...)
+ *
+ * @param doc
*/
- public NominatimUpdater(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) {
- BasicDataSource dataSource = NominatimConnector.buildDataSource(host, port, database, username, password, true);
+ private void completePlace(PhotonDoc doc) {
+ final List<AddressRow> addresses = getAddresses(doc);
+ final AddressType doctype = doc.getAddressType();
+ for (AddressRow address : addresses) {
+ AddressType atype = address.getAddressType();
- exporter = new NominatimConnector(host, port, database, username, password, dataAdapter);
- template = new JdbcTemplate(dataSource);
+ if (atype != null
+ && (atype == doctype || !doc.setAddressPartIfNew(atype, address.getName()))
+ && address.isUsefulForContext()) {
+ // no specifically handled item, check if useful for context
+ doc.getContext().add(address.getName());
+ }
+ }
}
- public NominatimUpdater(String host, int port, String database, String username, String password) {
- this(host, port, database, username, password, new PostgisDataAdapter());
+ List<AddressRow> getAddresses(PhotonDoc doc) {
+ RowMapper<AddressRow> rowMapper = (rs, rowNum) -> new AddressRow(
+ dbutils.getMap(rs, "name"),
+ rs.getString("class"),
+ rs.getString("type"),
+ rs.getInt("rank_address")
+ );
+
+ AddressType atype = doc.getAddressType();
+
+ if (atype == null || atype == AddressType.COUNTRY) {
+ return Collections.emptyList();
+ }
+
+ List<AddressRow> terms = null;
+
+ if (atype == AddressType.HOUSE) {
+ long placeId = doc.getParentPlaceId();
+ if (placeId != parentPlaceId) {
+ parentTerms = template.query(SELECT_COLS_ADDRESS
+ + " FROM placex p, place_addressline pa"
+ + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?"
+ + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress"
+ + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc",
+ rowMapper, placeId, placeId);
+
+ // need to add the term for the parent place ID itself
+ parentTerms.addAll(0, template.query(SELECT_COLS_ADDRESS + " FROM placex p WHERE p.place_id = ?",
+ rowMapper, placeId));
+ parentPlaceId = placeId;
+ }
+ terms = parentTerms;
+
+ } else {
+ long placeId = doc.getPlaceId();
+ terms = template.query(SELECT_COLS_ADDRESS
+ + " FROM placex p, place_addressline pa"
+ + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?"
+ + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress"
+ + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc",
+ rowMapper, placeId, placeId);
+ }
+
+ return terms;
}
}
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
index 2e2ac28a..77c9f696 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
@@ -22,7 +22,7 @@
class NominatimConnectorDBTest {
private EmbeddedDatabase db;
- private NominatimConnector connector;
+ private NominatimImporter connector;
private CollectingImporter importer;
private JdbcTemplate jdbc;
@@ -35,11 +35,11 @@ void setup() {
.build();
- connector = new NominatimConnector(null, 0, null, null, null, new H2DataAdapter());
+ connector = new NominatimImporter(null, 0, null, null, null, new H2DataAdapter());
importer = new CollectingImporter();
jdbc = new JdbcTemplate(db);
- ReflectionTestUtil.setFieldValue(connector, "template", jdbc);
+ ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "template", jdbc);
}
private void readEntireDatabase() {
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java
index cba15bb1..acab94cb 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java
@@ -31,8 +31,7 @@ void setup() {
connector.setUpdater(updater);
jdbc = new JdbcTemplate(db);
- ReflectionTestUtil.setFieldValue(connector, "template", jdbc);
- ReflectionTestUtil.setFieldValue(connector, "exporter", "template", jdbc);
+ ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "template", jdbc);
}
@Test
From e941c9882414f653c29387fc3dac5435952ea4b1 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 8 Nov 2024 17:25:41 +0100
Subject: [PATCH 10/17] clean code in readCountry
---
.../photon/nominatim/NominatimImporter.java | 76 +++++++++----------
1 file changed, 37 insertions(+), 39 deletions(-)
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
index 9f4020c7..ac041b76 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
@@ -1,10 +1,7 @@
package de.komoot.photon.nominatim;
import de.komoot.photon.PhotonDoc;
-import de.komoot.photon.nominatim.model.AddressRow;
-import de.komoot.photon.nominatim.model.AddressType;
-import de.komoot.photon.nominatim.model.OsmlineRowMapper;
-import de.komoot.photon.nominatim.model.PlaceRowMapper;
+import de.komoot.photon.nominatim.model.*;
import org.apache.commons.dbcp2.BasicDataSource;
import org.locationtech.jts.geom.Geometry;
import org.slf4j.Logger;
@@ -14,6 +11,7 @@
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Types;
import java.util.*;
/**
@@ -94,29 +92,49 @@ public void readCountry(String countryCode, ImportThread importThread) {
return;
}
+ final String countrySQL;
+ final Object[] sqlArgs;
+ final int[] sqlArgTypes;
+ if ("".equals(countryCode)) {
+ countrySQL = "country_code is null";
+ sqlArgs = new Object[0];
+ sqlArgTypes = new int[0];
+ } else {
+ countrySQL = "country_code = ?";
+ sqlArgs = new Object[]{countryCode};
+ sqlArgTypes = new int[]{Types.VARCHAR};
+ }
+
final PlaceRowMapper placeRowMapper = new PlaceRowMapper(dbutils);
- final RowCallbackHandler placeMapper = rs -> {
- final PhotonDoc doc = placeRowMapper.mapRow(rs, 0);
- assert (doc != null);
+ template.query(SELECT_COLS_PLACEX + " FROM placex " +
+ " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND " + countrySQL +
+ " ORDER BY geometry_sector, parent_place_id",
+ sqlArgs, sqlArgTypes, rs -> {
+ final PhotonDoc doc = placeRowMapper.mapRow(rs, 0);
+ assert (doc != null);
- final Map<String, String> address = dbutils.getMap(rs, "address");
+ final Map<String, String> address = dbutils.getMap(rs, "address");
- completePlace(doc);
- // Add address last, so it takes precedence.
- doc.address(address);
+ completePlace(doc);
+ // Add address last, so it takes precedence.
+ doc.address(address);
- doc.setCountry(cnames);
+ doc.setCountry(cnames);
- var result = NominatimResult.fromAddress(doc, address);
+ var result = NominatimResult.fromAddress(doc, address);
- if (result.isUsefulForIndex()) {
- importThread.addDocument(result);
- }
- };
+ if (result.isUsefulForIndex()) {
+ importThread.addDocument(result);
+ }
+ });
final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper();
- final RowCallbackHandler osmlineMapper = rs -> {
+ template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
+ " FROM location_property_osmline" +
+ " WHERE startnumber is not null AND " + countrySQL +
+ " ORDER BY geometry_sector, parent_place_id",
+ sqlArgs, sqlArgTypes, rs -> {
final PhotonDoc doc = osmlineRowMapper.mapRow(rs, 0);
completePlace(doc);
@@ -137,28 +155,8 @@ public void readCountry(String countryCode, ImportThread importThread) {
if (docs.isUsefulForIndex()) {
importThread.addDocument(docs);
}
- };
+ });
- if ("".equals(countryCode)) {
- template.query(SELECT_COLS_PLACEX + " FROM placex " +
- " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code is null" +
- " ORDER BY geometry_sector, parent_place_id; ", placeMapper);
-
- template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
- " FROM location_property_osmline" +
- " WHERE startnumber is not null AND country_code is null" +
- " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper);
- } else {
- template.query(SELECT_COLS_PLACEX + " FROM placex " +
- " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND country_code = ?" +
- " ORDER BY geometry_sector, parent_place_id; ", placeMapper, countryCode);
-
- template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
- " FROM location_property_osmline" +
- " WHERE startnumber is not null AND country_code = ?" +
- " ORDER BY geometry_sector, parent_place_id; ", osmlineMapper, countryCode);
-
- }
}
/**
From b2b66228eb08d15ff3c72d00a631fe02de97e976 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 8 Nov 2024 23:28:09 +0100
Subject: [PATCH 11/17] precompute address rows on import
---
.../photon/nominatim/DBDataAdapter.java | 5 +
.../photon/nominatim/NominatimConnector.java | 5 -
.../photon/nominatim/NominatimImporter.java | 195 ++++++++++--------
.../photon/nominatim/NominatimUpdater.java | 5 +
.../photon/nominatim/PostgisDataAdapter.java | 5 +
.../photon/nominatim/model/AddressRow.java | 10 +
.../model/NominatimAddressCache.java | 67 ++++++
.../nominatim/NominatimConnectorDBTest.java | 24 +--
.../nominatim/testdb/H2DataAdapter.java | 4 +
.../nominatim/testdb/PlacexTestRow.java | 6 +
10 files changed, 219 insertions(+), 107 deletions(-)
create mode 100644 src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
diff --git a/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java b/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java
index 7608a9ab..6e76426f 100644
--- a/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java
+++ b/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java
@@ -30,4 +30,9 @@ public interface DBDataAdapter {
* Wrap a DELETE statement with a RETURNING clause.
*/
String deleteReturning(String deleteSQL, String columns);
+
+ /**
+ * Wrap function to create a json array from a SELECT.
+ */
+ String jsonArrayFromSelect(String valueSQL, String fromSQL);
}
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 9c1b89af..8b6c0c8a 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -15,11 +15,6 @@
* Base class for workers connecting to a Nominatim database
*/
public class NominatimConnector {
- protected static final String SELECT_COLS_PLACEX = "SELECT place_id, osm_type, osm_id, class, type, name, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_address, rank_search, importance, country_code, centroid";
- protected static final String SELECT_COLS_ADDRESS = "SELECT p.name, p.class, p.type, p.rank_address";
- protected static final String SELECT_OSMLINE_OLD_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
- protected static final String SELECT_OSMLINE_NEW_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo";
-
protected final DBDataAdapter dbutils;
protected final JdbcTemplate template;
protected Map<String, Map<String, String>> countryNames;
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
index ac041b76..f1ce7fe4 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
@@ -2,17 +2,12 @@
import de.komoot.photon.PhotonDoc;
import de.komoot.photon.nominatim.model.*;
-import org.apache.commons.dbcp2.BasicDataSource;
import org.locationtech.jts.geom.Geometry;
import org.slf4j.Logger;
-import org.springframework.jdbc.core.JdbcTemplate;
-import org.springframework.jdbc.core.RowCallbackHandler;
-import org.springframework.jdbc.core.RowMapper;
-import java.sql.ResultSet;
-import java.sql.SQLException;
import java.sql.Types;
-import java.util.*;
+import java.util.List;
+import java.util.Map;
/**
* Importer for data from a Nominatim database.
@@ -20,10 +15,6 @@
public class NominatimImporter extends NominatimConnector {
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimImporter.class);
- // One-item cache for address lookup. Speeds up rank 30 processing.
- private long parentPlaceId = -1;
- private List<AddressRow> parentTerms = null;
-
public NominatimImporter(String host, int port, String database, String username, String password) {
this(host, port, database, username, password, new PostgisDataAdapter());
}
@@ -33,52 +24,6 @@ public NominatimImporter(String host, int port, String database, String username
}
- List<AddressRow> getAddresses(PhotonDoc doc) {
- RowMapper<AddressRow> rowMapper = (rs, rowNum) -> new AddressRow(
- dbutils.getMap(rs, "name"),
- rs.getString("class"),
- rs.getString("type"),
- rs.getInt("rank_address")
- );
-
- AddressType atype = doc.getAddressType();
-
- if (atype == null || atype == AddressType.COUNTRY) {
- return Collections.emptyList();
- }
-
- List<AddressRow> terms = null;
-
- if (atype == AddressType.HOUSE) {
- long placeId = doc.getParentPlaceId();
- if (placeId != parentPlaceId) {
- parentTerms = template.query(SELECT_COLS_ADDRESS
- + " FROM placex p, place_addressline pa"
- + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?"
- + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress"
- + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc",
- rowMapper, placeId, placeId);
-
- // need to add the term for the parent place ID itself
- parentTerms.addAll(0, template.query(SELECT_COLS_ADDRESS + " FROM placex p WHERE p.place_id = ?",
- rowMapper, placeId));
- parentPlaceId = placeId;
- }
- terms = parentTerms;
-
- } else {
- long placeId = doc.getPlaceId();
- terms = template.query(SELECT_COLS_ADDRESS
- + " FROM placex p, place_addressline pa"
- + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?"
- + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress"
- + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc",
- rowMapper, placeId, placeId);
- }
-
- return terms;
- }
-
/**
* Parse every relevant row in placex and location_osmline
* for the given country. Also imports place from county-less places.
@@ -105,21 +50,74 @@ public void readCountry(String countryCode, ImportThread importThread) {
sqlArgTypes = new int[]{Types.VARCHAR};
}
+ NominatimAddressCache addressCache = new NominatimAddressCache();
+ addressCache.loadCountryAddresses(template, dbutils, countryCode);
+
final PlaceRowMapper placeRowMapper = new PlaceRowMapper(dbutils);
- template.query(SELECT_COLS_PLACEX + " FROM placex " +
- " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND " + countrySQL +
- " ORDER BY geometry_sector, parent_place_id",
+ // First read ranks below 30, independent places
+ template.query(
+ "SELECT place_id, osm_type, osm_id, class, type, name, postcode," +
+ " address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id," +
+ " linked_place_id, rank_address, rank_search, importance, country_code, centroid," +
+ dbutils.jsonArrayFromSelect(
+ "address_place_id",
+ "FROM place_addressline pa " +
+ " WHERE pa.place_id = p.place_id AND isaddress" +
+ " ORDER BY cached_rank_address DESC") + " as addresslines" +
+ " FROM placex p" +
+ " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND " + countrySQL +
+ " AND rank_search < 30" +
+ " ORDER BY geometry_sector, parent_place_id",
sqlArgs, sqlArgTypes, rs -> {
final PhotonDoc doc = placeRowMapper.mapRow(rs, 0);
+ final Map<String, String> address = dbutils.getMap(rs, "address");
+
assert (doc != null);
- final Map<String, String> address = dbutils.getMap(rs, "address");
+ final var addressPlaces = addressCache.getAddressList(rs.getString("addresslines"));
+ completePlace(doc, addressPlaces);
+ doc.address(address); // take precedence over computed address
+ doc.setCountry(cnames);
+
+ var result = NominatimResult.fromAddress(doc, address);
+ if (result.isUsefulForIndex()) {
+ importThread.addDocument(result);
+ }
+ });
+
+ // Next get all POIs/housenumbers.
+ template.query(
+ "SELECT p.place_id, p.osm_type, p.osm_id, p.class, p.type, p.name, p.postcode," +
+ " p.address, p.extratags, ST_Envelope(p.geometry) AS bbox, p.parent_place_id," +
+ " p.linked_place_id, p.rank_address, p.rank_search, p.importance, p.country_code, p.centroid," +
+ " parent.class as parent_class, parent.type as parent_type," +
+ " parent.rank_address as parent_rank_address, parent.name as parent_name, " +
+ dbutils.jsonArrayFromSelect(
+ "address_place_id",
+ "FROM place_addressline pa " +
+ " WHERE pa.place_id IN (p.place_id, coalesce(p.parent_place_id, p.place_id)) AND isaddress" +
+ " ORDER BY cached_rank_address DESC, pa.place_id = p.place_id DESC") + " as addresslines" +
+ " FROM placex p LEFT JOIN placex parent ON p.parent_place_id = parent.place_id" +
+ " WHERE p.linked_place_id IS NULL AND p.centroid IS NOT NULL AND p." + countrySQL +
+ " AND p.rank_search = 30 " +
+ " ORDER BY p.geometry_sector",
+ sqlArgs, sqlArgTypes, rs -> {
+ final PhotonDoc doc = placeRowMapper.mapRow(rs, 0);
+ final Map<String, String> address = dbutils.getMap(rs, "address");
- completePlace(doc);
- // Add address last, so it takes precedence.
- doc.address(address);
+ assert (doc != null);
+ final var addressPlaces = addressCache.getAddressList(rs.getString("addresslines"));
+ if (rs.getString("parent_class") != null) {
+ addressPlaces.add(0, new AddressRow(
+ dbutils.getMap(rs, "parent_name"),
+ rs.getString("parent_class"),
+ rs.getString("parent_type"),
+ rs.getInt("parent_rank_address")));
+ }
+ completePlace(doc, addressPlaces);
+ doc.address(address); // take precedence over computed address
doc.setCountry(cnames);
var result = NominatimResult.fromAddress(doc, address);
@@ -130,32 +128,50 @@ public void readCountry(String countryCode, ImportThread importThread) {
});
final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper();
- template.query((hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) +
- " FROM location_property_osmline" +
- " WHERE startnumber is not null AND " + countrySQL +
- " ORDER BY geometry_sector, parent_place_id",
+ template.query(
+ "SELECT p.place_id, p.osm_id, p.parent_place_id, p.startnumber, p.endnumber, p.postcode, p.country_code, p.linegeo," +
+ (hasNewStyleInterpolation ? " p.step," : " p.interpolationtype,") +
+ " parent.class as parent_class, parent.type as parent_type," +
+ " parent.rank_address as parent_rank_address, parent.name as parent_name, " +
+ dbutils.jsonArrayFromSelect(
+ "address_place_id",
+ "FROM place_addressline pa " +
+ " WHERE pa.place_id IN (p.place_id, coalesce(p.parent_place_id, p.place_id)) AND isaddress" +
+ " ORDER BY cached_rank_address DESC, pa.place_id = p.place_id DESC") + " as addresslines" +
+ " FROM location_property_osmline p LEFT JOIN placex parent ON p.parent_place_id = parent.place_id" +
+ " WHERE startnumber is not null AND p." + countrySQL +
+ " ORDER BY p.geometry_sector, p.parent_place_id",
sqlArgs, sqlArgTypes, rs -> {
- final PhotonDoc doc = osmlineRowMapper.mapRow(rs, 0);
-
- completePlace(doc);
- doc.setCountry(cnames);
-
- final Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
- final NominatimResult docs;
- if (hasNewStyleInterpolation) {
- docs = NominatimResult.fromInterpolation(
- doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
- rs.getLong("step"), geometry);
- } else {
- docs = NominatimResult.fromInterpolation(
- doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
- rs.getString("interpolationtype"), geometry);
- }
+ final PhotonDoc doc = osmlineRowMapper.mapRow(rs, 0);
+
+ final var addressPlaces = addressCache.getAddressList(rs.getString("addresslines"));
+ if (rs.getString("parent_class") != null) {
+ addressPlaces.add(0, new AddressRow(
+ dbutils.getMap(rs, "parent_name"),
+ rs.getString("parent_class"),
+ rs.getString("parent_type"),
+ rs.getInt("parent_rank_address")));
+ }
+ completePlace(doc, addressPlaces);
- if (docs.isUsefulForIndex()) {
- importThread.addDocument(docs);
- }
- });
+ doc.setCountry(cnames);
+
+ final Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
+ final NominatimResult docs;
+ if (hasNewStyleInterpolation) {
+ docs = NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
+ rs.getLong("step"), geometry);
+ } else {
+ docs = NominatimResult.fromInterpolation(
+ doc, rs.getLong("startnumber"), rs.getLong("endnumber"),
+ rs.getString("interpolationtype"), geometry);
+ }
+
+ if (docs.isUsefulForIndex()) {
+ importThread.addDocument(docs);
+ }
+ });
}
@@ -164,8 +180,7 @@ public void readCountry(String countryCode, ImportThread importThread) {
*
* @param doc
*/
- private void completePlace(PhotonDoc doc) {
- final List<AddressRow> addresses = getAddresses(doc);
+ private void completePlace(PhotonDoc doc, List<AddressRow> addresses) {
final AddressType doctype = doc.getAddressType();
for (AddressRow address : addresses) {
AddressType atype = address.getAddressType();
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
index 1a0f5033..2fb938de 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
@@ -15,6 +15,11 @@
public class NominatimUpdater extends NominatimConnector {
private static final org.slf4j.Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimUpdater.class);
+ private static final String SELECT_COLS_PLACEX = "SELECT place_id, osm_type, osm_id, class, type, name, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_address, rank_search, importance, country_code, centroid";
+ private static final String SELECT_COLS_ADDRESS = "SELECT p.name, p.class, p.type, p.rank_address";
+ private static final String SELECT_OSMLINE_OLD_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
+ private static final String SELECT_OSMLINE_NEW_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo";
+
private static final String TRIGGER_SQL =
"DROP TABLE IF EXISTS photon_updates;"
+ "CREATE TABLE photon_updates (rel TEXT, place_id BIGINT,"
diff --git a/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java b/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java
index bac2f4a9..37e6718f 100644
--- a/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java
+++ b/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java
@@ -58,4 +58,9 @@ public Boolean mapRow(ResultSet resultSet, int i) throws SQLException {
public String deleteReturning(String deleteSQL, String columns) {
return deleteSQL + " RETURNING " + columns;
}
+
+ @Override
+ public String jsonArrayFromSelect(String valueSQL, String fromSQL) {
+ return "(SELECT json_agg(val) FROM (SELECT " + valueSQL + " as val " + fromSQL + ") xxx)";
+ }
}
diff --git a/src/main/java/de/komoot/photon/nominatim/model/AddressRow.java b/src/main/java/de/komoot/photon/nominatim/model/AddressRow.java
index 97a02e67..a693b687 100644
--- a/src/main/java/de/komoot/photon/nominatim/model/AddressRow.java
+++ b/src/main/java/de/komoot/photon/nominatim/model/AddressRow.java
@@ -37,4 +37,14 @@ public boolean isUsefulForContext() {
public Map<String, String> getName() {
return this.name;
}
+
+ @Override
+ public String toString() {
+ return "AddressRow{" +
+ "name=" + name.getOrDefault("name", "?") +
+ ", osmKey='" + osmKey + '\'' +
+ ", osmValue='" + osmValue + '\'' +
+ ", rankAddress=" + rankAddress +
+ '}';
+ }
}
diff --git a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
new file mode 100644
index 00000000..d9a7e2fe
--- /dev/null
+++ b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
@@ -0,0 +1,67 @@
+package de.komoot.photon.nominatim.model;
+
+import de.komoot.photon.nominatim.DBDataAdapter;
+import org.json.JSONArray;
+import org.slf4j.Logger;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.RowCallbackHandler;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Container for caching information about address parts.
+ */
+public class NominatimAddressCache {
+ private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimAddressCache.class);
+
+ private static final String BASE_COUNTRY_QUERY =
+ "SELECT place_id, name, class, type, rank_address FROM placex" +
+ " WHERE rank_address between 5 and 25 AND linked_place_id is null";
+
+ private final Map<Integer, AddressRow> addresses = new HashMap<>();
+
+ public void loadCountryAddresses(JdbcTemplate template, DBDataAdapter dbutils, String countryCode) {
+ final RowCallbackHandler rowMapper = (rs) -> {
+ addresses.put(
+ rs.getInt("place_id"),
+ new AddressRow(
+ dbutils.getMap(rs, "name"),
+ rs.getString("class"),
+ rs.getString("type"),
+ rs.getInt("rank_address")
+ ));
+ };
+
+ if (countryCode == null) {
+ template.query(BASE_COUNTRY_QUERY + " AND country_code is null", rowMapper);
+ } else {
+ template.query(BASE_COUNTRY_QUERY + " AND country_code = ?", rowMapper, countryCode);
+ }
+
+ if (addresses.size() > 0) {
+ LOGGER.info("Loaded {} address places for country {}", addresses.size(), countryCode);
+ }
+ }
+
+ public List<AddressRow> getAddressList(String addressline) {
+ ArrayList<AddressRow> outlist = new ArrayList<>();
+
+ if (addressline != null && !addressline.isBlank()) {
+ JSONArray addressPlaces = new JSONArray(addressline);
+ for (int i = 0; i < addressPlaces.length(); ++i) {
+ Integer place_id = addressPlaces.optInt(i);
+ if (place_id != null) {
+ AddressRow row = addresses.get(place_id);
+ if (row != null) {
+ outlist.add(row);
+ }
+ }
+ }
+ }
+
+ return outlist;
+ }
+}
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
index 77c9f696..4830b5f6 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
@@ -98,11 +98,11 @@ void testPlaceAddress() throws ParseException {
PlacexTestRow place = PlacexTestRow.make_street("Burg").add(jdbc);
place.addAddresslines(jdbc,
- new PlacexTestRow("place", "neighbourhood").name("Le Coin").rankAddress(24).add(jdbc),
- new PlacexTestRow("place", "suburb").name("Crampton").rankAddress(20).add(jdbc),
- new PlacexTestRow("place", "city").name("Grand Junction").rankAddress(16).add(jdbc),
- new PlacexTestRow("place", "county").name("Lost County").rankAddress(12).add(jdbc),
- new PlacexTestRow("place", "state").name("Le Havre").rankAddress(8).add(jdbc));
+ new PlacexTestRow("place", "neighbourhood").name("Le Coin").ranks(24).add(jdbc),
+ new PlacexTestRow("place", "suburb").name("Crampton").ranks(20).add(jdbc),
+ new PlacexTestRow("place", "city").name("Grand Junction").ranks(16).add(jdbc),
+ new PlacexTestRow("place", "county").name("Lost County").ranks(12).add(jdbc),
+ new PlacexTestRow("place", "state").name("Le Havre").ranks(8).add(jdbc));
readEntireDatabase();
@@ -123,8 +123,8 @@ void testPlaceAddressAddressRank0() throws ParseException {
PlacexTestRow place = new PlacexTestRow("natural", "water").name("Lake Tee").rankAddress(0).rankSearch(20).add(jdbc);
place.addAddresslines(jdbc,
- new PlacexTestRow("place", "county").name("Lost County").rankAddress(12).add(jdbc),
- new PlacexTestRow("place", "state").name("Le Havre").rankAddress(8).add(jdbc));
+ new PlacexTestRow("place", "county").name("Lost County").ranks(12).add(jdbc),
+ new PlacexTestRow("place", "state").name("Le Havre").ranks(8).add(jdbc));
readEntireDatabase();
@@ -142,7 +142,7 @@ void testPoiAddress() throws ParseException {
PlacexTestRow parent = PlacexTestRow.make_street("Burg").add(jdbc);
parent.addAddresslines(jdbc,
- new PlacexTestRow("place", "city").name("Grand Junction").rankAddress(16).add(jdbc));
+ new PlacexTestRow("place", "city").name("Grand Junction").ranks(16).add(jdbc));
PlacexTestRow place = new PlacexTestRow("place", "house").name("House").parent(parent).add(jdbc);
@@ -224,11 +224,11 @@ void testInterpolationWithSteps() throws ParseException {
@Test
void testAddressMappingDuplicate() {
PlacexTestRow place = PlacexTestRow.make_street("Main Street").add(jdbc);
- PlacexTestRow munip = new PlacexTestRow("place", "municipality").name("Gemeinde").rankAddress(14).add(jdbc);
+ PlacexTestRow munip = new PlacexTestRow("place", "municipality").name("Gemeinde").ranks(14).add(jdbc);
place.addAddresslines(jdbc,
munip,
- new PlacexTestRow("place", "village").name("Dorf").rankAddress(16).add(jdbc));
+ new PlacexTestRow("place", "village").name("Dorf").ranks(16).add(jdbc));
readEntireDatabase();
@@ -245,8 +245,8 @@ void testAddressMappingDuplicate() {
*/
@Test
void testAddressMappingAvoidSameTypeAsPlace() {
- PlacexTestRow village = new PlacexTestRow("place", "village").name("Dorf").rankAddress(16).add(jdbc);
- PlacexTestRow munip = new PlacexTestRow("place", "municipality").name("Gemeinde").rankAddress(14).add(jdbc);
+ PlacexTestRow village = new PlacexTestRow("place", "village").name("Dorf").ranks(16).add(jdbc);
+ PlacexTestRow munip = new PlacexTestRow("place", "municipality").name("Gemeinde").ranks(14).add(jdbc);
village.addAddresslines(jdbc, munip);
diff --git a/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java b/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java
index ec21b4eb..d97c5723 100644
--- a/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java
+++ b/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java
@@ -45,4 +45,8 @@ public String deleteReturning(String deleteSQL, String columns) {
return "SELECT " + columns + " FROM OLD TABLE (" + deleteSQL + ")";
}
+ @Override
+ public String jsonArrayFromSelect(String valueSQL, String fromSQL) {
+ return "json_array((SELECT " + valueSQL + " " + fromSQL + ") FORMAT JSON)";
+ }
}
diff --git a/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java b/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java
index eb5190e3..85a14553 100644
--- a/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java
+++ b/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java
@@ -104,6 +104,12 @@ public PlacexTestRow rankAddress(int rank) {
return this;
}
+ public PlacexTestRow ranks(int rank) {
+ this.rankAddress = rank;
+ this.rankSearch = rank;
+ return this;
+ }
+
public PlacexTestRow parent(PlacexTestRow row) {
this.parentPlaceId = row.getPlaceId();
return this;
From d8719456d2a805f86020a743ad0041169ad63642 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 8 Nov 2024 23:50:51 +0100
Subject: [PATCH 12/17] avoid duplication of completePlace function
---
src/main/java/de/komoot/photon/PhotonDoc.java | 18 +++++++++++++
.../photon/nominatim/NominatimImporter.java | 25 +++----------------
.../photon/nominatim/NominatimUpdater.java | 23 ++---------------
3 files changed, 23 insertions(+), 43 deletions(-)
diff --git a/src/main/java/de/komoot/photon/PhotonDoc.java b/src/main/java/de/komoot/photon/PhotonDoc.java
index 147cf151..95d22a15 100644
--- a/src/main/java/de/komoot/photon/PhotonDoc.java
+++ b/src/main/java/de/komoot/photon/PhotonDoc.java
@@ -1,5 +1,6 @@
package de.komoot.photon;
+import de.komoot.photon.nominatim.model.AddressRow;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.Point;
@@ -241,6 +242,23 @@ public boolean setAddressPartIfNew(AddressType addressType, Map<String, String>
return addressParts.computeIfAbsent(addressType, k -> names) == names;
}
+ /**
+ * Complete address data from a list of address rows.
+ */
+ public void completePlace(List<AddressRow> addresses) {
+ final AddressType doctype = getAddressType();
+ for (AddressRow address : addresses) {
+ final AddressType atype = address.getAddressType();
+
+ if (atype != null
+ && (atype == doctype || !setAddressPartIfNew(atype, address.getName()))
+ && address.isUsefulForContext()) {
+ // no specifically handled item, check if useful for context
+ getContext().add(address.getName());
+ }
+ }
+ }
+
public void setCountry(Map<String, String> names) {
addressParts.put(AddressType.COUNTRY, names);
}
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
index f1ce7fe4..c66b4842 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
@@ -74,8 +74,7 @@ public void readCountry(String countryCode, ImportThread importThread) {
assert (doc != null);
- final var addressPlaces = addressCache.getAddressList(rs.getString("addresslines"));
- completePlace(doc, addressPlaces);
+ doc.completePlace(addressCache.getAddressList(rs.getString("addresslines")));
doc.address(address); // take precedence over computed address
doc.setCountry(cnames);
@@ -116,7 +115,7 @@ public void readCountry(String countryCode, ImportThread importThread) {
rs.getString("parent_type"),
rs.getInt("parent_rank_address")));
}
- completePlace(doc, addressPlaces);
+ doc.completePlace(addressPlaces);
doc.address(address); // take precedence over computed address
doc.setCountry(cnames);
@@ -152,7 +151,7 @@ public void readCountry(String countryCode, ImportThread importThread) {
rs.getString("parent_type"),
rs.getInt("parent_rank_address")));
}
- completePlace(doc, addressPlaces);
+ doc.completePlace(addressPlaces);
doc.setCountry(cnames);
@@ -175,24 +174,6 @@ public void readCountry(String countryCode, ImportThread importThread) {
}
- /**
- * Query Nominatim's address hierarchy to complete photon doc with missing data (like country, city, street, ...)
- *
- * @param doc
- */
- private void completePlace(PhotonDoc doc, List<AddressRow> addresses) {
- final AddressType doctype = doc.getAddressType();
- for (AddressRow address : addresses) {
- AddressType atype = address.getAddressType();
-
- if (atype != null
- && (atype == doctype || !doc.setAddressPartIfNew(atype, address.getName()))
- && address.isUsefulForContext()) {
- // no specifically handled item, check if useful for context
- doc.getContext().add(address.getName());
- }
- }
- }
/**
* Prepare the database for export.
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
index 2fb938de..0b2c6186 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
@@ -89,7 +89,7 @@ public NominatimUpdater(String host, int port, String database, String username,
Map<String, String> address = dbutils.getMap(rs, "address");
- completePlace(doc);
+ doc.completePlace(getAddresses(doc));
// Add address last, so it takes precedence.
doc.address(address);
@@ -104,7 +104,7 @@ public NominatimUpdater(String host, int port, String database, String username,
osmlineToNominatimResult = (rs, rownum) -> {
PhotonDoc doc = osmlineRowMapper.mapRow(rs, rownum);
- completePlace(doc);
+ doc.completePlace(getAddresses(doc));
doc.setCountry(countryNames.get(rs.getString("country_code")));
Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
@@ -272,25 +272,6 @@ public List<PhotonDoc> getInterpolationsByPlaceId(long placeId) {
return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber();
}
- /**
- * Query Nominatim's address hierarchy to complete photon doc with missing data (like country, city, street, ...)
- *
- * @param doc
- */
- private void completePlace(PhotonDoc doc) {
- final List<AddressRow> addresses = getAddresses(doc);
- final AddressType doctype = doc.getAddressType();
- for (AddressRow address : addresses) {
- AddressType atype = address.getAddressType();
-
- if (atype != null
- && (atype == doctype || !doc.setAddressPartIfNew(atype, address.getName()))
- && address.isUsefulForContext()) {
- // no specifically handled item, check if useful for context
- doc.getContext().add(address.getName());
- }
- }
- }
List<AddressRow> getAddresses(PhotonDoc doc) {
RowMapper<AddressRow> rowMapper = (rs, rowNum) -> new AddressRow(
From 6b4dd50a8be3128ec6b68f89b2678456606ce670 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Sat, 9 Nov 2024 16:33:37 +0100
Subject: [PATCH 13/17] add transaction manager and go back to manual commit
With autocommit, server-side cursors don't work, slowing down the
large queries.
Without autocommit without a transaction manager, all queries will be
rolled back. So protect at least the writing queries.
---
.../photon/nominatim/NominatimConnector.java | 9 +++-
.../photon/nominatim/NominatimImporter.java | 26 +++++----
.../photon/nominatim/NominatimUpdater.java | 53 +++++++++++--------
.../nominatim/NominatimConnectorDBTest.java | 5 ++
.../nominatim/NominatimUpdaterDBTest.java | 5 ++
5 files changed, 65 insertions(+), 33 deletions(-)
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
index 8b6c0c8a..e5b28d6e 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
@@ -3,6 +3,8 @@
import org.apache.commons.dbcp2.BasicDataSource;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.transaction.support.TransactionTemplate;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -17,6 +19,7 @@
public class NominatimConnector {
protected final DBDataAdapter dbutils;
protected final JdbcTemplate template;
+ protected final TransactionTemplate txTemplate;
protected Map<String, Map<String, String>> countryNames;
protected final boolean hasNewStyleInterpolation;
@@ -28,7 +31,11 @@ protected NominatimConnector(String host, int port, String database, String user
if (password != null) {
dataSource.setPassword(password);
}
- dataSource.setDefaultAutoCommit(true);
+
+ // Keep disabled or server-side cursors won't work.
+ dataSource.setDefaultAutoCommit(false);
+
+ txTemplate = new TransactionTemplate(new DataSourceTransactionManager(dataSource));
template = new JdbcTemplate(dataSource);
template.setFetchSize(100000);
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
index c66b4842..8c5ba686 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java
@@ -1,12 +1,14 @@
package de.komoot.photon.nominatim;
import de.komoot.photon.PhotonDoc;
-import de.komoot.photon.nominatim.model.*;
+import de.komoot.photon.nominatim.model.AddressRow;
+import de.komoot.photon.nominatim.model.NominatimAddressCache;
+import de.komoot.photon.nominatim.model.OsmlineRowMapper;
+import de.komoot.photon.nominatim.model.PlaceRowMapper;
import org.locationtech.jts.geom.Geometry;
import org.slf4j.Logger;
import java.sql.Types;
-import java.util.List;
import java.util.Map;
/**
@@ -182,14 +184,18 @@ public void readCountry(String countryCode, ImportThread importThread) {
* not will create them. This may take a while.
*/
public void prepareDatabase() {
- Integer indexRowNum = template.queryForObject(
- "SELECT count(*) FROM pg_indexes WHERE tablename = 'placex' AND indexdef LIKE '%(country_code)'",
- Integer.class);
-
- if (indexRowNum == null || indexRowNum == 0) {
- LOGGER.info("Creating index over countries.");
- template.execute("CREATE INDEX ON placex (country_code)");
- }
+ txTemplate.execute(status -> {
+ Integer indexRowNum = template.queryForObject(
+ "SELECT count(*) FROM pg_indexes WHERE tablename = 'placex' AND indexdef LIKE '%(country_code)'",
+ Integer.class);
+
+ if (indexRowNum == null || indexRowNum == 0) {
+ LOGGER.info("Creating index over countries.");
+ template.execute("CREATE INDEX ON placex (country_code)");
+ }
+
+ return 0;
+ });
}
public String[] getCountriesFromDatabase() {
diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
index 0b2c6186..d1afabfc 100644
--- a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
+++ b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
@@ -5,6 +5,8 @@
import de.komoot.photon.nominatim.model.*;
import org.locationtech.jts.geom.Geometry;
import org.springframework.jdbc.core.RowMapper;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionCallbackWithoutResult;
import java.util.*;
import java.util.concurrent.locks.ReentrantLock;
@@ -138,8 +140,13 @@ public void setUpdater(Updater updater) {
public void initUpdates(String updateUser) {
LOGGER.info("Creating tracking tables");
- template.execute(TRIGGER_SQL);
- template.execute("GRANT SELECT, DELETE ON photon_updates TO \"" + updateUser + '"');
+ txTemplate.execute(new TransactionCallbackWithoutResult() {
+ @Override
+ protected void doInTransactionWithoutResult(TransactionStatus status) {
+ template.execute(TRIGGER_SQL);
+ template.execute("GRANT SELECT, DELETE ON photon_updates TO \"" + updateUser + '"');
+ }
+ });
}
public void update() {
@@ -230,28 +237,30 @@ private void updateFromInterpolations() {
}
private List<UpdateRow> getPlaces(String table) {
- List<UpdateRow> results = template.query(dbutils.deleteReturning(
- "DELETE FROM photon_updates WHERE rel = ?", "place_id, operation, indexed_date"),
- (rs, rowNum) -> {
- boolean isDelete = "DELETE".equals(rs.getString("operation"));
- return new UpdateRow(rs.getLong("place_id"), isDelete, rs.getTimestamp("indexed_date"));
- }, table);
-
- // For each place only keep the newest item.
- // Order doesn't really matter because updates of each place are independent now.
- results.sort(Comparator.comparing(UpdateRow::getPlaceId).thenComparing(
- Comparator.comparing(UpdateRow::getUpdateDate).reversed()));
-
- ArrayList<UpdateRow> todo = new ArrayList<>();
- long prevId = -1;
- for (UpdateRow row: results) {
- if (row.getPlaceId() != prevId) {
- prevId = row.getPlaceId();
- todo.add(row);
+ return txTemplate.execute(status -> {
+ List<UpdateRow> results = template.query(dbutils.deleteReturning(
+ "DELETE FROM photon_updates WHERE rel = ?", "place_id, operation, indexed_date"),
+ (rs, rowNum) -> {
+ boolean isDelete = "DELETE".equals(rs.getString("operation"));
+ return new UpdateRow(rs.getLong("place_id"), isDelete, rs.getTimestamp("indexed_date"));
+ }, table);
+
+ // For each place only keep the newest item.
+ // Order doesn't really matter because updates of each place are independent now.
+ results.sort(Comparator.comparing(UpdateRow::getPlaceId).thenComparing(
+ Comparator.comparing(UpdateRow::getUpdateDate).reversed()));
+
+ ArrayList<UpdateRow> todo = new ArrayList<>();
+ long prevId = -1;
+ for (UpdateRow row : results) {
+ if (row.getPlaceId() != prevId) {
+ prevId = row.getPlaceId();
+ todo.add(row);
+ }
}
- }
- return todo;
+ return todo;
+ });
}
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
index 4830b5f6..f2bd4ddf 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java
@@ -16,15 +16,18 @@
import java.util.Date;
import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
+import org.springframework.transaction.support.TransactionTemplate;
class NominatimConnectorDBTest {
private EmbeddedDatabase db;
private NominatimImporter connector;
private CollectingImporter importer;
private JdbcTemplate jdbc;
+ private TransactionTemplate txTemplate;
@BeforeEach
void setup() {
@@ -39,7 +42,9 @@ void setup() {
importer = new CollectingImporter();
jdbc = new JdbcTemplate(db);
+ txTemplate = new TransactionTemplate(new DataSourceTransactionManager(db));
ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "template", jdbc);
+ ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "txTemplate", txTemplate);
}
private void readEntireDatabase() {
diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java
index acab94cb..6c30218f 100644
--- a/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java
+++ b/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java
@@ -5,9 +5,11 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
+import org.springframework.transaction.support.TransactionTemplate;
import static org.junit.jupiter.api.Assertions.*;
@@ -16,6 +18,7 @@ class NominatimUpdaterDBTest {
private NominatimUpdater connector;
private CollectingUpdater updater;
private JdbcTemplate jdbc;
+ private TransactionTemplate txTemplate;
@BeforeEach
void setup() {
@@ -31,7 +34,9 @@ void setup() {
connector.setUpdater(updater);
jdbc = new JdbcTemplate(db);
+ txTemplate = new TransactionTemplate(new DataSourceTransactionManager(db));
ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "template", jdbc);
+ ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "txTemplate", txTemplate);
}
@Test
From 6f32d32dd3c5d03d2257b2f2922716ef6e71b8a1 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Tue, 12 Nov 2024 12:13:11 +0100
Subject: [PATCH 14/17] place_ids need a 64-bit value
---
.../photon/nominatim/model/NominatimAddressCache.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
index d9a7e2fe..a1a3a487 100644
--- a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
+++ b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
@@ -21,12 +21,12 @@ public class NominatimAddressCache {
"SELECT place_id, name, class, type, rank_address FROM placex" +
" WHERE rank_address between 5 and 25 AND linked_place_id is null";
- private final Map<Integer, AddressRow> addresses = new HashMap<>();
+ private final Map<Long, AddressRow> addresses = new HashMap<>();
public void loadCountryAddresses(JdbcTemplate template, DBDataAdapter dbutils, String countryCode) {
final RowCallbackHandler rowMapper = (rs) -> {
addresses.put(
- rs.getInt("place_id"),
+ rs.getLong("place_id"),
new AddressRow(
dbutils.getMap(rs, "name"),
rs.getString("class"),
@@ -52,7 +52,7 @@ public List<AddressRow> getAddressList(String addressline) {
if (addressline != null && !addressline.isBlank()) {
JSONArray addressPlaces = new JSONArray(addressline);
for (int i = 0; i < addressPlaces.length(); ++i) {
- Integer place_id = addressPlaces.optInt(i);
+ Long place_id = addressPlaces.optLong(i);
if (place_id != null) {
AddressRow row = addresses.get(place_id);
if (row != null) {
From b50c3e455006da2a64bde6f8c7a732eb634fb281 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Wed, 13 Nov 2024 10:20:04 +0100
Subject: [PATCH 15/17] do not mutate names from the address cache
---
src/main/java/de/komoot/photon/PhotonDoc.java | 14 ++++++++++++--
.../nominatim/model/NominatimAddressCache.java | 2 +-
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/src/main/java/de/komoot/photon/PhotonDoc.java b/src/main/java/de/komoot/photon/PhotonDoc.java
index 95d22a15..43b90cf4 100644
--- a/src/main/java/de/komoot/photon/PhotonDoc.java
+++ b/src/main/java/de/komoot/photon/PhotonDoc.java
@@ -218,17 +218,27 @@ public boolean isUsefulForIndex() {
private void extractAddress(Map<String, String> address, AddressType addressType, String addressFieldName) {
String field = address.get(addressFieldName);
- if (field != null) {
- Map<String, String> map = addressParts.computeIfAbsent(addressType, k -> new HashMap<>());
+ if (field == null) {
+ return;
+ }
+ Map<String, String> map = addressParts.get(addressType);
+ if (map == null) {
+ map = new HashMap<>();
+ map.put("name", field);
+ addressParts.put(addressType, map);
+ } else {
String existingName = map.get("name");
if (!field.equals(existingName)) {
+ // Make a copy of the original name map because the map is reused for other addresses.
+ map = new HashMap<>(map);
LOGGER.debug("Replacing {} name '{}' with '{}' for osmId #{}", addressFieldName, existingName, field, osmId);
// we keep the former name in the context as it might be helpful when looking up typos
if (!Objects.isNull(existingName)) {
context.add(Collections.singletonMap("formerName", existingName));
}
map.put("name", field);
+ addressParts.put(addressType, map);
}
}
}
diff --git a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
index a1a3a487..e257ae07 100644
--- a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
+++ b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
@@ -28,7 +28,7 @@ public void loadCountryAddresses(JdbcTemplate template, DBDataAdapter dbutils, S
addresses.put(
rs.getLong("place_id"),
new AddressRow(
- dbutils.getMap(rs, "name"),
+ Map.copyOf(dbutils.getMap(rs, "name")),
rs.getString("class"),
rs.getString("type"),
rs.getInt("rank_address")
From b32c234d76be7a5621ca12684057d80756ec85b6 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Thu, 14 Nov 2024 09:24:37 +0100
Subject: [PATCH 16/17] clean style issues
---
.../photon/opensearch/OpenSearchResult.java | 1 -
.../model/NominatimAddressCache.java | 26 +++++++++----------
2 files changed, 13 insertions(+), 14 deletions(-)
diff --git a/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java b/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java
index a6147fc6..3cbf9506 100644
--- a/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java
+++ b/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java
@@ -1,7 +1,6 @@
package de.komoot.photon.opensearch;
import de.komoot.photon.searcher.PhotonResult;
-import jakarta.json.JsonArray;
import org.json.JSONObject;
import java.util.Map;
diff --git a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
index e257ae07..bba02732 100644
--- a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
+++ b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
@@ -24,16 +24,16 @@ public class NominatimAddressCache {
private final Map<Long, AddressRow> addresses = new HashMap<>();
public void loadCountryAddresses(JdbcTemplate template, DBDataAdapter dbutils, String countryCode) {
- final RowCallbackHandler rowMapper = (rs) -> {
- addresses.put(
- rs.getLong("place_id"),
- new AddressRow(
- Map.copyOf(dbutils.getMap(rs, "name")),
- rs.getString("class"),
- rs.getString("type"),
- rs.getInt("rank_address")
- ));
- };
+ final RowCallbackHandler rowMapper = rs ->
+ addresses.put(
+ rs.getLong("place_id"),
+ new AddressRow(
+ Map.copyOf(dbutils.getMap(rs, "name")),
+ rs.getString("class"),
+ rs.getString("type"),
+ rs.getInt("rank_address")
+ ));
+
if (countryCode == null) {
template.query(BASE_COUNTRY_QUERY + " AND country_code is null", rowMapper);
@@ -52,9 +52,9 @@ public List<AddressRow> getAddressList(String addressline) {
if (addressline != null && !addressline.isBlank()) {
JSONArray addressPlaces = new JSONArray(addressline);
for (int i = 0; i < addressPlaces.length(); ++i) {
- Long place_id = addressPlaces.optLong(i);
- if (place_id != null) {
- AddressRow row = addresses.get(place_id);
+ Long placeId = addressPlaces.optLong(i);
+ if (placeId != null) {
+ AddressRow row = addresses.get(placeId);
if (row != null) {
outlist.add(row);
}
From 23c18efdc286354bcc10ad02fd322df580f9c010 Mon Sep 17 00:00:00 2001
From: Sarah Hoffmann <lonvia@denofr.de>
Date: Fri, 15 Nov 2024 14:10:06 +0100
Subject: [PATCH 17/17] fix loading addresses for non-country places
---
.../de/komoot/photon/nominatim/model/NominatimAddressCache.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
index bba02732..67399b6c 100644
--- a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
+++ b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java
@@ -35,7 +35,7 @@ public void loadCountryAddresses(JdbcTemplate template, DBDataAdapter dbutils, S
));
- if (countryCode == null) {
+ if ("".equals(countryCode)) {
template.query(BASE_COUNTRY_QUERY + " AND country_code is null", rowMapper);
} else {
template.query(BASE_COUNTRY_QUERY + " AND country_code = ?", rowMapper, countryCode);