Skip to content

Commit 3bac9aa

Browse files
lonvialonvia
authored
Merge pull request #844 from lonvia/decouple-db-version
Decouple database versions for ES and OS implementation
2 parents f6e3025 + 5779890 commit 3bac9aa

File tree

10 files changed

+75
-98
lines changed

10 files changed

+75
-98
lines changed

‎app/es_embedded/src/main/java/de/komoot/photon/Server.java

+23-13
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,17 @@
3737
* Helper class to start/stop ElasticSearch node and get ElasticSearch clients.
3838
*/
3939
public class Server {
40+
/**
41+
* Database version created by new imports with the current code.
42+
*
43+
* Format must be: major.minor.patch-dev
44+
*
45+
* Increase to next to be released version when the database layout
46+
* changes in an incompatible way. If it is already at the next released
47+
* version, increase the dev version.
48+
*/
49+
public static final String DATABASE_VERSION = "0.3.6-1";
50+
4051
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(Server.class);
4152

4253
public static final String PROPERTY_DOCUMENT_ID = "DATABASE_PROPERTIES";
@@ -173,9 +184,7 @@ public DatabaseProperties recreateIndex(String[] languages, Date importDate, boo
173184

174185
createAndPutIndexMapping(languages, supportStructuredQueries);
175186

176-
DatabaseProperties dbProperties = new DatabaseProperties()
177-
.setLanguages(languages)
178-
.setImportDate(importDate);
187+
DatabaseProperties dbProperties = new DatabaseProperties(languages, importDate, false);
179188
saveToDatabase(dbProperties);
180189

181190
return dbProperties;
@@ -195,8 +204,7 @@ public void updateIndexSettings(String synonymFile) throws IOException {
195204
// Load the settings from the database to make sure it is at the right
196205
// version. If the version is wrong, we should not be messing with the
197206
// index.
198-
DatabaseProperties dbProperties = new DatabaseProperties();
199-
loadFromDatabase(dbProperties);
207+
DatabaseProperties dbProperties = loadFromDatabase();
200208

201209
loadIndexSettings().setSynonymFile(synonymFile).updateIndex(esClient, PhotonIndex.NAME);
202210

@@ -228,7 +236,7 @@ private void deleteIndex() {
228236
*/
229237
public void saveToDatabase(DatabaseProperties dbProperties) throws IOException {
230238
final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(BASE_FIELD)
231-
.field(FIELD_VERSION, DatabaseProperties.DATABASE_VERSION)
239+
.field(FIELD_VERSION, DATABASE_VERSION)
232240
.field(FIELD_LANGUAGES, String.join(",", dbProperties.getLanguages()))
233241
.field(FIELD_IMPORT_DATE, dbProperties.getImportDate() instanceof Date ? dbProperties.getImportDate().toInstant() : null)
234242
.endObject().endObject();
@@ -246,12 +254,12 @@ public void saveToDatabase(DatabaseProperties dbProperties) throws IOException
246254
* Currently does nothing when the property entry is missing. Later versions with a higher
247255
* database version will then fail.
248256
*/
249-
public void loadFromDatabase(DatabaseProperties dbProperties) {
257+
public DatabaseProperties loadFromDatabase() {
250258
GetResponse response = esClient.prepareGet(PhotonIndex.NAME, PhotonIndex.TYPE, PROPERTY_DOCUMENT_ID).execute().actionGet();
251259

252260
// We are currently at the database version where versioning was introduced.
253261
if (!response.isExists()) {
254-
return;
262+
throw new UsageException("Cannot find database properties. Your database version is too old. Please reimport.");
255263
}
256264

257265
Map<String, String> properties = (Map<String, String>) response.getSource().get(BASE_FIELD);
@@ -261,16 +269,18 @@ public void loadFromDatabase(DatabaseProperties dbProperties) {
261269
}
262270

263271
String version = properties.getOrDefault(FIELD_VERSION, "");
264-
if (!DatabaseProperties.DATABASE_VERSION.equals(version)) {
265-
LOGGER.error("Database has incompatible version '{}'. Expected: {}", version, DatabaseProperties.DATABASE_VERSION);
272+
if (!DATABASE_VERSION.equals(version)) {
273+
LOGGER.error("Database has incompatible version '{}'. Expected: {}",
274+
version, DATABASE_VERSION);
266275
throw new UsageException("Incompatible database.");
267276
}
268277

269278
String langString = properties.get(FIELD_LANGUAGES);
270-
dbProperties.setLanguages(langString == null ? null : langString.split(","));
271-
272279
String importDateString = properties.get(FIELD_IMPORT_DATE);
273-
dbProperties.setImportDate(importDateString == null ? null : Date.from(Instant.parse(importDateString)));
280+
281+
return new DatabaseProperties(langString == null ? null : langString.split(","),
282+
importDateString == null ? null : Date.from(Instant.parse(importDateString)),
283+
false);
274284
}
275285

276286
public Importer createImporter(String[] languages, String[] extraTags) {

‎app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java

+2-5
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,11 @@ class ServerTest extends ESBaseTester {
1515
void testSaveAndLoadFromDatabase() throws IOException {
1616
setUpES();
1717

18-
DatabaseProperties prop = new DatabaseProperties();
19-
prop.setLanguages(new String[]{"en", "de", "fr"});
2018
Date now = new Date();
21-
prop.setImportDate(now);
19+
DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, now, false);
2220
getServer().saveToDatabase(prop);
2321

24-
prop = new DatabaseProperties();
25-
getServer().loadFromDatabase(prop);
22+
prop = getServer().loadFromDatabase();
2623

2724
assertArrayEquals(new String[]{"en", "de", "fr"}, prop.getLanguages());
2825
assertEquals(now, prop.getImportDate());

‎app/opensearch/src/main/java/de/komoot/photon/Server.java

+20-13
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,17 @@
1919
import java.util.Date;
2020

2121
public class Server {
22+
/**
23+
* Database version created by new imports with the current code.
24+
*
25+
* Format must be: major.minor.patch-dev
26+
*
27+
* Increase to next to be released version when the database layout
28+
* changes in an incompatible way. If it is already at the next released
29+
* version, increase the dev version.
30+
*/
31+
public static final String DATABASE_VERSION = "0.3.6-1";
32+
2233
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(Server.class);
2334

2435
protected OpenSearchClient client;
@@ -108,18 +119,14 @@ public DatabaseProperties recreateIndex(String[] languages, Date importDate, boo
108119

109120
(new IndexMapping(supportStructuredQueries)).addLanguages(languages).putMapping(client, PhotonIndex.NAME);
110121

111-
var dbProperties = new DatabaseProperties()
112-
.setLanguages(languages)
113-
.setSupportStructuredQueries(supportStructuredQueries)
114-
.setImportDate(importDate);
122+
var dbProperties = new DatabaseProperties(languages, importDate, supportStructuredQueries);
115123
saveToDatabase(dbProperties);
116124

117125
return dbProperties;
118126
}
119127

120128
public void updateIndexSettings(String synonymFile) throws IOException {
121-
var dbProperties = new DatabaseProperties();
122-
loadFromDatabase(dbProperties);
129+
var dbProperties = loadFromDatabase();
123130

124131
(new IndexSettingBuilder()).setSynonymFile(synonymFile).updateIndex(client, PhotonIndex.NAME);
125132

@@ -134,11 +141,11 @@ public void saveToDatabase(DatabaseProperties dbProperties) throws IOException {
134141
client.index(r -> r
135142
.index(PhotonIndex.NAME)
136143
.id(PhotonIndex.PROPERTY_DOCUMENT_ID)
137-
.document(new DBPropertyEntry(dbProperties))
144+
.document(new DBPropertyEntry(dbProperties, DATABASE_VERSION))
138145
);
139146
}
140147

141-
public void loadFromDatabase(DatabaseProperties dbProperties) throws IOException {
148+
public DatabaseProperties loadFromDatabase() throws IOException {
142149
var dbEntry = client.get(r -> r
143150
.index(PhotonIndex.NAME)
144151
.id(PhotonIndex.PROPERTY_DOCUMENT_ID),
@@ -148,15 +155,15 @@ public void loadFromDatabase(DatabaseProperties dbProperties) throws IOException
148155
throw new UsageException("Cannot access property record. Database too old?");
149156
}
150157

151-
if (!DatabaseProperties.DATABASE_VERSION.equals(dbEntry.source().databaseVersion)) {
158+
if (!DATABASE_VERSION.equals(dbEntry.source().databaseVersion)) {
152159
LOGGER.error("Database has incompatible version '{}'. Expected: {}",
153-
dbEntry.source().databaseVersion, DatabaseProperties.DATABASE_VERSION);
160+
dbEntry.source().databaseVersion, DATABASE_VERSION);
154161
throw new UsageException("Incompatible database.");
155162
}
156163

157-
dbProperties.setLanguages(dbEntry.source().languages);
158-
dbProperties.setImportDate(dbEntry.source().importDate);
159-
dbProperties.setSupportStructuredQueries(dbEntry.source().supportStructuredQueries);
164+
return new DatabaseProperties(dbEntry.source().languages,
165+
dbEntry.source().importDate,
166+
dbEntry.source().supportStructuredQueries);
160167
}
161168

162169
public Importer createImporter(String[] languages, String[] extraTags) {

‎app/opensearch/src/main/java/de/komoot/photon/opensearch/DBPropertyEntry.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ public class DBPropertyEntry {
1212

1313
public DBPropertyEntry() {}
1414

15-
public DBPropertyEntry(DatabaseProperties props) {
16-
databaseVersion = DatabaseProperties.DATABASE_VERSION;
15+
public DBPropertyEntry(DatabaseProperties props, String databaseVersion) {
16+
this.databaseVersion = databaseVersion;
1717
importDate = props.getImportDate();
1818
languages = props.getLanguages();
1919
supportStructuredQueries = props.getSupportStructuredQueries();

‎app/opensearch/src/test/java/de/komoot/photon/ServerTest.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,13 @@ class ServerTest extends ESBaseTester {
1313
void testSaveAndLoadFromDatabase() throws IOException {
1414
setUpES();
1515

16-
DatabaseProperties prop = new DatabaseProperties();
17-
prop.setLanguages(new String[]{"en", "de", "fr"});
1816
Date now = new Date();
19-
prop.setImportDate(now);
17+
DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"},
18+
now,
19+
false);
2020
getServer().saveToDatabase(prop);
2121

22-
prop = new DatabaseProperties();
23-
getServer().loadFromDatabase(prop);
22+
prop = getServer().loadFromDatabase();
2423

2524
assertArrayEquals(new String[]{"en", "de", "fr"}, prop.getLanguages());
2625
assertEquals(now, prop.getImportDate());

‎src/main/java/de/komoot/photon/App.java

+3-6
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,8 @@ private static void startNominatimUpdateInit(CommandLineArgs args) {
145145
private static void startNominatimUpdate(NominatimUpdater nominatimUpdater, Server esServer) {
146146
nominatimUpdater.update();
147147

148-
DatabaseProperties dbProperties = new DatabaseProperties();
149148
try {
150-
esServer.loadFromDatabase(dbProperties);
149+
DatabaseProperties dbProperties = esServer.loadFromDatabase();
151150
Date importDate = nominatimUpdater.getLastImportDate();
152151
dbProperties.setImportDate(importDate);
153152
esServer.saveToDatabase(dbProperties);
@@ -162,8 +161,7 @@ private static void startNominatimUpdate(NominatimUpdater nominatimUpdater, Serv
162161
*/
163162
private static NominatimUpdater setupNominatimUpdater(CommandLineArgs args, Server server)throws IOException {
164163
// Get database properties and ensure that the version is compatible.
165-
DatabaseProperties dbProperties = new DatabaseProperties();
166-
server.loadFromDatabase(dbProperties);
164+
DatabaseProperties dbProperties = server.loadFromDatabase();
167165

168166
NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
169167
nominatimUpdater.setUpdater(server.createUpdater(dbProperties.getLanguages(), args.getExtraTags()));
@@ -175,8 +173,7 @@ private static NominatimUpdater setupNominatimUpdater(CommandLineArgs args, Serv
175173
*/
176174
private static void startApi(CommandLineArgs args, Server server) throws IOException {
177175
// Get database properties and ensure that the version is compatible.
178-
DatabaseProperties dbProperties = new DatabaseProperties();
179-
server.loadFromDatabase(dbProperties);
176+
DatabaseProperties dbProperties = server.loadFromDatabase();
180177
if (args.getLanguages(false).length > 0) {
181178
dbProperties.restrictLanguages(args.getLanguages());
182179
}

‎src/main/java/de/komoot/photon/DatabaseProperties.java

+15-40
Original file line numberDiff line numberDiff line change
@@ -8,29 +8,21 @@
88
* The server is responsible for making the data persistent in the Photon database.
99
*/
1010
public class DatabaseProperties {
11-
/**
12-
* Database version created by new imports with the current code.
13-
*
14-
* Format must be: major.minor.patch-dev
15-
*
16-
* Increase to next to be released version when the database layout
17-
* changes in an incompatible way. If it is already at the next released
18-
* version, increase the dev version.
19-
*/
20-
public static final String DATABASE_VERSION = "0.3.6-1";
21-
22-
private String[] languages = null;
23-
24-
/**
25-
* The OSM data date
26-
*/
11+
private String[] languages;
2712
private Date importDate;
13+
private final boolean supportStructuredQueries;
2814

29-
private boolean supportStructuredQueries;
15+
public DatabaseProperties(String[] languages, Date importDate, boolean supportStructuredQueries) {
16+
this.languages = languages;
17+
this.importDate = importDate;
18+
this.supportStructuredQueries = supportStructuredQueries;
19+
}
3020

3121
/**
3222
* Return the list of languages for which the database is configured.
33-
* @return
23+
* If no list was set, then the default is returned.
24+
*
25+
* @return List of supported languages.
3426
*/
3527
public String[] getLanguages() {
3628
if (languages == null) {
@@ -40,18 +32,6 @@ public String[] getLanguages() {
4032
return languages;
4133
}
4234

43-
/**
44-
* Replace the language list with the given list.
45-
*
46-
* @param languages Array of two-letter language codes.
47-
*
48-
* @return This object for function chaining.
49-
*/
50-
public DatabaseProperties setLanguages(String[] languages) {
51-
this.languages = languages;
52-
return this;
53-
}
54-
5535
/**
5636
* Set language list to the intersection between the existing list and the given list.
5737
*
@@ -83,21 +63,16 @@ public void restrictLanguages(String[] languageList) {
8363
}
8464
}
8565

86-
public Date getImportDate() {
87-
return this.importDate;
66+
public void setImportDate(Date importDate) {
67+
this.importDate = importDate;
8868
}
8969

90-
public DatabaseProperties setImportDate(Date importDate) {
91-
this.importDate = importDate;
92-
return this;
70+
71+
public Date getImportDate() {
72+
return this.importDate;
9373
}
9474

9575
public boolean getSupportStructuredQueries() {
9676
return supportStructuredQueries;
9777
}
98-
99-
public DatabaseProperties setSupportStructuredQueries(boolean supportStructuredQueries) {
100-
this.supportStructuredQueries = supportStructuredQueries;
101-
return this;
102-
}
10378
}

‎src/main/java/de/komoot/photon/StatusRequestHandler.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ protected StatusRequestHandler(String path, Server server) {
1919

2020
@Override
2121
public String handle(Request request, Response response) throws IOException {
22-
DatabaseProperties dbProperties = new DatabaseProperties();
23-
server.loadFromDatabase(dbProperties);
22+
DatabaseProperties dbProperties = server.loadFromDatabase();
2423
String importDateStr = null;
2524
if (dbProperties.getImportDate() instanceof Date) {
2625
importDateStr = dbProperties.getImportDate().toInstant().toString();

‎src/test/java/de/komoot/photon/ApiIntegrationTest.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -139,8 +139,7 @@ void testApi(String osmValue, String url) throws Exception {
139139
void testApiStatus() throws Exception {
140140
App.main(new String[]{"-cluster", TEST_CLUSTER_NAME, "-listen-port", Integer.toString(LISTEN_PORT), "-transport-addresses", "127.0.0.1"});
141141
awaitInitialization();
142-
DatabaseProperties prop = new DatabaseProperties();
143-
getServer().loadFromDatabase(prop);
142+
DatabaseProperties prop = getServer().loadFromDatabase();
144143
HttpURLConnection connection = (HttpURLConnection) new URL("http://127.0.0.1:" + port() + "/status").openConnection();
145144
JSONObject json = new JSONObject(
146145
new BufferedReader(new InputStreamReader(connection.getInputStream())).lines().collect(Collectors.joining("\n")));

‎src/test/java/de/komoot/photon/DatabasePropertiesTest.java

+4-10
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,19 @@ class DatabasePropertiesTest extends ESBaseTester {
1616
*/
1717
@Test
1818
void testSetLanguages() {
19-
DatabaseProperties prop = new DatabaseProperties();
19+
var now = new Date();
20+
DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "bg", "de"}, now, false);
2021

21-
prop.setLanguages(new String[]{"en", "bg", "de"});
22-
Date now = new Date();
23-
prop.setImportDate(now);
2422
assertArrayEquals(new String[]{"en", "bg", "de"}, prop.getLanguages());
2523
assertEquals(now, prop.getImportDate());
26-
27-
prop.setLanguages(new String[]{"ru"});
28-
assertArrayEquals(new String[]{"ru"}, prop.getLanguages());
2924
}
3025

3126
/**
3227
* If languages is not set, then the restricted language set is used as is.
3328
*/
3429
@Test
3530
void testRestrictLanguagesUnsetLanguages() {
36-
DatabaseProperties prop = new DatabaseProperties();
31+
DatabaseProperties prop = new DatabaseProperties(null, null, false);
3732
prop.restrictLanguages(new String[]{"en", "bg", "de"});
3833

3934
assertArrayEquals(new String[]{"en", "bg", "de"}, prop.getLanguages());
@@ -45,8 +40,7 @@ void testRestrictLanguagesUnsetLanguages() {
4540
*/
4641
@Test
4742
void testRestrictLanguagesAlreadySet() {
48-
DatabaseProperties prop = new DatabaseProperties();
49-
prop.setLanguages(new String[]{"en", "de", "fr"});
43+
DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, null, false);
5044

5145
prop.restrictLanguages(new String[]{"cn", "de", "en", "es"});
5246

0 commit comments

Comments
 (0)