Rework update mechanism to be independent from Nominatim updates

.github/workflows/ci.yml

@@ -59,7 +59,7 @@ jobs:
         run: echo "cache_key=$(/bin/date -u "+%Y%W")" >> $GITHUB_ENV
         shell: bash
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
             path: |
                Nominatim/data/country_osm_grid.sql.gz
@@ -117,11 +117,12 @@ jobs:
       - name: Import Photon
         run: |
             java -jar target/photon-*.jar -nominatim-import -database nominatim -user runner -password foobar
+            java -jar target/photon-*.jar -nominatim-update-init-for runner -database nominatim -user runner -password foobar
 
       - name: Update Nominatim
         run: |
           nominatim replication --init
-          nominatim replication --no-index --once
+          nominatim replication --once
         shell: bash
         working-directory: data-env
         env:

README.md

@@ -142,25 +142,56 @@ The import of worldwide data set will take some hours/days, SSD/NVME disks are r
 
 #### Updating from OSM via Nominatim
 
-In order to update nominatim from OSM and then photon from nominatim, you must start photon with the nominatim database credentials on the command line:
+To update an existing Photon database from Nominatim, first prepare the
+Nominatim database with the appropriate triggers:
 
 ```bash
-java -jar photon-*.jar -host localhost -port 5432 -database nominatim -user nominatim -password ...
+java -jar photon-*.jar -database nominatim -user nominatim -password ... -nominatim-update-init-for update_user
 ```
 
-A nominatim setup is also a requirement to have continuous updates. To keep nominatim in sync with the latest OSM changes and to update photon with nominatim afterwards run:
+This script must be run with a user that has the right to create tables,
+functions and triggers.
+
+'update-user' is the PostgreSQL user that will be used when updating the
+Photon database. The user needs read rights on the database. The necessary
+update rights will be granted during initialisation.
+
+Now you can run updates on Nominatim using the usual methods as described
+in the [documentation](https://nominatim.org/release-docs/latest/admin/Update/).
+To bring the Photon database up-to-date, stop the Nominatim updates and
+then run the Photon update process:
 
 ```bash
-export NOMINATIM_DIR=/home/nominatim/...
-./continuously_update_from_nominatim.sh
+java -jar photon-*.jar -database nominatim -user nominatim -password ... -nominatim-update
+```
+
+You can also run the photon process with the update API enabled:
+
+```bash
+java -jar photon-*.jar -enable-update-api -database nominatim -user nominatim -password ...
 ```
 
-If you have updated nominatim with another method, photon can be updated by making a HTTP GET request to `/nominatim-update`, e.g. with this command:
+Then you can trigger updates like this:
 
 ```bash
 curl http://localhost:2322/nominatim-update
 ```
 
+For your convenience, this repository contains a script to continuously update
+both Nominatim and Photon. To run it, first customize some environment
+variables according to your installation:
+
+```bash
+export NOMINATIM_DIR=/srv/nominatim/...
+export PHOTON_JAR=photon.jar
+export PHOTON_DB_NAME=nominatim
+export PHOTON_DB_USER=nominatim
+export PHOTON_DB_PASSWORD=...
+./continuously_update_from_nominatim.sh
+```
+
+If you have Nominatim < 3.7, please read the comments in the script carefully.
+
 ### Search API
 
 #### Search

continuously_update_from_nominatim.sh

@@ -2,7 +2,16 @@
 
 # For Nominatim < 3.7 set this to the Nominatim build directory.
 # For newer versions, this must be the project directory of your import.
-NOMINATIM_DIR=
+: ${NOMINATIM_DIR:=.}
+# Path to Photon Jar file
+: ${PHOTON_JAR:=photon.jar}
+# Name of Nominatim database.
+: ${PHOTON_DB_NAME:=nominatim}
+# PostgreSQL user name to use for update in Photon
+: ${PHOTON_DB_USER:=nominatim}
+# Password for PostgreSQL user
+: ${PHOTON_DB_PASSWORD:=}
+
 
 while true
 do
@@ -12,14 +21,13 @@ do
     # The important part here is to leave out the indexing step. This
     # will be handled by Photon.
 
-    cd $NOMINATIM_DIR
     # For Nominatim versions < 3.7 use the following:
-    # ./utils/update.php --import-osmosis --no-index
-    nominatim replication --once --no-index
+    # ./utils/update.php --import-osmosis
+    nominatim replication --project-dir $NOMINATIM_DIR  --once
 
     # Now tell Photon to finish the updates and copy the new data into its
     # own database.
-    curl http://localhost:2322/nominatim-update
+    java -jar $PHOTON_JAR -database $PHOTON_DB_NAME -user $PHOTON_DB_USER -password $PHOTON_DB_PASSWORD -nominatim-update
 
     # Sleep a bit if updates take less than a minute.
     # If you consume hourly or daily diffs adapt the period accordingly.

src/main/java/de/komoot/photon/App.java

@@ -29,6 +29,11 @@ public static void main(String[] rawArgs) throws Exception {
             return;
         }
 
+        if (args.getNominatimUpdateInit() != null) {
+            startNominatimUpdateInit(args);
+            return;
+        }
+
         boolean shutdownES = false;
         final Server esServer = new Server(args.getDataDirectory()).start(args.getCluster(), args.getTransportAddresses());
         try {
@@ -126,9 +131,15 @@ private static void startNominatimImport(CommandLineArgs args, Server esServer)
         log.info("imported data from nominatim to photon with languages: " + String.join(",", dbProperties.getLanguages()));
     }
 
-    /**
-     * Prepare Nominatim updater.
-     */
+    private static void startNominatimUpdateInit(CommandLineArgs args) {
+        NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
+        nominatimUpdater.initUpdates(args.getNominatimUpdateInit());
+    }
+
+
+        /**
+         * Prepare Nominatim updater.
+         */
     private static NominatimUpdater setupNominatimUpdater(CommandLineArgs args, Server server) {
         // Get database properties and ensure that the version is compatible.
         DatabaseProperties dbProperties = new DatabaseProperties();

src/main/java/de/komoot/photon/CommandLineArgs.java

@@ -23,6 +23,9 @@ public class CommandLineArgs {
     @Parameter(names = "-nominatim-import", description = "import nominatim database into photon (this will delete previous index)")
     private boolean nominatimImport = false;
 
+    @Parameter(names = "-nominatim-update-init-for", description = "set up tracking of updates in the Nominatim database for the given user and exit")
+    private String nominatimUpdateInit = null;
+
     @Parameter(names = "-nominatim-update", description = "fetch updates from nominatim database into photon and exit (this updates the index only without offering an API)")
     private boolean nominatimUpdate = false;
 
@@ -60,7 +63,7 @@ public class CommandLineArgs {
     private String user = "nominatim";
 
     @Parameter(names = "-password", description = "postgres password (default '')")
-    private String password = "";
+    private String password = null;
 
     @Parameter(names = "-data-dir", description = "data directory (default '.')")
     private String dataDirectory = new File(".").getAbsolutePath();

src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java

@@ -25,4 +25,9 @@ public interface DBDataAdapter {
      * Check if a table has the given column.
      */
     boolean hasColumn(JdbcTemplate template, String table, String column);
+
+    /**
+     * Wrap a DELETE statement with a RETURNING clause.
+     */
+    String deleteReturning(String deleteSQL, String columns);
 }

src/main/java/de/komoot/photon/nominatim/NominatimConnector.java

@@ -13,10 +13,7 @@
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 /**
  * Export nominatim data
@@ -38,6 +35,7 @@ public class NominatimConnector {
      */
     private final RowMapper<NominatimResult> osmlineRowMapper;
     private final String selectOsmlineSql;
+    private Importer importer;
 
 
     /**
@@ -75,7 +73,6 @@ public NominatimResult mapRow(ResultSet rs, int rowNum) throws SQLException {
             return result;
         }
     };
-    private Importer importer;
 
     /**
      * @param host     database host
@@ -150,7 +147,9 @@ static BasicDataSource buildDataSource(String host, int port, String database, S
 
         dataSource.setUrl(String.format("jdbc:postgres_jts://%s:%d/%s", host, port, database));
         dataSource.setUsername(username);
-        dataSource.setPassword(password);
+        if (password != null) {
+            dataSource.setPassword(password);
+        }
         dataSource.setDriverClassName(JtsWrapper.class.getCanonicalName());
         dataSource.setDefaultAutoCommit(autocommit);
         return dataSource;
@@ -172,18 +171,22 @@ public void setImporter(Importer importer) {
     }
 
     public List<PhotonDoc> getByPlaceId(long placeId) {
-        NominatimResult result = template.queryForObject(SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ?",
+        List<NominatimResult> result = template.query(SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ?",
                                                          placeRowMapper, placeId);
-        assert(result != null);
-        return result.getDocsWithHousenumber();
+        if (result.size() == 0)
+            return null;
+
+        return result.get(0).getDocsWithHousenumber();
     }
 
     public List<PhotonDoc> getInterpolationsByPlaceId(long placeId) {
-        NominatimResult result = template.queryForObject(selectOsmlineSql
+        List<NominatimResult> result = template.query(selectOsmlineSql
                                                           + " FROM location_property_osmline WHERE place_id = ?",
                                                           osmlineRowMapper, placeId);
-        assert(result != null);
-        return result.getDocsWithHousenumber();
+        if (result.size() == 0)
+            return null;
+
+        return result.get(0).getDocsWithHousenumber();
     }
 
     private long parentPlaceId = -1;
@@ -313,4 +316,8 @@ private void completePlace(PhotonDoc doc) {
             }
         }
     }
+
+    public DBDataAdapter getDataAdaptor() {
+        return dbutils;
+    }
 }