diff --git a/src/main/java/org/neo4j/gis/spatial/AbstractGeometryEncoder.java b/src/main/java/org/neo4j/gis/spatial/AbstractGeometryEncoder.java index 75e6b4ff..ff43f09e 100644 --- a/src/main/java/org/neo4j/gis/spatial/AbstractGeometryEncoder.java +++ b/src/main/java/org/neo4j/gis/spatial/AbstractGeometryEncoder.java @@ -29,11 +29,10 @@ public abstract class AbstractGeometryEncoder implements GeometryEncoder, Constants { protected String bboxProperty = PROP_BBOX; - - // Public methods + protected Layer layer; @Override - public void init(Layer layer) { + public void init(Transaction tx, Layer layer) { this.layer = layer; } @@ -116,8 +115,4 @@ public Object getAttribute(Node geomNode, String name) { public String getSignature() { return "GeometryEncoder(bbox='" + bboxProperty + "')"; } - - // Attributes - - protected Layer layer; } diff --git a/src/main/java/org/neo4j/gis/spatial/DefaultLayer.java b/src/main/java/org/neo4j/gis/spatial/DefaultLayer.java index 4b1b34bf..efaeb690 100644 --- a/src/main/java/org/neo4j/gis/spatial/DefaultLayer.java +++ b/src/main/java/org/neo4j/gis/spatial/DefaultLayer.java @@ -40,6 +40,7 @@ import org.neo4j.gis.spatial.rtree.filter.SearchFilter; import org.neo4j.gis.spatial.utilities.GeotoolsAdapter; import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.Transaction; /** @@ -55,7 +56,12 @@ */ public class DefaultLayer implements Constants, Layer, SpatialDataset { - // Public methods + private String name; + protected String layerNodeId; + private GeometryEncoder geometryEncoder; + private GeometryFactory geometryFactory; + protected LayerIndexReader indexReader; + protected SpatialIndexWriter indexWriter; @Override public String getName() { @@ -231,7 +237,7 @@ public void initialize(Transaction tx, IndexManager indexManager, String name, N } else { this.geometryEncoder = new WKBGeometryEncoder(); } - this.geometryEncoder.init(this); + this.geometryEncoder.init(tx, this); // index must be created *after* geometryEncoder if (layerNode.hasProperty(PROP_INDEX_CLASS)) { @@ -273,6 +279,10 @@ public Node getLayerNode(Transaction tx) { public void delete(Transaction tx, Listener monitor) { indexWriter.removeAll(tx, true, monitor); Node layerNode = getLayerNode(tx); + for (Relationship rel : layerNode.getRelationships()) { + System.out.printf("Unexpected relationship in layer %s: %s%n", getName(), rel); + rel.delete(); + } layerNode.delete(); layerNodeId = null; } @@ -283,15 +293,6 @@ public void delete(Transaction tx, Listener monitor) { // return spatialDatabase.getDatabase(); // } - // Attributes - - //private SpatialDatabaseService spatialDatabase; - private String name; - protected String layerNodeId = null; - private GeometryEncoder geometryEncoder; - private GeometryFactory geometryFactory; - protected LayerIndexReader indexReader; - protected SpatialIndexWriter indexWriter; @Override public SpatialDataset getDataset() { diff --git a/src/main/java/org/neo4j/gis/spatial/GeometryEncoder.java b/src/main/java/org/neo4j/gis/spatial/GeometryEncoder.java index 59acaf6c..a24bd668 100644 --- a/src/main/java/org/neo4j/gis/spatial/GeometryEncoder.java +++ b/src/main/java/org/neo4j/gis/spatial/GeometryEncoder.java @@ -53,10 +53,8 @@ public interface GeometryEncoder extends EnvelopeDecoder { * that represents a layer. This node is expected to have a property containing the class name * of the GeometryEncoder for that layer, and it will be constructed and passed the layer using * this method, allowing the Layer and the GeometryEncoder to interact. - * - * @param layer recently created Layer class */ - void init(Layer layer); + void init(Transaction tx, Layer layer); /** * This method is called to store a bounding box for the geometry to the database. It should write it to the diff --git a/src/main/java/org/neo4j/gis/spatial/SpatialTopologyUtils.java b/src/main/java/org/neo4j/gis/spatial/SpatialTopologyUtils.java index 7403994a..d8ccd4fa 100644 --- a/src/main/java/org/neo4j/gis/spatial/SpatialTopologyUtils.java +++ b/src/main/java/org/neo4j/gis/spatial/SpatialTopologyUtils.java @@ -152,7 +152,7 @@ public static List findClosestEdges(Transaction tx, Point point, La * @see SDO_LRS.LOCATE_PT * @see LengthIndexedLine + * href="https://locationtech.github.io/jts/javadoc/org/locationtech/jts/linearref/LengthIndexedLine.html">LengthIndexedLine */ public static Point locatePoint(Layer layer, Geometry geometry, double measure) { return layer.getGeometryFactory().createPoint(locatePoint(geometry, measure)); @@ -171,7 +171,7 @@ public static Point locatePoint(Layer layer, Geometry geometry, double measure) * @see SDO_LRS.LOCATE_PT * @see LengthIndexedLine + * href="https://locationtech.github.io/jts/javadoc/org/locationtech/jts/linearref/LengthIndexedLine.html">LengthIndexedLine */ public static Coordinate locatePoint(Geometry geometry, double measure) { return new LengthIndexedLine(geometry).extractPoint(measure); @@ -194,7 +194,7 @@ public static Coordinate locatePoint(Geometry geometry, double measure) { * @see SDO_LRS.LOCATE_PT * @see LengthIndexedLine + * href="https://locationtech.github.io/jts/javadoc/org/locationtech/jts/linearref/LengthIndexedLine.html">LengthIndexedLine */ public static Point locatePoint(Layer layer, Geometry geometry, double measure, double offset) { return layer.getGeometryFactory().createPoint(locatePoint(geometry, measure, offset)); @@ -215,7 +215,7 @@ public static Point locatePoint(Layer layer, Geometry geometry, double measure, * @see SDO_LRS.LOCATE_PT * @see LengthIndexedLine + * href="https://locationtech.github.io/jts/javadoc/org/locationtech/jts/linearref/LengthIndexedLine.html">LengthIndexedLine */ public static Coordinate locatePoint(Geometry geometry, double measure, double offset) { return new LengthIndexedLine(geometry).extractPoint(measure, offset); diff --git a/src/main/java/org/neo4j/gis/spatial/merge/MergeUtils.java b/src/main/java/org/neo4j/gis/spatial/merge/MergeUtils.java new file mode 100644 index 00000000..4e02a701 --- /dev/null +++ b/src/main/java/org/neo4j/gis/spatial/merge/MergeUtils.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j Spatial. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gis.spatial.merge; + +import java.util.ArrayList; +import org.locationtech.jts.geom.Geometry; +import org.neo4j.gis.spatial.EditableLayer; +import org.neo4j.gis.spatial.GeometryEncoder; +import org.neo4j.gis.spatial.encoders.Configurable; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Transaction; + +public class MergeUtils { + + public interface Mergeable { + + long mergeFrom(Transaction tx, EditableLayer other); + } + + private static boolean encodersIdentical(EditableLayer layer, EditableLayer mergeLayer) { + GeometryEncoder layerEncoder = layer.getGeometryEncoder(); + GeometryEncoder mergeEncoder = mergeLayer.getGeometryEncoder(); + Class layerGeometryClass = layerEncoder.getClass(); + Class mergeGeometryClass = mergeEncoder.getClass(); + if (layerGeometryClass.isAssignableFrom(mergeGeometryClass)) { + if (mergeEncoder instanceof Configurable && layerEncoder instanceof Configurable) { + String mergeConfig = ((Configurable) mergeEncoder).getConfiguration(); + String layerConfig = ((Configurable) layerEncoder).getConfiguration(); + return mergeConfig.equals(layerConfig); + } else { + // If one is configurable, but not the other, they are not identical + return !(mergeEncoder instanceof Configurable || layerEncoder instanceof Configurable); + } + } + return false; + } + + public static long mergeLayerInto(Transaction tx, EditableLayer layer, EditableLayer mergeLayer) { + long count; + Class layerClass = layer.getClass(); + Class mergeClass = mergeLayer.getClass(); + if (layer instanceof Mergeable) { + count = ((Mergeable) layer).mergeFrom(tx, mergeLayer); + } else if (layerClass.isAssignableFrom(mergeClass)) { + if (encodersIdentical(layer, mergeLayer)) { + // With identical encoders, we can simply add the node as is, but must remove it first + ArrayList toAdd = new ArrayList<>(); + for (Node node : mergeLayer.getIndex().getAllIndexedNodes(tx)) { + toAdd.add(node); + } + for (Node node : toAdd) { + // Remove each from the previous index before adding to the new index, so as not to have multiple incoming RTREE_REFERENCE + mergeLayer.removeFromIndex(tx, node.getElementId()); + layer.add(tx, node); + } + count = toAdd.size(); + } else { + // With differing encoders, we must decode and re-encode each geometry, so we also create new nodes and remove and delete the actual nodes later + ArrayList toRemove = new ArrayList<>(); + GeometryEncoder fromEncoder = mergeLayer.getGeometryEncoder(); + for (Node node : mergeLayer.getIndex().getAllIndexedNodes(tx)) { + toRemove.add(node); + Geometry geometry = fromEncoder.decodeGeometry(node); + layer.add(tx, geometry); + } + for (Node remove : toRemove) { + mergeLayer.removeFromIndex(tx, remove.getElementId()); + remove.delete(); + } + count = toRemove.size(); + } + } else { + throw new IllegalArgumentException(String.format( + "Cannot merge '%s' into '%s': layer classes are not compatible: '%s' cannot be caste as '%s'", + mergeLayer.getName(), layer.getName(), mergeClass.getSimpleName(), layerClass.getSimpleName())); + } + return count; + } +} diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMDataset.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMDataset.java index ceaf53f0..fef5cbf1 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMDataset.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMDataset.java @@ -19,10 +19,14 @@ */ package org.neo4j.gis.spatial.osm; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.Objects; import javax.annotation.Nonnull; +import javax.xml.bind.DatatypeConverter; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -33,9 +37,11 @@ import org.neo4j.gis.spatial.SpatialRelationshipTypes; import org.neo4j.gis.spatial.utilities.RelationshipTraversal; import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.schema.IndexDefinition; import org.neo4j.graphdb.traversal.Evaluation; import org.neo4j.graphdb.traversal.Evaluators; import org.neo4j.graphdb.traversal.TraversalDescription; @@ -46,11 +52,18 @@ public class OSMDataset implements SpatialDataset, Iterator { private final OSMLayer layer; private final String datasetNodeId; private Iterator wayNodeIterator; + private final LabelHasher labelHasher; - public OSMDataset(OSMLayer layer, String datasetNodeId) { + private OSMDataset(OSMLayer layer, String datasetNodeId) { this.layer = layer; this.datasetNodeId = datasetNodeId; this.layer.setDataset(this); + try { + this.labelHasher = new LabelHasher(layer.getName()); + } catch (NoSuchAlgorithmException e) { + throw new SpatialDatabaseException( + "Failed to initialize OSM dataset '" + layer.getName() + "': " + e.getMessage(), e); + } } /** @@ -80,49 +93,30 @@ public static OSMDataset fromLayer(Transaction tx, OSMLayer layer) { Relationship rel = layer.getLayerNode(tx) .getSingleRelationship(SpatialRelationshipTypes.LAYERS, Direction.INCOMING); if (rel == null) { - throw new SpatialDatabaseException("Layer '" + layer + "' does not have an associated dataset"); + Node datasetNode = tx.createNode(OSMModel.LABEL_DATASET); + datasetNode.setProperty("name", layer.getName()); + datasetNode.setProperty("type", "osm"); + datasetNode.createRelationshipTo(layer.getLayerNode(tx), SpatialRelationshipTypes.LAYERS); + return new OSMDataset(layer, datasetNode.getElementId()); } String datasetNodeId = rel.getStartNode().getElementId(); return new OSMDataset(layer, datasetNodeId); } public Iterable getAllUserNodes(Transaction tx) { - TraversalDescription td = new MonoDirectionalTraversalDescription() - .depthFirst() - .relationships(OSMRelation.USERS, Direction.OUTGOING) - .relationships(OSMRelation.OSM_USER, Direction.OUTGOING) - .evaluator(Evaluators.includeWhereLastRelationshipTypeIs(OSMRelation.OSM_USER)); - return td.traverse(tx.getNodeByElementId(datasetNodeId)).nodes(); + return () -> tx.findNodes(labelHasher.getLabelHashed(OSMModel.LABEL_USER)); } public Iterable getAllChangesetNodes(Transaction tx) { - TraversalDescription td = new MonoDirectionalTraversalDescription() - .depthFirst() - .relationships(OSMRelation.USERS, Direction.OUTGOING) - .relationships(OSMRelation.OSM_USER, Direction.OUTGOING) - .relationships(OSMRelation.USER, Direction.INCOMING) - .evaluator(Evaluators.includeWhereLastRelationshipTypeIs(OSMRelation.USER)); - return td.traverse(tx.getNodeByElementId(datasetNodeId)).nodes(); + return () -> tx.findNodes(labelHasher.getLabelHashed(OSMModel.LABEL_CHANGESET)); } public Iterable getAllWayNodes(Transaction tx) { - TraversalDescription td = new MonoDirectionalTraversalDescription() - .depthFirst() - .relationships(OSMRelation.WAYS, Direction.OUTGOING) - .relationships(OSMRelation.NEXT, Direction.OUTGOING) - .evaluator(Evaluators.excludeStartPosition()); - return td.traverse(tx.getNodeByElementId(datasetNodeId)).nodes(); + return () -> tx.findNodes(labelHasher.getLabelHashed(OSMModel.LABEL_WAY)); } public Iterable getAllPointNodes(Transaction tx) { - TraversalDescription td = new MonoDirectionalTraversalDescription() - .depthFirst() - .relationships(OSMRelation.WAYS, Direction.OUTGOING) - .relationships(OSMRelation.NEXT, Direction.OUTGOING) - .relationships(OSMRelation.FIRST_NODE, Direction.OUTGOING) - .relationships(OSMRelation.NODE, Direction.OUTGOING) - .evaluator(Evaluators.includeWhereLastRelationshipTypeIs(OSMRelation.NODE)); - return td.traverse(tx.getNodeByElementId(datasetNodeId)).nodes(); + return () -> tx.findNodes(labelHasher.getLabelHashed(OSMModel.LABEL_NODE)); } public Iterable getWayNodes(Node way) { @@ -154,6 +148,10 @@ public Node getUser(Node nodeWayOrChangeset) { return RelationshipTraversal.getFirstNode(td.traverse(nodeWayOrChangeset).nodes()); } + public Node getDatasetNode(Transaction tx) { + return tx.getNodeByElementId(datasetNodeId); + } + public Way getWayFromId(Transaction tx, String id) { return getWayFrom(tx.getNodeByElementId(id)); } @@ -371,4 +369,66 @@ public int getChangesetCount(Transaction tx) { public int getUserCount(Transaction tx) { return (Integer) tx.getNodeByElementId(this.datasetNodeId).getProperty("userCount", 0); } + + public String getIndexName(Transaction tx, Label label, String propertyKey) { + Node datasetNode = tx.getNodeByElementId(this.datasetNodeId); + String indexKey = indexKeyFor(label, propertyKey); + return (String) datasetNode.getProperty(indexKey, null); + } + + public IndexDefinition getIndex(Transaction tx, Label label, String propertyKey) { + String indexName = getIndexName(tx, label, propertyKey); + if (indexName == null) { + throw new IllegalArgumentException( + String.format("OSM Dataset '%s' does not have an index for label '%s' and property '%s'", + this.layer.getName(), label.name(), propertyKey)); + } else { + return tx.schema().getIndexByName(indexName); + } + } + + public static String indexKeyFor(Label label, String propertyKey) { + return String.format("Index:%s:%s", label.name(), propertyKey); + } + + public static String indexNameFor(String layerName, String hashedLabel, String propertyKey) { + return String.format("OSM-%s-%s-%s", layerName, hashedLabel, propertyKey); + } + + public static Label hashedLabelFrom(String indexName) { + String[] fields = indexName.split("-"); + if (fields.length == 4) { + return Label.label(fields[2]); + } else { + throw new IllegalArgumentException( + String.format("Index name '%s' is not correctly formatted - cannot extract label hash", indexName)); + } + } + + public static class LabelHasher { + + private final String layerHash; + private final HashMap hashedLabels = new HashMap<>(); + + public LabelHasher(String layerName) throws NoSuchAlgorithmException { + this.layerHash = md5Hash(layerName); + } + + public Label getLabelHashed(Label label) { + if (hashedLabels.containsKey(label)) { + return hashedLabels.get(label); + } else { + Label hashed = Label.label(label.name() + "_" + layerHash); + hashedLabels.put(label, hashed); + return hashed; + } + } + + public static String md5Hash(String text) throws NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(text.getBytes()); + byte[] digest = md.digest(); + return DatatypeConverter.printHexBinary(digest).toUpperCase(); + } + } } diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java index 56f3d3f3..a9630083 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMGeometryEncoder.java @@ -19,6 +19,19 @@ */ package org.neo4j.gis.spatial.osm; +import static org.neo4j.gis.spatial.osm.OSMModel.LABEL_CHANGESET; +import static org.neo4j.gis.spatial.osm.OSMModel.LABEL_NODE; +import static org.neo4j.gis.spatial.osm.OSMModel.LABEL_RELATION; +import static org.neo4j.gis.spatial.osm.OSMModel.LABEL_USER; +import static org.neo4j.gis.spatial.osm.OSMModel.LABEL_WAY; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_CHANGESET; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_NODE_ID; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_NODE_LAT; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_NODE_LON; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_RELATION_ID; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_USER_ID; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_USER_NAME; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_WAY_ID; import static org.neo4j.gis.spatial.utilities.TraverserFactory.createTraverserInBackwardsCompatibleWay; import java.io.Serial; @@ -36,11 +49,13 @@ import org.locationtech.jts.geom.LinearRing; import org.locationtech.jts.geom.Polygon; import org.neo4j.gis.spatial.AbstractGeometryEncoder; +import org.neo4j.gis.spatial.Layer; import org.neo4j.gis.spatial.SpatialDatabaseException; import org.neo4j.gis.spatial.SpatialDatabaseService; import org.neo4j.gis.spatial.rtree.Envelope; import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.Entity; +import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Path; import org.neo4j.graphdb.Relationship; @@ -50,14 +65,18 @@ public class OSMGeometryEncoder extends AbstractGeometryEncoder { + public static final String PROP_MAX_FAKE_NODE_ID = "max_fake_node_osm_id"; + public static final String PROP_MAX_FAKE_WAY_ID = "max_fake_way_osm_id"; + public static final String PROP_MAX_FAKE_RELATION_ID = "max_fake_relation_osm_id"; private static int decodedCount = 0; private static int overrunCount = 0; - private static int nodeId = 0; - private static int wayId = 0; - private static int relationId = 0; + private FakeOSMId fake_node_osm_id = new FakeOSMId(PROP_MAX_FAKE_NODE_ID); + private FakeOSMId fake_way_osm_id = new FakeOSMId(PROP_MAX_FAKE_WAY_ID); + private FakeOSMId fake_relation_osm_id = new FakeOSMId(PROP_MAX_FAKE_RELATION_ID); private DateFormat dateTimeFormatter; private int vertices; - private int vertexMistmaches = 0; + private int vertexMismatches = 0; + private final HashMap labelHashes = new HashMap<>(); /** * This class allows for OSM to avoid having empty tags nodes when there are @@ -120,6 +139,46 @@ public void delete() { } } + /** + * This class tracks fake OSM ids. + * Since this encoder can create geometries outside of OpenStreetMap itself, + * it needs to generate fake ids. The convention used here is to start at -1 + * and decrement with each addition. The use of negative numbers ensures that + * we never re-use an ID from real OpenStreetMap. + * To allow server restarts, we also save the current ID on every create, + * and restore the max ID when initializing the layer. + */ + private static class FakeOSMId { + + private final String propertyKey; + private long osm_id; + + private FakeOSMId(String propertyKey) { + this.propertyKey = propertyKey; + this.osm_id = 0; + } + + private void init(Node layerNode) { + if (layerNode != null) { + if (layerNode.hasProperty(propertyKey)) { + osm_id = (Long) layerNode.getProperty(propertyKey); + } + } + } + + private void save(Node layerNode, long osm_id) { + this.osm_id = osm_id; + layerNode.setProperty(propertyKey, osm_id); + } + + private long next(Transaction tx, Layer layer) { + osm_id++; + Node layerNode = layer.getLayerNode(tx); + layerNode.setProperty(propertyKey, osm_id); + return -osm_id; + } + } + public static class OSMGraphException extends SpatialDatabaseException { @Serial @@ -141,11 +200,29 @@ private static Node testIsNode(Entity container) { return (Node) container; } + /** + * Mostly for testing, this method allows pre-seeding the fake OSM-ids with starting values + */ + public void configure(Transaction tx, long nodeId, long wayId, long relId) { + Node layerNode = layer.getLayerNode(tx); + fake_node_osm_id.save(layerNode, nodeId); + fake_way_osm_id.save(layerNode, wayId); + fake_relation_osm_id.save(layerNode, relId); + } + + @Override + public void init(Transaction tx, Layer layer) { + super.init(tx, layer); + Node layerNode = this.layer.getLayerNode(tx); + fake_node_osm_id.init(layerNode); + fake_way_osm_id.init(layerNode); + fake_relation_osm_id.init(layerNode); + } + @Override public Envelope decodeEnvelope(Entity container) { Node geomNode = testIsNode(container); double[] bbox = (double[]) geomNode.getProperty(PROP_BBOX); - // double xmin, double xmax, double ymin, double ymax return new Envelope(bbox[0], bbox[1], bbox[2], bbox[3]); } @@ -155,7 +232,12 @@ public void encodeEnvelope(Envelope mbb, Entity container) { } public static Node getOSMNodeFromGeometryNode(Node geomNode) { - return geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING).getStartNode(); + Relationship rel = geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING); + if (rel != null) { + return rel.getStartNode(); + } else { + throw new IllegalArgumentException("No geom rel"); + } } public static Node getGeometryNodeFromOSMNode(Node osmNode) { @@ -205,17 +287,19 @@ public Geometry decodeGeometry(Entity container) { try { GeometryFactory geomFactory = layer.getGeometryFactory(); Node osmNode = getOSMNodeFromGeometryNode(geomNode); - if (osmNode.hasProperty("node_osm_id")) { - return geomFactory.createPoint(new Coordinate((Double) osmNode.getProperty("lon", 0.0), (Double) osmNode - .getProperty("lat", 0.0))); + if (osmNode.hasProperty(PROP_NODE_ID)) { + return geomFactory.createPoint(new Coordinate( + (Double) osmNode.getProperty(PROP_NODE_LON, 0.0), + (Double) osmNode.getProperty(PROP_NODE_LAT, 0.0))); } - if (osmNode.hasProperty("way_osm_id")) { + if (osmNode.hasProperty(PROP_WAY_ID)) { int vertices = (Integer) geomNode.getProperty("vertices"); int gtype = (Integer) geomNode.getProperty(PROP_TYPE); return decodeGeometryFromWay(osmNode, gtype, vertices, geomFactory); } int gtype = (Integer) geomNode.getProperty(PROP_TYPE); return decodeGeometryFromRelation(osmNode, gtype, geomFactory); + } catch (Exception e) { throw new OSMGraphException("Failed to decode OSM geometry: " + e.getMessage(), e); } @@ -336,7 +420,8 @@ private Geometry decodeGeometryFromWay(Node wayNode, int gtype, int vertices, Ge overrunCount++; break; } - coordinates.add(new Coordinate((Double) node.getProperty("lon"), (Double) node.getProperty("lat"))); + coordinates.add(new Coordinate((Double) node.getProperty(PROP_NODE_LON), + (Double) node.getProperty(OSMModel.PROP_NODE_LAT))); } decodedCount++; if (overrun) { @@ -345,12 +430,12 @@ private Geometry decodeGeometryFromWay(Node wayNode, int gtype, int vertices, Ge + ")"); } if (coordinates.size() != vertices) { - if (vertexMistmaches++ < 10) { + if (vertexMismatches++ < 10) { System.err.println( "Mismatching vertices size for " + SpatialDatabaseService.convertGeometryTypeToName(gtype) + ":" + wayNode + ": " + coordinates.size() + " != " + vertices); - } else if (vertexMistmaches % 100 == 0) { - System.err.println("Mismatching vertices found " + vertexMistmaches + " times"); + } else if (vertexMismatches % 100 == 0) { + System.err.println("Mismatching vertices found " + vertexMismatches + " times"); } } return switch (coordinates.size()) { @@ -363,13 +448,14 @@ yield switch (gtype) { case GTYPE_POLYGON -> geomFactory.createPolygon(geomFactory.createLinearRing(coords), new LinearRing[0]); default -> geomFactory.createMultiPointFromCoords(coords); - }; + } + ; } }; } /** - * For OSM data we can build basic geometry shapes as sub-graphs. This code should produce the same kinds of + * For OSM data we can build basic geometry shapes as sub-graphs. This code should produce the same kinds of* * structures that the utilities in the OSMDataset create. However, those structures are created from original OSM * data, while here we attempt to create equivalent graphs from JTS Geometries. Note that this code is unable to * connect the resulting sub-graph into the OSM data model, since the only node it has is the geometry node. Those @@ -379,57 +465,90 @@ yield switch (gtype) { protected void encodeGeometryShape(Transaction tx, Geometry geometry, Entity container) { Node geomNode = testIsNode(container); vertices = 0; - int gtype = SpatialDatabaseService.convertJtsClassToGeometryType(geometry.getClass()); - switch (gtype) { - case GTYPE_POINT: - makeOSMNode(tx, geometry, geomNode); - break; - case GTYPE_LINESTRING: - case GTYPE_MULTIPOINT: - case GTYPE_POLYGON: - makeOSMWay(tx, geometry, geomNode, gtype); - break; - case GTYPE_MULTILINESTRING: - case GTYPE_MULTIPOLYGON: - int gsubtype = gtype == GTYPE_MULTIPOLYGON ? GTYPE_POLYGON : GTYPE_LINESTRING; - Node relationNode = makeOSMRelation(geometry, geomNode); - int num = geometry.getNumGeometries(); - for (int i = 0; i < num; i++) { - Geometry geom = geometry.getGeometryN(i); - Node wayNode = makeOSMWay(tx, geom, tx.createNode(), gsubtype); - relationNode.createRelationshipTo(wayNode, OSMRelation.MEMBER); - } - break; - default: - throw new SpatialDatabaseException("Unsupported geometry: " + geometry.getClass()); + try { + int gtype = SpatialDatabaseService.convertJtsClassToGeometryType(geometry.getClass()); + switch (gtype) { + case GTYPE_POINT: + makeOSMNode(tx, geometry, geomNode); + break; + case GTYPE_LINESTRING: + case GTYPE_MULTIPOINT: + case GTYPE_POLYGON: + makeOSMWay(tx, geometry, geomNode, gtype); + break; + case GTYPE_MULTILINESTRING: + case GTYPE_MULTIPOLYGON: + int gsubtype = gtype == GTYPE_MULTIPOLYGON ? GTYPE_POLYGON : GTYPE_LINESTRING; + Node relationNode = makeOSMRelation(geometry, geomNode); + int num = geometry.getNumGeometries(); + for (int i = 0; i < num; i++) { + Geometry geom = geometry.getGeometryN(i); + Node wayNode = makeOSMWay(tx, geom, tx.createNode(), gsubtype); + relationNode.createRelationshipTo(wayNode, OSMRelation.MEMBER); + } + break; + default: + throw new SpatialDatabaseException("Unsupported geometry: " + geometry.getClass()); + } + geomNode.setProperty("vertices", vertices); + } catch (Exception e) { + throw new SpatialDatabaseException( + "Failed to encode geometry '" + geometry.getGeometryType() + "': " + e.getMessage(), e); } - geomNode.setProperty("vertices", vertices); } - private Node makeOSMNode(Transaction tx, Geometry geometry, Node geomNode) { + private void makeOSMNode(Transaction tx, Geometry geometry, Node geomNode) { Node node = makeOSMNode(tx, geometry.getCoordinate()); node.createRelationshipTo(geomNode, OSMRelation.GEOM); - return node; + } + + private void addLabelHash(Transaction tx, OSMDataset dataset, Label label, String propertyKey) { + String indexName = dataset.getIndexName(tx, label, propertyKey); + if (indexName != null) { + labelHashes.put(label, OSMDataset.hashedLabelFrom(indexName)); + } + } + + private void loadLabelHash(Transaction tx) { + if (labelHashes.isEmpty()) { + OSMDataset dataset = OSMDataset.fromLayer(tx, (OSMLayer) layer); + addLabelHash(tx, dataset, LABEL_NODE, PROP_NODE_ID); + addLabelHash(tx, dataset, LABEL_WAY, PROP_WAY_ID); + addLabelHash(tx, dataset, LABEL_RELATION, PROP_RELATION_ID); + addLabelHash(tx, dataset, LABEL_USER, PROP_USER_ID); + addLabelHash(tx, dataset, LABEL_CHANGESET, PROP_CHANGESET); + } + } + + private Label getLabelHash(Transaction tx, Label label) { + loadLabelHash(tx); + return labelHashes.get(label); } private Node makeOSMNode(Transaction tx, Coordinate coordinate) { vertices++; - nodeId++; - Node node = tx.createNode(); - // TODO: Generate a valid osm id - node.setProperty(OSMId.NODE.toString(), nodeId); - node.setProperty("lat", coordinate.y); - node.setProperty("lon", coordinate.x); + Node node = tx.createNode(LABEL_NODE); + Label hashed = getLabelHash(tx, LABEL_NODE); + if (hashed != null) { + // This allows this node to be found using the same index that the OSMImporter uses + node.addLabel(hashed); + } + node.setProperty(PROP_NODE_ID, fake_node_osm_id.next(tx, layer)); + node.setProperty(PROP_NODE_LAT, coordinate.y); + node.setProperty(PROP_NODE_LON, coordinate.x); node.setProperty("timestamp", getTimestamp()); // TODO: Add other common properties, like changeset, uid, user, version return node; } private Node makeOSMWay(Transaction tx, Geometry geometry, Node geomNode, int gtype) { - wayId++; - Node way = tx.createNode(); - // TODO: Generate a valid osm id - way.setProperty(OSMId.WAY.toString(), wayId); + Node way = tx.createNode(LABEL_WAY); + Label hashed = getLabelHash(tx, LABEL_WAY); + if (hashed != null) { + // This allows this node to be found using the same index that the OSMImporter uses + way.addLabel(hashed); + } + way.setProperty(PROP_WAY_ID, fake_way_osm_id.next(tx, layer)); way.setProperty("timestamp", getTimestamp()); // TODO: Add other common properties, like changeset, uid, user, // version, name @@ -452,8 +571,8 @@ private Node makeOSMWay(Transaction tx, Geometry geometry, Node geomNode, int gt return way; } + @SuppressWarnings("unused") private Node makeOSMRelation(Geometry geometry, Node geomNode) { - relationId++; throw new SpatialDatabaseException("Unimplemented: makeOSMRelation()"); } @@ -480,10 +599,10 @@ private class CombinedAttributes { properties = node.getSingleRelationship(OSMRelation.TAGS, Direction.OUTGOING).getEndNode(); Node changeset = node.getSingleRelationship(OSMRelation.CHANGESET, Direction.OUTGOING).getEndNode(); if (changeset != null) { - extra.put("changeset", changeset.getProperty("changeset", null)); + extra.put(PROP_CHANGESET, changeset.getProperty(PROP_CHANGESET, null)); Node user = changeset.getSingleRelationship(OSMRelation.USER, Direction.OUTGOING).getEndNode(); if (user != null) { - extra.put("user", user.getProperty("name", null)); + extra.put(PROP_USER_NAME, user.getProperty("name", null)); extra.put("user_id", user.getProperty("uid", null)); } } diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java index 51d5cd4a..81a13908 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMImporter.java @@ -20,6 +20,10 @@ package org.neo4j.gis.spatial.osm; import static java.util.Arrays.asList; +import static org.neo4j.gis.spatial.osm.OSMModel.LABEL_GEOM; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_NODE_LAT; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_NODE_LON; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_TIMESTAMP; import java.io.File; import java.io.FileInputStream; @@ -50,6 +54,7 @@ import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.dbms.api.DatabaseManagementServiceBuilder; import org.neo4j.gis.spatial.Constants; +import org.neo4j.gis.spatial.SpatialDatabaseException; import org.neo4j.gis.spatial.SpatialDatabaseService; import org.neo4j.gis.spatial.index.IndexManager; import org.neo4j.gis.spatial.rtree.Envelope; @@ -95,9 +100,9 @@ public class OSMImporter implements Constants { public static final String PROP_WAY_ID = "way_osm_id"; public static final String PROP_RELATION_ID = "relation_osm_id"; - protected boolean nodesProcessingFinished = false; private final String layerName; - private final StatsManager stats = new StatsManager(); + private final StatsManager tagStats = new StatsManager(); + private final GeomStats geomStats = new GeomStats(); private String osm_dataset = null; private long missingChangesets = 0; private final Listener monitor; @@ -116,15 +121,13 @@ private static class TagStats { this.name = name; } - int add(String key) { + void add(String key) { count++; if (stats.containsKey(key)) { int num = stats.get(key); stats.put(key, ++num); - return num; } stats.put(key, 1); - return 1; } /** @@ -153,7 +156,6 @@ public String toString() { private static class StatsManager { private final HashMap tagStats = new HashMap<>(); - private final HashMap geomStats = new HashMap<>(); TagStats getTagStats(String type) { if (!tagStats.containsKey(type)) { @@ -162,17 +164,15 @@ TagStats getTagStats(String type) { return tagStats.get(type); } - int addToTagStats(String type, String key) { + void addToTagStats(String type, String key) { getTagStats("all").add(key); - return getTagStats(type).add(key); + getTagStats(type).add(key); } - int addToTagStats(String type, Collection keys) { - int count = 0; + void addToTagStats(String type, Collection keys) { for (String key : keys) { - count += addToTagStats(type, key); + addToTagStats(type, key); } - return count; } void printTagStats() { @@ -182,6 +182,11 @@ void printTagStats() { System.out.println("\t" + key + ": " + stats); } } + } + + public static class GeomStats { + + private final HashMap geomStats = new HashMap<>(); void addGeomStats(Node geomNode) { if (geomNode != null) { @@ -201,7 +206,6 @@ void dumpGeomStats() { } geomStats.clear(); } - } public OSMImporter(String layerName) { @@ -252,57 +256,28 @@ public long reIndex(GraphDatabaseService database, int commitInterval, boolean i OSMLayer.class); dataset = OSMDataset.withDatasetId(tx, layer, osm_dataset); tx.commit(); + } catch (Exception e) { + throw new SpatialDatabaseException("Failed to re-index layer " + layerName + ": " + e.getMessage(), e); } try (Transaction tx = beginTx(database)) { layer.clear(tx); // clear the index without destroying underlying data tx.commit(); } - TraversalDescription traversal = new MonoDirectionalTraversalDescription(); long startTime = System.currentTimeMillis(); - org.neo4j.graphdb.traversal.TraversalDescription findWays = traversal.depthFirst() - .evaluator(Evaluators.excludeStartPosition()) - .relationships(OSMRelation.WAYS, Direction.OUTGOING) - .relationships(OSMRelation.NEXT, Direction.OUTGOING); - org.neo4j.graphdb.traversal.TraversalDescription findNodes = traversal.depthFirst() - .evaluator(Evaluators.excludeStartPosition()) - .relationships(OSMRelation.FIRST_NODE, Direction.OUTGOING) - .relationships(OSMRelation.NEXT, Direction.OUTGOING); Transaction tx = beginTx(database); boolean useWays = missingChangesets > 0; int count = 0; try { - layer.setExtraPropertyNames(stats.getTagStats("all").getTags(), tx); + OSMIndexer indexer = new OSMIndexer(layer, geomStats, includePoints); + layer.setExtraPropertyNames(tagStats.getTagStats("all").getTags(), tx); if (useWays) { beginProgressMonitor(dataset.getWayCount(tx)); - for (Node way : toList(findWays.traverse(tx.getNodeByElementId(osm_dataset)).nodes())) { + for (Node way : indexer.allWays(tx)) { updateProgressMonitor(count); incrLogContext(); - stats.addGeomStats(layer.addWay(tx, way, true)); - if (includePoints) { - long badProxies = 0; - long goodProxies = 0; - for (Node proxy : findNodes.traverse(way).nodes()) { - Relationship nodeRel = proxy.getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING); - if (nodeRel == null) { - badProxies++; - } else { - goodProxies++; - Node node = proxy.getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING) - .getEndNode(); - stats.addGeomStats(layer.addWay(tx, node, true)); - } - } - if (badProxies > 0) { - System.out.println("Unexpected dangling proxies for way: " + way); - if (way.hasProperty(PROP_WAY_ID)) { - System.out.println("\tWay: " + way.getProperty(PROP_WAY_ID)); - } - System.out.println("\tBad Proxies: " + badProxies); - System.out.println("\tGood Proxies: " + goodProxies); - } - } + indexer.indexByWay(tx, way); if (++count % commitInterval == 0) { tx.commit(); tx.close(); @@ -311,16 +286,12 @@ public long reIndex(GraphDatabaseService database, int commitInterval, boolean i } // TODO ask charset to user? } else { beginProgressMonitor(dataset.getChangesetCount(tx)); - for (Node unsafeNode : toList(dataset.getAllChangesetNodes(tx))) { + for (Node unsafeNode : indexer.allChangesets(tx)) { WrappedNode changeset = new WrappedNode(unsafeNode); changeset.refresh(tx); updateProgressMonitor(count); incrLogContext(); - try (var relationships = changeset.getRelationships(Direction.INCOMING, OSMRelation.CHANGESET)) { - for (Relationship rel : relationships) { - stats.addGeomStats(layer.addWay(tx, rel.getStartNode(), true)); - } - } + indexer.indexByChangeset(tx, changeset.inner); if (++count % commitInterval == 0) { tx.commit(); tx.close(); @@ -329,6 +300,8 @@ public long reIndex(GraphDatabaseService database, int commitInterval, boolean i } // TODO ask charset to user? } tx.commit(); + } catch (Exception e) { + throw new SpatialDatabaseException("Failed to re-index layer " + layerName + ": " + e.getMessage(), e); } finally { endProgressMonitor(); tx.close(); @@ -337,19 +310,85 @@ public long reIndex(GraphDatabaseService database, int commitInterval, boolean i if (verboseLog) { long stopTime = System.currentTimeMillis(); log("info | Re-indexing elapsed time in seconds: " + (1.0 * (stopTime - startTime) / 1000.0)); - stats.dumpGeomStats(); + geomStats.dumpGeomStats(); } return count; } - private List toList(Iterable iterable) { - ArrayList list = new ArrayList<>(); - if (iterable != null) { - for (Node e : iterable) { - list.add(e); + public static class OSMIndexer { + + private static final TraversalDescription traversal = new MonoDirectionalTraversalDescription(); + private static final org.neo4j.graphdb.traversal.TraversalDescription findNodes = traversal.depthFirst() + .evaluator(Evaluators.excludeStartPosition()) + .relationships(OSMRelation.FIRST_NODE, Direction.OUTGOING) + .relationships(OSMRelation.NEXT, Direction.OUTGOING); + private final OSMLayer layer; + private final boolean includePoints; + private final GeomStats stats; + + public OSMIndexer(OSMLayer layer, GeomStats stats, boolean includePoints) { + this.layer = layer; + this.stats = stats; + this.includePoints = includePoints; + } + + public void indexByGeometryNode(Transaction tx, Node geomNode) { + if (!layer.getIndex().isNodeIndexed(tx, geomNode.getElementId())) { + layer.addGeomNode(tx, geomNode, false); + } + } + + public void indexByWay(Transaction tx, Node way) { + stats.addGeomStats(layer.addWay(tx, way, true)); + if (includePoints) { + long badProxies = 0; + long goodProxies = 0; + for (Node proxy : findNodes.traverse(way).nodes()) { + Relationship nodeRel = proxy.getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING); + if (nodeRel == null) { + badProxies++; + } else { + goodProxies++; + Node node = proxy.getSingleRelationship(OSMRelation.NODE, Direction.OUTGOING).getEndNode(); + stats.addGeomStats(layer.addWay(tx, node, true)); + } + } + if (badProxies > 0) { + System.out.println("Unexpected dangling proxies for way: " + way); + if (way.hasProperty(PROP_WAY_ID)) { + System.out.println("\tWay: " + way.getProperty(PROP_WAY_ID)); + } + System.out.println("\tBad Proxies: " + badProxies); + System.out.println("\tGood Proxies: " + goodProxies); + } + } + } + + public void indexByChangeset(Transaction tx, Node changeset) { + for (Relationship rel : changeset.getRelationships(Direction.INCOMING, OSMRelation.CHANGESET)) { + stats.addGeomStats(layer.addWay(tx, rel.getStartNode(), true)); } } - return list; + + public List allWays(Transaction tx) { + OSMDataset dataset = OSMDataset.fromLayer(tx, layer); + return toList(dataset.getAllWayNodes(tx)); + } + + public List allChangesets(Transaction tx) { + OSMDataset dataset = OSMDataset.fromLayer(tx, layer); + return toList(dataset.getAllChangesetNodes(tx)); + } + + private List toList(Iterable iterable) { + ArrayList list = new ArrayList<>(); + if (iterable != null) { + for (Node e : iterable) { + list.add(e); + } + } + return list; + } } private static class GeometryMetaData { @@ -414,19 +453,21 @@ private Envelope getBBox() { private static abstract class OSMWriter { private static final int UNKNOWN_CHANGESET = -1; - final StatsManager statsManager; + final StatsManager tagStats; + GeomStats geomStats; final OSMImporter osmImporter; T osm_dataset; long missingChangesets = 0; - private OSMWriter(StatsManager statsManager, OSMImporter osmImporter) { - this.statsManager = statsManager; + private OSMWriter(StatsManager tagStats, GeomStats geomStats, OSMImporter osmImporter) { + this.tagStats = tagStats; + this.geomStats = geomStats; this.osmImporter = osmImporter; } static OSMWriter fromGraphDatabase(GraphDatabaseService graphDb, SecurityContext securityContext, - StatsManager stats, OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException { - return new OSMGraphWriter(graphDb, securityContext, stats, osmImporter, txInterval); + StatsManager tagStats, GeomStats geomStats, OSMImporter osmImporter, int txInterval) { + return new OSMGraphWriter(graphDb, securityContext, tagStats, geomStats, osmImporter, txInterval); } protected abstract void startWays(); @@ -457,8 +498,6 @@ void createRelationship(T from, T to, OSMRelation relType) { long firstFindTime = 0; long lastFindTime = 0; long firstLogTime = 0; - static int foundNodes = 0; - static int createdNodes = 0; int foundOSMNodes = 0; int missingUserCount = 0; @@ -496,26 +535,22 @@ void logNodesFound(long currentTime) { if (currentTime > 0) { duration = (int) ((currentTime - firstFindTime) / 1000); } - System.out.println(new Date(currentTime) + ": Found " - + foundOSMNodes + " nodes during " - + duration + "s way creation: "); + System.out.printf("%s: Found %d nodes during %ds way creation:%n", new Date(currentTime), foundOSMNodes, + duration); for (String type : nodeFindStats.keySet()) { LogCounter found = nodeFindStats.get(type); double rate = 0.0f; if (found.totalTime > 0) { rate = (1000.0 * (float) found.count / (float) found.totalTime); } - System.out.println("\t" + type + ": \t" + found.count - + "/" + (found.totalTime / 1000) - + "s" + " \t(" + rate - + " nodes/second)"); + System.out.printf("\t%s: \t%d/%ds \t%f nodes/second%n", type, found.count, (found.totalTime / 1000), + rate); } findTime = currentTime; } } - void logNodeAddition(LinkedHashMap tags, - String type) { + void logNodeAddition(String type) { Integer count = stats.get(type); if (count == null) { count = 1; @@ -529,9 +564,9 @@ void logNodeAddition(LinkedHashMap tags, logTime = currentTime; } if (currentTime - logTime > 1432) { - System.out.println( - new Date(currentTime) + ": Saving " + type + " " + count + " \t(" + (1000.0 * (float) count - / (float) (currentTime - firstLogTime)) + " " + type + "/second)"); + double rate = (1000.0 * (float) count / (float) (currentTime - firstLogTime)); + System.out.printf("%s: Saving %s %d \t(%f %s/second)%n", new Date(currentTime), type, count, rate, + type); logTime = currentTime; } } @@ -552,18 +587,17 @@ void describeLoaded() { private void missingNode(long ndRef) { if (missingNodeCount++ < 10) { - osmImporter.error("Cannot find node for osm-id " + ndRef); + osmImporter.errorf("Cannot find node for osm-id %d%n", ndRef); } } private void describeMissing() { if (missingNodeCount > 0) { - osmImporter.error("When processing the ways, there were " - + missingNodeCount + " missing nodes"); + osmImporter.errorf("When processing the ways, there were %d missing nodes%n", missingNodeCount); } if (missingMemberCount > 0) { - osmImporter.error("When processing the relations, there were " - + missingMemberCount + " missing members"); + osmImporter.errorf("When processing the relations, there were %d missing members%n", + missingMemberCount); } } @@ -571,13 +605,10 @@ private void describeMissing() { private void missingMember(String description) { if (missingMemberCount++ < 10) { - osmImporter.error("Cannot find member: " + description); + osmImporter.errorf("Cannot find member: %s%n", description); } } - T currentNode = null; - T prev_way = null; - T prev_relation = null; int nodeCount = 0; int poiCount = 0; int wayCount = 0; @@ -594,34 +625,26 @@ void addOSMBBox(Map bboxProperties) { } /** - * Create a new OSM node from the specified attributes (including - * location, user, changeset). The node is stored in the currentNode - * field, so that it can be used in the subsequent call to - * addOSMNodeTags after we close the XML tag for OSM nodes. - * - * @param nodeProps HashMap of attributes for the OSM-node + * Create a new OSM node from the specified attributes (including location, user, changeset). */ - void createOSMNode(Map nodeProps) { - T userNode = getUserNode(nodeProps); - T changesetNode = getChangesetNode(nodeProps, userNode); - currentNode = addNode(LABEL_NODE, nodeProps, PROP_NODE_ID); - createRelationship(currentNode, changesetNode, OSMRelation.CHANGESET); - nodeCount++; - } - - private void addOSMNodeTags(boolean allPoints, + private void createOSMNode(Map nodeProperties, boolean allPoints, LinkedHashMap currentNodeTags) { + T userNode = getUserNode(nodeProperties); + T changesetNode = getChangesetNode(nodeProperties, userNode); + T node = addNode(LABEL_NODE, nodeProperties, PROP_NODE_ID); + createRelationship(node, changesetNode, OSMRelation.CHANGESET); + nodeCount++; currentNodeTags.remove("created_by"); // redundant information // Nodes with tags get added to the index as point geometries if (allPoints || !currentNodeTags.isEmpty()) { - Map nodeProps = getNodeProperties(currentNode); + Map nodeProps = getNodeProperties(node); double[] location = new double[]{ - (Double) nodeProps.get("lon"), - (Double) nodeProps.get("lat")}; - addNodeGeometry(currentNode, GTYPE_POINT, new Envelope(location), 1); + (Double) nodeProps.get(PROP_NODE_LON), + (Double) nodeProps.get(PROP_NODE_LAT)}; + addNodeGeometry(node, GTYPE_POINT, new Envelope(location), 1); poiCount++; } - addNodeTags(currentNode, currentNodeTags, "node"); + addNodeTags(node, currentNodeTags, "node"); } protected void debugNodeWithId(T node, String idName, long[] idValues) { @@ -634,8 +657,8 @@ protected void debugNodeWithId(T node, String idName, long[] idValues) { } } - protected void createOSMWay(Map wayProperties, - ArrayList wayNodes, LinkedHashMap wayTags) { + protected void createOSMWay(Map wayProperties, ArrayList wayNodes, + LinkedHashMap wayTags) { RoadDirection direction = getRoadDirection(wayTags); String name = (String) wayTags.get("name"); int geometry = GTYPE_LINESTRING; @@ -654,12 +677,6 @@ protected void createOSMWay(Map wayProperties, T changesetNode = getChangesetNode(wayProperties, userNode); T way = addNode(LABEL_WAY, wayProperties, PROP_WAY_ID); createRelationship(way, changesetNode, OSMRelation.CHANGESET); - if (prev_way == null) { - createRelationship(osm_dataset, way, OSMRelation.WAYS); - } else { - createRelationship(prev_way, way, OSMRelation.NEXT); - } - prev_way = way; addNodeTags(way, wayTags, "way"); Envelope bbox = null; T firstNode = null; @@ -687,8 +704,8 @@ protected void createOSMWay(Map wayProperties, createRelationship(proxyNode, pointNode, OSMRelation.NODE, null); Map nodeProps = getNodeProperties(pointNode); double[] location = new double[]{ - (Double) nodeProps.get("lon"), - (Double) nodeProps.get("lat")}; + (Double) nodeProps.get(PROP_NODE_LON), + (Double) nodeProps.get(PROP_NODE_LAT)}; if (bbox == null) { bbox = new Envelope(location); } else { @@ -698,7 +715,8 @@ protected void createOSMWay(Map wayProperties, createRelationship(way, proxyNode, OSMRelation.FIRST_NODE); } else { relProps.clear(); - double[] prevLoc = new double[]{(Double) prevProps.get("lon"), (Double) prevProps.get("lat")}; + double[] prevLoc = new double[]{(Double) prevProps.get(PROP_NODE_LON), + (Double) prevProps.get(PROP_NODE_LAT)}; double length = distance(prevLoc[0], prevLoc[1], location[0], location[1]); relProps.put("length", length); /* @@ -734,12 +752,6 @@ private void createOSMRelation(Map relationProperties, relationProperties.put("name", name); } T relation = addNode(LABEL_RELATION, relationProperties, PROP_RELATION_ID); - if (prev_relation == null) { - createRelationship(osm_dataset, relation, OSMRelation.RELATIONS); - } else { - createRelationship(prev_relation, relation, OSMRelation.NEXT); - } - prev_relation = relation; addNodeTags(relation, relationTags, "relation"); // We will test for cases that invalidate multilinestring further down GeometryMetaData metaGeom = new GeometryMetaData(GTYPE_MULTILINESTRING); @@ -764,13 +776,14 @@ private void createOSMRelation(Map relationProperties, continue; } if (member == relation) { - osmImporter.error("Cannot add relation to same member: relation[" + relationTags + "] - member[" - + memberProps + "]"); + osmImporter.errorf("Cannot add relation to same member: relation[%s] - member[%s]%n", + relationTags, memberProps); continue; } Map nodeProps = getNodeProperties(member); if (memberType.equals("node")) { - double[] location = new double[]{(Double) nodeProps.get("lon"), (Double) nodeProps.get("lat")}; + double[] location = new double[]{(Double) nodeProps.get(PROP_NODE_LON), + (Double) nodeProps.get(PROP_NODE_LAT)}; metaGeom.expandToIncludePoint(location); } else if (memberType.equals("nodes")) { System.err.println("Unexpected 'nodes' member type"); @@ -792,8 +805,7 @@ private void createOSMRelation(Map relationProperties, } } if (metaGeom.isValid()) { - addNodeGeometry(relation, metaGeom.getGeometryType(), - metaGeom.getBBox(), metaGeom.getVertices()); + addNodeGeometry(relation, metaGeom.getGeometryType(), metaGeom.getBBox(), metaGeom.getVertices()); } this.relationCount++; } @@ -811,8 +823,8 @@ private void createOSMRelation(Map relationProperties, protected abstract T getOSMNode(long osmId, T changesetNode); - protected abstract void updateGeometryMetaDataFromMember(T member, - GeometryMetaData metaGeom, Map nodeProps); + protected abstract void updateGeometryMetaDataFromMember(T member, GeometryMetaData metaGeom, + Map nodeProps); protected abstract void finish(); @@ -889,7 +901,6 @@ private static class OSMGraphWriter extends OSMWriter { private WrappedNode currentChangesetNode; private long currentUserId = -1; private WrappedNode currentUserNode; - private WrappedNode usersNode; private final HashMap changesetNodes = new HashMap<>(); private Transaction tx; private int checkCount = 0; @@ -899,19 +910,24 @@ private static class OSMGraphWriter extends OSMWriter { private IndexDefinition relationIndex; private IndexDefinition changesetIndex; private IndexDefinition userIndex; - private final String layerHash; - private final HashMap hashedLabels = new HashMap<>(); + private final OSMDataset.LabelHasher labelHasher; - private OSMGraphWriter(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager statsManager, - OSMImporter osmImporter, int txInterval) throws NoSuchAlgorithmException { - super(statsManager, osmImporter); + private OSMGraphWriter(GraphDatabaseService graphDb, SecurityContext securityContext, StatsManager tagsStats, + GeomStats geomStats, + OSMImporter osmImporter, int txInterval) { + super(tagsStats, geomStats, osmImporter); this.graphDb = graphDb; this.securityContext = securityContext; this.txInterval = txInterval; if (this.txInterval < 100) { System.err.println("Warning: Unusually short txInterval, expect bad insert performance"); } - this.layerHash = md5Hash(osmImporter.layerName); + try { + this.labelHasher = new OSMDataset.LabelHasher(osmImporter.layerName); + } catch (NoSuchAlgorithmException e) { + throw new SpatialDatabaseException( + "Failed to create OSMGraphWriter for '" + osmImporter.layerName + "': " + e.getMessage(), e); + } checkTx(null); // Opens transaction for future writes } @@ -949,12 +965,8 @@ private static Transaction beginTx(GraphDatabaseService database, SecurityContex private void beginTx() { tx = beginTx(graphDb); recoverNode(osm_dataset); - recoverNode(currentNode); - recoverNode(prev_relation); - recoverNode(prev_way); recoverNode(currentChangesetNode); recoverNode(currentUserNode); - recoverNode(usersNode); changesetNodes.forEach((id, node) -> node.refresh(tx)); } @@ -982,7 +994,7 @@ private WrappedNode findNodeByName(Label label, String name) { } private WrappedNode createNodeWithLabel(Transaction tx, Label label) { - Label hashed = getLabelHashed(label); + Label hashed = labelHasher.getLabelHashed(label); return WrappedNode.fromNode(tx.createNode(label, hashed)); } @@ -1017,23 +1029,14 @@ protected void optimize() { } } - private Label getLabelHashed(Label label) { - if (hashedLabels.containsKey(label)) { - return hashedLabels.get(label); - } - Label hashed = Label.label(label.name() + "_" + layerHash); - hashedLabels.put(label, hashed); - return hashed; - } - private Node findNodeByLabelProperty(Transaction tx, Label label, String propertyKey, Object value) { - Label hashed = getLabelHashed(label); + Label hashed = labelHasher.getLabelHashed(label); return tx.findNode(hashed, propertyKey, value); } private IndexDefinition createIndex(Label label, String propertyKey) { - Label hashed = getLabelHashed(label); - String indexName = String.format("OSM-%s-%s-%s", osmImporter.layerName, hashed.name(), propertyKey); + Label hashed = labelHasher.getLabelHashed(label); + String indexName = OSMDataset.indexNameFor(osmImporter.layerName, hashed.name(), propertyKey); IndexDefinition index = findIndex(tx, indexName, hashed, propertyKey); if (index == null) { successTx(); @@ -1043,10 +1046,25 @@ private IndexDefinition createIndex(Label label, String propertyKey) { } System.out.println("Created index " + index.getName()); beginTx(); + saveIndexName(label, propertyKey, indexName); } return index; } + private void saveIndexName(Label label, String propertyKey, String indexName) { + String indexKey = OSMDataset.indexKeyFor(label, propertyKey); + String previousIndex = (String) osm_dataset.getProperty(indexKey, null); + if (previousIndex == null) { + osm_dataset.setProperty(indexKey, indexName); + } else if (previousIndex.equals(indexName)) { + System.out.printf("OSMLayer '%s' already has matching index definition for '%s': %s%n", + osm_dataset.getProperty("name", ""), indexKey, previousIndex); + } else { + throw new IllegalStateException(String.format("OSMLayer '%s' already has index definition for '%s': %s", + osm_dataset.getProperty("name", ""), indexKey, previousIndex)); + } + } + private IndexDefinition createIndexIfNotNull(IndexDefinition index, Label label, String propertyKey) { if (index == null) { index = createIndex(label, propertyKey); @@ -1105,9 +1123,9 @@ private void addProperties(Entity node, Map properties) { @Override protected void addNodeTags(WrappedNode node, LinkedHashMap tags, String type) { - logNodeAddition(tags, type); + logNodeAddition(type); if (node != null && !tags.isEmpty()) { - statsManager.addToTagStats(type, tags.keySet()); + tagStats.addToTagStats(type, tags.keySet()); WrappedNode tagsNode = createNodeWithLabel(tx, LABEL_TAGS); addProperties(tagsNode.inner, tags); node.createRelationshipTo(tagsNode, OSMRelation.TAGS); @@ -1121,13 +1139,13 @@ protected void addNodeGeometry(WrappedNode node, int gtype, Envelope bbox, int v if (gtype == GTYPE_GEOMETRY) { gtype = vertices > 1 ? GTYPE_MULTIPOINT : GTYPE_POINT; } - Node geomNode = tx.createNode(); + Node geomNode = tx.createNode(LABEL_GEOM); geomNode.setProperty("gtype", gtype); geomNode.setProperty("vertices", vertices); geomNode.setProperty(PROP_BBOX, new double[]{bbox.getMinX(), bbox.getMaxX(), bbox.getMinY(), bbox.getMaxY()}); node.createRelationshipTo(geomNode, OSMRelation.GEOM); - statsManager.addGeomStats(gtype); + geomStats.addGeomStats(gtype); } } @@ -1248,7 +1266,7 @@ protected WrappedNode getChangesetNode(Map nodeProps, WrappedNod } else { LinkedHashMap changesetProps = new LinkedHashMap<>(); changesetProps.put(PROP_CHANGESET, currentChangesetId); - changesetProps.put("timestamp", nodeProps.get("timestamp")); + changesetProps.put(PROP_TIMESTAMP, nodeProps.get(PROP_TIMESTAMP)); currentChangesetNode = addNode(LABEL_CHANGESET, changesetProps, PROP_CHANGESET); changesetCount++; if (userNode != null) { @@ -1278,15 +1296,10 @@ protected WrappedNode getUserNode(Map nodeProps) { } else { LinkedHashMap userProps = new LinkedHashMap<>(); userProps.put(PROP_USER_ID, currentUserId); - userProps.put("name", name); - userProps.put("timestamp", nodeProps.get("timestamp")); + userProps.put(PROP_USER_NAME, name); + userProps.put(PROP_TIMESTAMP, nodeProps.get(PROP_TIMESTAMP)); currentUserNode = addNode(LABEL_USER, userProps, PROP_USER_ID); userCount++; - if (usersNode == null) { - usersNode = createNodeWithLabel(tx, LABEL_USER); - osm_dataset.createRelationshipTo(usersNode, OSMRelation.USERS); - } - usersNode.createRelationshipTo(currentUserNode, OSMRelation.OSM_USER); } } } catch (Exception e) { @@ -1313,7 +1326,8 @@ public void importFile(GraphDatabaseService database, String dataset, int txInte public void importFile(GraphDatabaseService database, String dataset, boolean allPoints, int txInterval) throws Exception { - importFile(OSMWriter.fromGraphDatabase(database, securityContext, stats, this, txInterval), dataset, allPoints, + importFile(OSMWriter.fromGraphDatabase(database, securityContext, tagStats, geomStats, this, txInterval), + dataset, allPoints, charset); } @@ -1327,14 +1341,6 @@ public CountedFileReader(String path, Charset charset) throws FileNotFoundExcept this.length = (new File(path)).length(); } - public long getCharsRead() { - return charsRead; - } - - public long getlength() { - return length; - } - public double getProgress() { return length > 0 ? (double) charsRead / (double) length : 0; } @@ -1344,8 +1350,7 @@ public int getPercentRead() { } @Override - public int read(@Nonnull char[] cbuf, int offset, int length) - throws IOException { + public int read(@Nonnull char[] cbuf, int offset, int length) throws IOException { int read = super.read(cbuf, offset, length); if (read > 0) { charsRead += read; @@ -1407,6 +1412,7 @@ private void importFile(OSMWriter osmWriter, String dataset, boolean allPoint try { ArrayList currentXMLTags = new ArrayList<>(); int depth = 0; + Map nodeProperties = null; Map wayProperties = null; ArrayList wayNodes = new ArrayList<>(); Map relationProperties = null; @@ -1429,15 +1435,7 @@ private void importFile(OSMWriter osmWriter, String dataset, boolean allPoint osmWriter.addOSMBBox(extractProperties(PROP_BBOX, parser)); } else if (tagPath.equals("[osm, node]")) { /* */ - boolean includeNode = true; - Map nodeProperties = extractProperties("node", parser); - if (filterEnvelope != null) { - includeNode = filterEnvelope.contains((Double) nodeProperties.get("lon"), - (Double) nodeProperties.get("lat")); - } - if (includeNode) { - osmWriter.createOSMNode(nodeProperties); - } + nodeProperties = extractProperties("node", parser); } else if (tagPath.equals("[osm, way]")) { /* */ if (!startedWays) { @@ -1472,16 +1470,13 @@ private void importFile(OSMWriter osmWriter, String dataset, boolean allPoint } if (startedRelations) { if (countXMLTags < 10) { - debug("Starting tag at depth " + depth + ": " - + currentXMLTags.get(depth) + " - " - + currentXMLTags); + debugf("Starting tag at depth %d: %s - %s%n", depth, currentXMLTags.get(depth), + currentXMLTags); for (int i = 0; i < parser.getAttributeCount(); i++) { - debug("\t" + currentXMLTags + ": " - + parser.getAttributeLocalName(i) + "[" - + parser.getAttributeNamespace(i) + "," - + parser.getAttributePrefix(i) + "," - + parser.getAttributeType(i) + "," - + "] = " + parser.getAttributeValue(i)); + debugf("\t%s: %s[%s,%s,%s] = %s%n", currentXMLTags, parser.getAttributeLocalName(i), + parser.getAttributeNamespace(i), + parser.getAttributePrefix(i), parser.getAttributeType(i), + parser.getAttributeValue(i)); } } countXMLTags++; @@ -1491,7 +1486,9 @@ private void importFile(OSMWriter osmWriter, String dataset, boolean allPoint case javax.xml.stream.XMLStreamConstants.END_ELEMENT: switch (currentXMLTags.toString()) { case "[osm, node]": - osmWriter.addOSMNodeTags(allPoints, currentNodeTags); + if (nodeFilterMatches(nodeProperties)) { + osmWriter.createOSMNode(nodeProperties, allPoints, currentNodeTags); + } break; case "[osm, way]": osmWriter.createOSMWay(wayProperties, wayNodes, currentNodeTags); @@ -1521,8 +1518,18 @@ private void importFile(OSMWriter osmWriter, String dataset, boolean allPoint long stopTime = System.currentTimeMillis(); log("info | Elapsed time in seconds: " + (1.0 * (stopTime - startTime) / 1000.0)); - stats.dumpGeomStats(); - stats.printTagStats(); + geomStats.dumpGeomStats(); + tagStats.printTagStats(); + } + } + + private boolean nodeFilterMatches(Map nodeProperties) { + if (filterEnvelope == null) { + return true; + } else { + Double x = (Double) nodeProperties.get(PROP_NODE_LON); + Double y = (Double) nodeProperties.get(PROP_NODE_LAT); + return x != null && y != null && filterEnvelope.contains(x, y); } } @@ -1554,7 +1561,7 @@ private Map extractProperties(String name, XMLStreamReader parse prop = name + "_osm_id"; name = null; } - if (prop.equals("lat") || prop.equals("lon")) { + if (prop.equals(PROP_NODE_LAT) || prop.equals(PROP_NODE_LON)) { properties.put(prop, Double.parseDouble(value)); } else if (name != null && prop.equals("version")) { properties.put(prop, Integer.parseInt(value)); @@ -1562,7 +1569,7 @@ private Map extractProperties(String name, XMLStreamReader parse if (!value.equals("true") && !value.equals("1")) { properties.put(prop, false); } - } else if (prop.equals("timestamp")) { + } else if (prop.equals(PROP_TIMESTAMP)) { try { Date timestamp = timestampFormat.parse(value); properties.put(prop, timestamp.getTime()); @@ -1620,24 +1627,31 @@ private void log(PrintStream out, String message) { out.println(message); } + private void logf(PrintStream out, String format, Object... args) { + if (logContext != null) { + format = logContext + "[" + contextLine + "]: " + format; + } + out.printf(format, args); + } + private void log(String message) { if (verboseLog) { log(System.out, message); } } - private void debug(String message) { + private void debugf(String format, Object... args) { if (debugLog) { - log(System.out, message); + logf(System.out, format, args); } } - private void error(String message) { - log(System.err, message); + private void errorf(String format, Object... args) { + logf(System.err, format, args); } private void error(String message, Exception e) { - log(System.err, message); + logf(System.err, message + ": %s", e.getMessage()); e.printStackTrace(System.err); } @@ -1647,8 +1661,7 @@ private void error(String message, Exception e) { private boolean verboseLog = true; // "2008-06-11T12:36:28Z" - private final DateFormat timestampFormat = new SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss'Z'"); + private final DateFormat timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); public void setDebug(boolean verbose) { this.debugLog = verbose; @@ -1728,14 +1741,13 @@ private void loadTestOsmData(String layerName, int commitInterval) throws Except + " seconds ==="); } - private DatabaseLayout prepareLayout(boolean delete) throws IOException { + private void prepareLayout(boolean delete) throws IOException { Neo4jLayout homeLayout = Neo4jLayout.of(dbPath.toPath()); DatabaseLayout databaseLayout = homeLayout.databaseLayout(databaseName); if (delete) { FileUtils.deleteDirectory(databaseLayout.databaseDirectory()); FileUtils.deleteDirectory(databaseLayout.getTransactionLogsDirectory()); } - return databaseLayout; } private void prepareDatabase(boolean delete) throws IOException { diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java index 91064497..10b0f257 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayer.java @@ -27,8 +27,13 @@ import org.neo4j.gis.spatial.Constants; import org.neo4j.gis.spatial.DynamicLayer; import org.neo4j.gis.spatial.DynamicLayerConfig; +import org.neo4j.gis.spatial.EditableLayer; +import org.neo4j.gis.spatial.SpatialDatabaseException; import org.neo4j.gis.spatial.SpatialDatabaseService; import org.neo4j.gis.spatial.SpatialDataset; +import org.neo4j.gis.spatial.SpatialRelationshipTypes; +import org.neo4j.gis.spatial.merge.MergeUtils; +import org.neo4j.gis.spatial.rtree.Listener; import org.neo4j.gis.spatial.rtree.NullListener; import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.Node; @@ -40,7 +45,7 @@ * extends the DynamicLayer class because the OSM dataset can have many layers. * Only one is primary, the layer containing all ways. Other layers are dynamic. */ -public class OSMLayer extends DynamicLayer { +public class OSMLayer extends DynamicLayer implements MergeUtils.Mergeable { private OSMDataset osmDataset; @@ -61,54 +66,61 @@ public Integer getGeometryType() { /** * OSM always uses WGS84 CRS; so we return that. - * - * @param tx the transaction */ - @Override public CoordinateReferenceSystem getCoordinateReferenceSystem(Transaction tx) { - try { - return DefaultGeographicCRS.WGS84; - } catch (Exception e) { - System.err.println("Failed to decode WGS84 CRS: " + e.getMessage()); - e.printStackTrace(System.err); - return null; - } + return DefaultGeographicCRS.WGS84; } protected void clear(Transaction tx) { indexWriter.clear(tx, new NullListener()); } - public Node addWay(Transaction tx, Node way) { - return addWay(tx, way, false); + @Override + public void delete(Transaction tx, Listener monitor) { + Relationship datasetRel = this.getLayerNode(tx) + .getSingleRelationship(SpatialRelationshipTypes.LAYERS, Direction.INCOMING); + if (datasetRel != null) { + Node datasetNode = datasetRel.getStartNode(); + datasetRel.delete(); + datasetNode.delete(); + } + super.delete(tx, monitor); } public Node addWay(Transaction tx, Node way, boolean verifyGeom) { Relationship geomRel = way.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING); if (geomRel != null) { - Node geomNode = geomRel.getEndNode(); - try { - // This is a test of the validity of the geometry, throws exception on error - if (verifyGeom) { - getGeometryEncoder().decodeGeometry(geomNode); - } - indexWriter.add(tx, geomNode); - } catch (Exception e) { - System.err.println( - "Failed geometry test on node " + geomNode.getProperty("name", geomNode.toString()) + ": " - + e.getMessage()); - for (String key : geomNode.getPropertyKeys()) { - System.err.println("\t" + key + ": " + geomNode.getProperty(key)); - } + return addGeomNode(tx, geomRel.getEndNode(), verifyGeom); + } else { + return null; + } + } + + public Node addGeomNode(Transaction tx, Node geomNode, boolean verifyGeom) { + try { + // This is a test of the validity of the geometry, throws exception on error + if (verifyGeom) { + getGeometryEncoder().decodeGeometry(geomNode); + } + indexWriter.add(tx, geomNode); + } catch (Exception e) { + System.err.printf("Failed geometry test on node '%s': %s%n", + geomNode.getProperty("name", geomNode.toString()), e.getMessage()); + for (String key : geomNode.getPropertyKeys()) { + System.err.println("\t" + key + ": " + geomNode.getProperty(key)); + } + Relationship geomRel = geomNode.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING); + if (geomRel != null) { + Node way = geomRel.getStartNode(); System.err.println("For way node " + way); for (String key : way.getPropertyKeys()) { System.err.println("\t" + key + ": " + way.getProperty(key)); } - // e.printStackTrace(System.err); + } else { + System.err.printf("Geometry node %d has no connected OSM model node%n", geomNode.getId()); } - return geomNode; } - return null; + return geomNode; } /** @@ -144,7 +156,6 @@ public boolean removeDynamicLayer(Transaction tx, String name) { * to the way node and then to the tags node to test if the way is a * residential street. */ - @SuppressWarnings("unchecked") public DynamicLayerConfig addDynamicLayerOnWayTags(Transaction tx, String name, int type, HashMap tags) { JSONObject query = new JSONObject(); if (tags != null && !tags.isEmpty()) { @@ -276,4 +287,21 @@ public File getStyle() { // TODO: Replace with a proper resource lookup, since this will be in the JAR return new File("dev/neo4j/neo4j-spatial/src/main/resources/sld/osm/osm.sld"); } + + @Override + public long mergeFrom(Transaction tx, EditableLayer other) { + if (other instanceof OSMLayer) { + try { + OSMMerger merger = new OSMMerger(this); + return merger.merge(tx, (OSMLayer) other); + } catch (Exception e) { + throw new SpatialDatabaseException( + "Failed to merge OSM layer " + other.getName() + " into " + this.getName() + ": " + + e.getMessage(), e); + } + } else { + throw new IllegalArgumentException( + "Cannot merge non-OSM layer into OSM layer: '" + other.getName() + "' is not OSM"); + } + } } diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayerToShapefileExporter.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayerToShapefileExporter.java index f8b249de..58228ea3 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMLayerToShapefileExporter.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMLayerToShapefileExporter.java @@ -15,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ package org.neo4j.gis.spatial.osm; diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java new file mode 100644 index 00000000..c7481a7d --- /dev/null +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMMerger.java @@ -0,0 +1,286 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j Spatial. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gis.spatial.osm; + +import static org.neo4j.gis.spatial.osm.OSMModel.LABEL_NODE; +import static org.neo4j.gis.spatial.osm.OSMModel.LABEL_RELATION; +import static org.neo4j.gis.spatial.osm.OSMModel.LABEL_WAY; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_BBOX; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_NODE_ID; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_NODE_LAT; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_NODE_LON; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_RELATION_ID; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_WAY_ID; + +import java.util.ArrayList; +import java.util.function.BiConsumer; +import org.neo4j.gis.spatial.GeometryEncoder; +import org.neo4j.gis.spatial.rtree.Envelope; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.ResourceIterator; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.schema.IndexDefinition; + +public class OSMMerger { + + private final OSMLayer layer; + + public OSMMerger(OSMLayer layer) { + this.layer = layer; + } + + private boolean checkNodeMoved(Node thisNode, long node_osm_id, String expectedPropKey, String nodePropKey, + String keyDescription, Object value) { + if (expectedPropKey.equals(nodePropKey)) { + double thatValue = (Double) value; + double thisValue = (Double) thisNode.getProperty(nodePropKey); + if (Math.abs(thisValue - thatValue) > 0.00000001) { + System.out.printf("Node '%d' has moved %s from %f to %f%n", node_osm_id, keyDescription, thisValue, + thatValue); + return true; + } + } + return false; + } + + private static final long INVALID_OSM_ID = Long.MIN_VALUE; + + private long getOSMId(String propertyKey, Node node) { + Object result = node.getProperty(propertyKey, null); + if (result == null) { + return INVALID_OSM_ID; + } else if (result instanceof Long) { + return (Long) result; + } else if (result instanceof Integer) { + return (Integer) result; + } else { + System.out.printf("Invalid type found for property '%s': %s%n", propertyKey, result); + return INVALID_OSM_ID; + } + } + + public long merge(Transaction tx, OSMLayer otherLayer) { + OSMDataset dataset = OSMDataset.fromLayer(tx, layer); + OSMDataset other = OSMDataset.fromLayer(tx, otherLayer); + otherLayer.clear(tx); + + // Merge all OSM nodes + MergeStats nodesStats = mergeNodes(tx, dataset, other); + + // Merge all OSM Ways + MergeStats waysStats = mergeWays(tx, dataset, other); + + // Merge all OSM Relations + MergeStats relationsStats = mergeRelations(tx, dataset, other); + + // TODO: relabel USERS and CHANGESETS + + // Reindex if necessary + if (nodesStats.needsReindexing() || waysStats.needsReindexing() || relationsStats.needsReindexing()) { + this.layer.clear(tx); + OSMImporter.GeomStats stats = new OSMImporter.GeomStats(); + OSMImporter.OSMIndexer indexer = new OSMImporter.OSMIndexer(layer, stats, false); + for (Node geomNode : nodesStats.geomNodesToAdd) { + indexer.indexByGeometryNode(tx, geomNode); + } + for (Node way : indexer.allWays(tx)) { + indexer.indexByWay(tx, way); + } + stats.dumpGeomStats(); + } + return nodesStats.changed() + waysStats.changed(); + } + + private static class MergeStats { + + String name; + long countMoved = 0; + long countReplaced = 0; + long countAdded = 0; + ArrayList geomNodesToAdd = new ArrayList<>(); + + MergeStats(String name) { + this.name = name; + } + + boolean needsReindexing() { + return countMoved > 0 || countAdded > 0; + } + + long changed() { + return countMoved + countAdded; + } + + void printStats() { + if (countReplaced > 0) { + System.out.printf("During merge we found %d existing %s which were replaced%n", countReplaced, name); + } + if (countMoved > 0) { + System.out.printf("During merge %d out of %d existing %s were moved - re-indexing required%n", + countMoved, countReplaced, name); + } + if (countAdded > 0) { + System.out.printf("During merge %d %s were added - re-indexing required%n", countAdded, name); + } + if (geomNodesToAdd.size() > 0) { + System.out.printf("During merge %d geometry %s were identified for use in re-indexing%n", + geomNodesToAdd.size(), name); + } + } + } + + private MergeStats mergeNodes(Transaction tx, OSMDataset dataset, OSMDataset other) { + IndexDefinition thisIndex = dataset.getIndex(tx, LABEL_NODE, PROP_NODE_ID); + IndexDefinition thatIndex = other.getIndex(tx, LABEL_NODE, PROP_NODE_ID); + Label thisLabelHash = OSMDataset.hashedLabelFrom(thisIndex.getName()); + Label thatLabelHash = OSMDataset.hashedLabelFrom(thatIndex.getName()); + ResourceIterator nodes = tx.findNodes(thatLabelHash); + MergeStats stats = new MergeStats("nodes"); + while (nodes.hasNext()) { + Node node = nodes.next(); + long node_osm_id = getOSMId(PROP_NODE_ID, node); + if (node_osm_id != INVALID_OSM_ID) { + Node thisNode = tx.findNode(thisLabelHash, PROP_NODE_ID, node_osm_id); + if (thisNode != null) { + // TODO: Consider comparing 'timestamp' field and always keeping the newer properties instead + stats.countReplaced++; + boolean moved = false; + for (String nodePropKey : node.getPropertyKeys()) { + Object value = node.getProperty(nodePropKey); + moved = moved || checkNodeMoved(thisNode, node_osm_id, PROP_NODE_LON, nodePropKey, "longitude", + value); + moved = moved || checkNodeMoved(thisNode, node_osm_id, PROP_NODE_LAT, nodePropKey, "latitude", + value); + thisNode.setProperty(nodePropKey, value); + } + if (moved) { + stats.countMoved++; + } + Relationship geomRel = thisNode.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING); + if (geomRel != null) { + Node geomNode = geomRel.getEndNode(); + stats.geomNodesToAdd.add(geomNode); + double x = (Double) node.getProperty(PROP_NODE_LON); + double y = (Double) node.getProperty(PROP_NODE_LAT); + geomNode.setProperty(PROP_BBOX, new double[]{x, x, y, y}); + } + } else { + stats.countAdded++; + node.addLabel(thisLabelHash); + node.removeLabel(thatLabelHash); + Relationship geomRel = node.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING); + if (geomRel != null) { + Node geomNode = geomRel.getEndNode(); + stats.geomNodesToAdd.add(geomNode); + } + } + } else { + System.out.println("Unexpectedly found OSM node without property: " + PROP_NODE_ID); + } + } + stats.printStats(); + return stats; + } + + private MergeStats mergeWays(Transaction tx, OSMDataset dataset, OSMDataset other) { + IndexDefinition thisIndex = dataset.getIndex(tx, LABEL_WAY, PROP_WAY_ID); + IndexDefinition thatIndex = other.getIndex(tx, LABEL_WAY, PROP_WAY_ID); + Label thisLabelHash = OSMDataset.hashedLabelFrom(thisIndex.getName()); + Label thatLabelHash = OSMDataset.hashedLabelFrom(thatIndex.getName()); + return mergeOSMEntities(tx, "ways", PROP_WAY_ID, thisLabelHash, thatLabelHash, this::wayOverlapMerger); + } + + private MergeStats mergeRelations(Transaction tx, OSMDataset dataset, OSMDataset other) { + IndexDefinition thisIndex = dataset.getIndex(tx, LABEL_RELATION, PROP_RELATION_ID); + IndexDefinition thatIndex = other.getIndex(tx, LABEL_RELATION, PROP_RELATION_ID); + Label thisLabelHash = OSMDataset.hashedLabelFrom(thisIndex.getName()); + Label thatLabelHash = OSMDataset.hashedLabelFrom(thatIndex.getName()); + return mergeOSMEntities(tx, "relations", PROP_RELATION_ID, thisLabelHash, thatLabelHash, + this::relationOverlapMerger); + } + + private void wayOverlapMerger(Node thisWay, Node thatWay) { + // TODO: Merge ways by sorting nodes + } + + private void relationOverlapMerger(Node thisWay, Node thatWay) { + // TODO: Merge relations + } + + private MergeStats mergeOSMEntities(Transaction tx, String entityName, String propertyKey, Label thisLabelHash, + Label thatLabelHash, BiConsumer overlapMerger) { + MergeStats stats = new MergeStats(entityName); + try (ResourceIterator nodes = tx.findNodes(thatLabelHash)) { + OSMGeometryEncoder geometryEncoder = (OSMGeometryEncoder) layer.getGeometryEncoder(); + while (nodes.hasNext()) { + Node thatEntityNode = nodes.next(); + long osm_id = getOSMId(propertyKey, thatEntityNode); + if (osm_id != INVALID_OSM_ID) { + Node thisEntityNode = tx.findNode(thisLabelHash, propertyKey, osm_id); + if (thisEntityNode != null) { + // TODO: consider getting 'timestamp' and always saving the newest data + stats.countReplaced++; + for (String nodePropKey : thatEntityNode.getPropertyKeys()) { + Object value = thatEntityNode.getProperty(nodePropKey); + thisEntityNode.setProperty(nodePropKey, value); + } + Envelope thisEnvelope = getGeometryEnvelope(thisEntityNode, geometryEncoder); + Envelope thatEnvelope = getGeometryEnvelope(thatEntityNode, geometryEncoder); + if (thisEnvelope == null && thatEnvelope == null) { + System.out.printf( + "While merging OSM %s, found nodes with %s = %d which have no geometry on either original or merge nodes%n", + entityName, propertyKey, osm_id); + } else if (thisEnvelope == null) { + System.out.printf( + "While merging OSM %s, found nodes with %s = %d where the original has no geometry, but the merge node has a geometry%n", + entityName, propertyKey, osm_id); + stats.countMoved++; + overlapMerger.accept(thisEntityNode, thatEntityNode); + } else if (thatEnvelope == null) { + System.out.printf( + "While merging OSM %s, found nodes with %s = %d where the original has a geometry, but the merge node does not%n", + entityName, propertyKey, osm_id); + } else if (!thisEnvelope.equals(thatEnvelope)) { + stats.countMoved++; + overlapMerger.accept(thisEntityNode, thatEntityNode); + } + } else { + stats.countAdded++; + thatEntityNode.addLabel(thisLabelHash); + thatEntityNode.removeLabel(thatLabelHash); + } + } else { + System.out.printf("Unexpectedly found OSM %s at %s without property: %s%n", entityName, + thatEntityNode, propertyKey); + } + } + stats.printStats(); + return stats; + } + } + + private Envelope getGeometryEnvelope(Node entityNode, GeometryEncoder geometryEncoder) { + Relationship geomRel = entityNode.getSingleRelationship(OSMRelation.GEOM, Direction.OUTGOING); + return geomRel == null ? null : geometryEncoder.decodeEnvelope(geomRel.getEndNode()); + } +} diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMModel.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMModel.java new file mode 100644 index 00000000..872c2bc6 --- /dev/null +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMModel.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j Spatial. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gis.spatial.osm; + +import org.neo4j.graphdb.Label; + +public class OSMModel { + + public static Label LABEL_DATASET = Label.label("OSMDataset"); + public static Label LABEL_BBOX = Label.label("OSMBBox"); + public static Label LABEL_CHANGESET = Label.label("OSMChangeset"); + public static Label LABEL_USER = Label.label("OSMUser"); + public static Label LABEL_TAGS = Label.label("OSMTags"); + public static Label LABEL_NODE = Label.label("OSMNode"); + public static Label LABEL_GEOM = Label.label("OSMGeometry"); + public static Label LABEL_WAY = Label.label("OSMWay"); + public static Label LABEL_WAY_NODE = Label.label("OSMWayNode"); + public static Label LABEL_RELATION = Label.label("OSMRelation"); + public static String PROP_BBOX = "bbox"; + public static String PROP_TIMESTAMP = "timestamp"; + public static String PROP_CHANGESET = "changeset"; + public static String PROP_USER_NAME = "user"; + public static String PROP_USER_ID = "uid"; + public static String PROP_NODE_ID = "node_osm_id"; + public static String PROP_NODE_LON = "lon"; + public static String PROP_NODE_LAT = "lat"; + public static String PROP_WAY_ID = "way_osm_id"; + public static String PROP_RELATION_ID = "relation_osm_id"; +} diff --git a/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java b/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java index 6baf843c..fb9f4dff 100644 --- a/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java +++ b/src/main/java/org/neo4j/gis/spatial/osm/OSMRelation.java @@ -17,23 +17,11 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -/* AWE - Amanzi Wireless Explorer - * http://awe.amanzi.org - * (C) 2008-2009, AmanziTel AB - * - * This library is provided under the terms of the Eclipse Public License - * as described at http://www.eclipse.org/legal/epl-v10.html. Any use, - * reproduction or distribution of the library constitutes recipient's - * acceptance of this agreement. - * - * This library is distributed WITHOUT ANY WARRANTY; without even the - * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - */ package org.neo4j.gis.spatial.osm; import org.neo4j.graphdb.RelationshipType; public enum OSMRelation implements RelationshipType { - FIRST_NODE, LAST_NODE, OTHER, NEXT, OSM, WAYS, RELATIONS, MEMBERS, MEMBER, TAGS, GEOM, BBOX, NODE, CHANGESET, USER, USERS, OSM_USER + FIRST_NODE, LAST_NODE, OTHER, NEXT, OSM, MEMBERS, MEMBER, TAGS, GEOM, BBOX, NODE, CHANGESET, USER } diff --git a/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java b/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java index bb6a820e..5dbe1285 100644 --- a/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java +++ b/src/main/java/org/neo4j/gis/spatial/procedures/SpatialProcedures.java @@ -66,6 +66,7 @@ import org.neo4j.gis.spatial.index.LayerGeohashPointIndex; import org.neo4j.gis.spatial.index.LayerHilbertPointIndex; import org.neo4j.gis.spatial.index.LayerZOrderPointIndex; +import org.neo4j.gis.spatial.merge.MergeUtils; import org.neo4j.gis.spatial.osm.OSMGeometryEncoder; import org.neo4j.gis.spatial.osm.OSMImporter; import org.neo4j.gis.spatial.osm.OSMLayer; @@ -767,6 +768,16 @@ public void run() { } } + @Procedure(value = "spatial.merge.into", mode = WRITE) + @Description("Merges two layers by copying geometries from the second layer into the first and deleting the second layer") + public Stream mergeLayerIntoLayer( + @Name("layerName") String layerName, + @Name("toMerge") String mergeName) { + EditableLayer layer = getEditableLayerOrThrow(tx, spatial(), layerName); + EditableLayer mergeLayer = getEditableLayerOrThrow(tx, spatial(), mergeName); + return Stream.of(new CountResult(MergeUtils.mergeLayerInto(tx, layer, mergeLayer))); + } + @Procedure(value = "spatial.bbox", mode = WRITE) @Description("Finds all geometry nodes in the given layer within the lower left and upper right coordinates of a box") public Stream findGeometriesInBBox( diff --git a/src/main/java/org/neo4j/gis/spatial/rtree/RTreeIndex.java b/src/main/java/org/neo4j/gis/spatial/rtree/RTreeIndex.java index d76158c1..0f80f443 100644 --- a/src/main/java/org/neo4j/gis/spatial/rtree/RTreeIndex.java +++ b/src/main/java/org/neo4j/gis/spatial/rtree/RTreeIndex.java @@ -707,7 +707,7 @@ public void removeAll(Transaction tx, final boolean deleteGeomNodes, final Liste @Override public void clear(Transaction tx, final Listener monitor) { - removeAll(tx, false, new NullListener()); + removeAll(tx, false, monitor); initIndexRoot(tx); initIndexMetadata(tx); } diff --git a/src/test/java/org/neo4j/gis/spatial/LayerMergeTest.java b/src/test/java/org/neo4j/gis/spatial/LayerMergeTest.java new file mode 100644 index 00000000..8f57ffde --- /dev/null +++ b/src/test/java/org/neo4j/gis/spatial/LayerMergeTest.java @@ -0,0 +1,393 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j Spatial. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gis.spatial; + +import static java.lang.String.format; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME; + +import java.io.File; +import java.util.ArrayList; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.CoordinateXY; +import org.locationtech.jts.geom.Geometry; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gis.spatial.index.IndexManager; +import org.neo4j.gis.spatial.merge.MergeUtils; +import org.neo4j.gis.spatial.osm.OSMDataset; +import org.neo4j.gis.spatial.osm.OSMGeometryEncoder; +import org.neo4j.gis.spatial.osm.OSMLayer; +import org.neo4j.gis.spatial.osm.OSMModel; +import org.neo4j.gis.spatial.rtree.ProgressLoggingListener; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Transaction; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.test.TestDatabaseManagementServiceBuilder; + +public class LayerMergeTest { + + private DatabaseManagementService databases; + private GraphDatabaseService graphDb; + private SpatialDatabaseService spatial; + + @BeforeEach + public void setup() throws KernelException { + databases = new TestDatabaseManagementServiceBuilder(new File("target/layers").toPath()).impermanent().build(); + graphDb = databases.database(DEFAULT_DATABASE_NAME); + spatial = new SpatialDatabaseService( + new IndexManager((GraphDatabaseAPI) graphDb, SecurityContext.AUTH_DISABLED)); + } + + @AfterEach + public void teardown() { + databases.shutdown(); + } + + @Test + public void shouldMergeEmptyWKBLayers() { + shouldMergeLayersWithGeometries(false, 0, 0, this::makeWKBLayer); + } + + @Test + public void shouldMergeWKBLayersWithGeometries() { + shouldMergeLayersWithGeometries(false, 10, 8, this::makeWKBLayer); + } + + @Test + public void shouldMergeEmptyWKTLayers() { + shouldMergeLayersWithGeometries(false, 0, 0, this::makeWKTLayer); + } + + @Test + public void shouldMergeWKTLayersWithGeometries() { + shouldMergeLayersWithGeometries(false, 10, 8, this::makeWKTLayer); + } + + @Test + public void shouldMergeWKTLayersIntoWKBLayer() { + shouldMergeLayersWithGeometries(false, 10, 8, this::makeWKBLayer, this::makeWKTLayer); + } + + @Test + public void shouldMergeWKBLayersIntoWKTLayer() { + shouldMergeLayersWithGeometries(false, 10, 8, this::makeWKTLayer, this::makeWKBLayer); + } + + @Test + public void shouldMergeEmptyOSMLayers() { + shouldMergeOSMLayersWithGeometries(false, 0, 0); + } + + @Test + public void shouldMergeOSMLayersWithOneGeometry() { + shouldMergeOSMLayersWithGeometries(false, 1, 8); + } + + @Test + public void shouldMergeOSMLayersWithManyGeometries() { + shouldMergeOSMLayersWithGeometries(false, 1000, 8); + } + + @Test + public void shouldMergeOSMLayersWithOneIdenticalGeometry() { + shouldMergeOSMLayersWithGeometries(true, 1, 8); + } + + @Test + public void shouldMergeOSMLayersWithManyIdenticalGeometries() { + shouldMergeOSMLayersWithGeometries(true, 1000, 8); + } + + private Layer makeWKBLayer(Transaction tx, String layerName) { + return spatial.createWKBLayer(tx, layerName); + } + + private Layer makeWKTLayer(Transaction tx, String layerName) { + return spatial.createLayer(tx, layerName, WKTGeometryEncoder.class, EditableLayerImpl.class); + } + + private Layer makeOSMLayer(Transaction tx, String layerName) { + return spatial.createLayer(tx, layerName, OSMGeometryEncoder.class, OSMLayer.class); + } + + private void createOSMIndex(OSMDataset dataset, OSMDataset.LabelHasher labelHasher, String layerName, Label label, + String propertyKey) { + Label hashed = labelHasher.getLabelHashed(label); + String indexName = OSMDataset.indexNameFor(layerName, hashed.name(), propertyKey); + // TODO: We should also have tests that verify correct behavior with security-context + try (Transaction indexTx = graphDb.beginTx()) { + indexTx.schema().indexFor(hashed).on(propertyKey).withName(indexName).create(); + indexTx.commit(); + } + try (Transaction tx = graphDb.beginTx()) { + String indexKey = OSMDataset.indexKeyFor(label, propertyKey); + Node node = dataset.getDatasetNode(tx); + node.setProperty(indexKey, indexName); + tx.commit(); + } + } + + private OSMDataset prepareOSMDataset(String layerName, long layerNum, boolean identicalOSMIds) { + OSMDataset dataset; + try (Transaction tx = graphDb.beginTx()) { + Layer layer = spatial.getLayer(tx, layerName); + OSMGeometryEncoder encoder = (OSMGeometryEncoder) layer.getGeometryEncoder(); + if (!identicalOSMIds) { + encoder.configure(tx, 100000 * layerNum, 10000 * layerNum, 1000 * layerNum); + } + dataset = OSMDataset.fromLayer(tx, (OSMLayer) layer); + tx.commit(); + } + return dataset; + } + + private void postCreateOSM(LayerConfig config, Integer layerNum) { + config.postCreate(layerNum); + } + + private void postCreateNone(LayerConfig config, Integer layerNum) { + } + + private class LayerConfig { + + protected boolean identicalLayers; + protected String[] layerNames; + + private LayerConfig(boolean identicalLayers, String... layerNames) { + this.identicalLayers = identicalLayers; + this.layerNames = layerNames; + } + + protected void postCreate(int layerNum) { + } + } + + private class OSMLayerConfig extends LayerConfig { + + private OSMLayerConfig(boolean identicalLayers, String... layerNames) { + super(identicalLayers, layerNames); + } + + @Override + protected void postCreate(int layerNum) { + try { + OSMDataset dataset = prepareOSMDataset(layerNames[layerNum], layerNum, identicalLayers); + OSMDataset.LabelHasher labelHasher = new OSMDataset.LabelHasher(layerNames[layerNum]); + createOSMIndex(dataset, labelHasher, layerNames[layerNum], OSMModel.LABEL_NODE, OSMModel.PROP_NODE_ID); + createOSMIndex(dataset, labelHasher, layerNames[layerNum], OSMModel.LABEL_WAY, OSMModel.PROP_WAY_ID); + createOSMIndex(dataset, labelHasher, layerNames[layerNum], OSMModel.LABEL_RELATION, + OSMModel.PROP_RELATION_ID); + try (Transaction indexTx = graphDb.beginTx()) { + indexTx.schema().awaitIndexesOnline(10, TimeUnit.SECONDS); + indexTx.commit(); + } + } catch (Exception e) { + throw new RuntimeException("Failed to created OSM indexes: " + e.getMessage(), e); + } + } + } + + private final LayerConfig DEFAULT = new LayerConfig(false, "testA", "testB", "testC", "testD"); + + @SuppressWarnings("SameParameterValue") + private void shouldMergeLayersWithGeometries(boolean identicalLayers, int numGeoms, int geomLength, + BiFunction layerMaker) { + shouldMergeLayersWithGeometries(identicalLayers, numGeoms, geomLength, layerMaker, layerMaker, + this::postCreateNone, DEFAULT); + } + + @SuppressWarnings("SameParameterValue") + private void shouldMergeOSMLayersWithGeometries(boolean identicalLayers, int numGeoms, int geomLength) { + BiFunction layerMaker = this::makeOSMLayer; + BiConsumer postCreate = this::postCreateOSM; + LayerConfig config = new OSMLayerConfig(identicalLayers, DEFAULT.layerNames); + shouldMergeLayersWithGeometries(identicalLayers, numGeoms, geomLength, layerMaker, layerMaker, postCreate, + config); + } + + @SuppressWarnings("SameParameterValue") + private void shouldMergeLayersWithGeometries(boolean identicalLayers, int numGeoms, int geomLength, + BiFunction mainMaker, BiFunction otherMaker) { + shouldMergeLayersWithGeometries(identicalLayers, numGeoms, geomLength, mainMaker, otherMaker, + this::postCreateNone, DEFAULT); + } + + private void shouldMergeLayersWithGeometries(boolean identicalLayers, int numGeoms, int geomLength, + BiFunction mainMaker, BiFunction otherMaker, + BiConsumer postCreate, LayerConfig layerConfig) { + double scale = 0.01; + boolean verbose = numGeoms < 10; + ArrayList> allAdded = new ArrayList<>(); + for (int l = 0; l < layerConfig.layerNames.length; l++) { + String layerName = layerConfig.layerNames[l]; + + // First create an empty layer + BiFunction layerMaker = l == 0 ? mainMaker : otherMaker; + inTx(tx -> layerMaker.apply(tx, layerName)); + + // Some models (OSM) need a special post-create setup. It cannot happen in the create step, because it makes Neo4j indexes which have to happen in a different transaction + postCreate.accept(layerConfig, l); + + // Now populate the layer with sample data + ArrayList geometries = new ArrayList<>(); + if (numGeoms > 0) { + double x = identicalLayers ? 0 : scale * scale * l; + double y = identicalLayers ? 0 : scale * scale * l; + inTx(tx -> { + Layer layer = spatial.getLayer(tx, layerName); + assertNotNull(layer); + assertThat(format("Layer %s should be editable", layerName), layer instanceof EditableLayer); + EditableLayer editable = (EditableLayer) layer; + for (int i = 0; i < numGeoms; i++) { + Coordinate[] coordinates = new Coordinate[geomLength]; + for (int j = 0; j < geomLength; j++) { + // Make a horizontal lineString, offset by both the layer number and the geometry number + coordinates[j] = new CoordinateXY(x + scale * j, y + scale * i); + } + Geometry geometry = layer.getGeometryFactory().createLineString(coordinates); + geometries.add(geometry); + editable.add(tx, geometry); + } + }); + } + allAdded.add(geometries); + + // Finally check that the layer has the number of geometries that were added + inTx(tx -> { + Layer layer = spatial.getLayer(tx, layerName); + int count = layer.getIndex().count(tx); + assertThat(format("Expected layer %s to have %d geometries, but found %d", layerName, numGeoms, count), + count == numGeoms); + }); + } + + // Test that all layers can be merged into the first layer, with all geometries moved over + for (int i = 1; i < layerConfig.layerNames.length; i++) { + String mainName = layerConfig.layerNames[0]; + String fromName = layerConfig.layerNames[i]; + inTx(tx -> { + EditableLayer main = (EditableLayer) spatial.getLayer(tx, mainName); + EditableLayer from = (EditableLayer) spatial.getLayer(tx, fromName); + IndexStateCapture other = new IndexStateCapture(tx, from); + IndexStateCapture before = new IndexStateCapture(tx, main); + long merged = MergeUtils.mergeLayerInto(tx, main, from); + IndexStateCapture after = new IndexStateCapture(tx, main); + System.out.printf("Before merge %s->%s there were %d geometries and after merge there were %d%n", + fromName, mainName, before.size(), after.size()); + if (verbose) { + before.debug("before"); + other.debug("other"); + after.debug("after"); + } + // OSM does a more complex merge and generates more changes than other storage layers + long expected = identicalLayers ? 0 + : (main instanceof OSMLayer) ? (long) numGeoms * (geomLength + 1) : numGeoms; + assertThat(format("Expected to merge %d geometries into %s from %s, but merged %d", numGeoms, mainName, + fromName, merged), merged == expected); + }); + } + + // After completing the merger, test all layers have the expected geometry counts (all geoms in first layer, and none in the others) + for (int i = 0; i < layerConfig.layerNames.length; i++) { + String layerName = layerConfig.layerNames[i]; + int expected = i == 0 ? (identicalLayers ? 1 : layerConfig.layerNames.length) * numGeoms : 0; + inTx(tx -> { + Layer layer = spatial.getLayer(tx, layerName); + int count = layer.getIndex().count(tx); + if (count != expected) { + if (expected != 0) { + System.out.printf("Expected layer %s to have %d geometries, but found %d", layerName, expected, + count); + IndexStateCapture found = new IndexStateCapture(tx, layer); + for (int l = 0; l < allAdded.size(); l++) { + found.validate(layerConfig.layerNames[l], allAdded.get(l)); + } + } + } + assertThat(format("Expected layer %s to have %d geometries, but found %d", layerName, expected, count), + count == expected); + }); + } + + // Cleanup (delete layers) + for (String layerName : layerConfig.layerNames) { + inTx(tx -> spatial.deleteLayer(tx, layerName, + new ProgressLoggingListener("deleting layer '" + layerName + "'", System.out))); + inTx(tx -> assertNull(spatial.getLayer(tx, layerName))); + } + } + + private static class IndexStateCapture { + + ArrayList geometries = new ArrayList<>(); + ArrayList geomNodes = new ArrayList<>(); + + private IndexStateCapture(Transaction tx, Layer layer) { + for (Node node : layer.getIndex().getAllIndexedNodes(tx)) { + geomNodes.add(node); + geometries.add(layer.getGeometryEncoder().decodeGeometry(node)); + } + } + + private boolean contains(Geometry geometry) { + return geometries.contains(geometry); + } + + private int size() { + return geomNodes.size(); + } + + public void validate(String layerName, ArrayList added) { + for (Geometry expectedGeometry : added) { + if (!contains(expectedGeometry)) { + System.out.printf("\tFailed to find expected geometry '%s' from layer %s%n", expectedGeometry, + layerName); + } + } + } + + public void debug(String title) { + System.out.printf("Index has %d nodes and geometries: %s%n", size(), title); + for (int i = 0; i < size(); i++) { + System.out.printf("\t%s: %s%n", geomNodes.get(i), geometries.get(i)); + } + } + } + + private void inTx(Consumer txFunction) { + try (Transaction tx = graphDb.beginTx()) { + txFunction.accept(tx); + tx.commit(); + } + } +} diff --git a/src/test/java/org/neo4j/gis/spatial/LayersTest.java b/src/test/java/org/neo4j/gis/spatial/LayersTest.java index 058e4f5b..fbb65f9d 100644 --- a/src/test/java/org/neo4j/gis/spatial/LayersTest.java +++ b/src/test/java/org/neo4j/gis/spatial/LayersTest.java @@ -263,8 +263,6 @@ public void testEditableLayers() { EditableLayerImpl.class); testSpecificEditableLayer("test editable layer with graph encoder", SimpleGraphEncoder.class, EditableLayerImpl.class); - testSpecificEditableLayer("test editable layer with OSM encoder", OSMGeometryEncoder.class, - EditableLayerImpl.class); } private String testSpecificEditableLayer(String layerName, Class geometryEncoderClass, diff --git a/src/test/java/org/neo4j/gis/spatial/OsmAnalysisTest.java b/src/test/java/org/neo4j/gis/spatial/OsmAnalysisTest.java index f93d8613..c91242f3 100644 --- a/src/test/java/org/neo4j/gis/spatial/OsmAnalysisTest.java +++ b/src/test/java/org/neo4j/gis/spatial/OsmAnalysisTest.java @@ -19,8 +19,9 @@ */ package org.neo4j.gis.spatial; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_CHANGESET; + import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -47,7 +48,6 @@ import org.neo4j.gis.spatial.filter.SearchRecords; import org.neo4j.gis.spatial.index.IndexManager; import org.neo4j.gis.spatial.osm.OSMDataset; -import org.neo4j.gis.spatial.osm.OSMImporter; import org.neo4j.gis.spatial.osm.OSMLayer; import org.neo4j.gis.spatial.osm.OSMRelation; import org.neo4j.gis.spatial.rtree.Envelope; @@ -130,7 +130,7 @@ protected void runAnalysis(String osm, int years, int days) throws Exception { testAnalysis2(osm, years, days); } - public void testAnalysis2(String osm, int years, int days) throws IOException { + public void testAnalysis2(String osm, int years, int days) throws Exception { SpatialDatabaseService spatial = new SpatialDatabaseService( new IndexManager((GraphDatabaseAPI) graphDb(), SecurityContext.AUTH_DISABLED)); LinkedHashMap slides = new LinkedHashMap<>(); @@ -145,7 +145,7 @@ public void testAnalysis2(String osm, int years, int days) throws IOException { for (Node cNode : dataset.getAllChangesetNodes(tx)) { long timestamp = (Long) cNode.getProperty("timestamp", 0L); Node userNode = dataset.getUser(cNode); - String name = (String) userNode.getProperty("name"); + String name = (String) userNode.getProperty("user"); User user = userIndex.get(name); if (user == null) { @@ -254,17 +254,18 @@ public void testAnalysis2(String osm, int years, int days) throws IOException { } public void testAnalysis(String osm) throws Exception { + + SpatialDatabaseService spatial = new SpatialDatabaseService( + new IndexManager((GraphDatabaseAPI) graphDb(), SecurityContext.AUTH_DISABLED)); SortedMap layers; ReferencedEnvelope bbox; try (Transaction tx = graphDb().beginTx()) { - Node osmImport = tx.findNode(OSMImporter.LABEL_DATASET, "name", osm); - Node usersNode = osmImport.getSingleRelationship(OSMRelation.USERS, Direction.OUTGOING).getEndNode(); + OSMLayer layer = (OSMLayer) spatial.getLayer(tx, osm); + OSMDataset dataset = OSMDataset.fromLayer(tx, layer); - Map userIndex = collectUserChangesetData(usersNode); + Map userIndex = collectUserChangesetData(dataset.getAllUserNodes(tx)); SortedSet topTen = getTopTen(userIndex); - SpatialDatabaseService spatial = new SpatialDatabaseService( - new IndexManager((GraphDatabaseAPI) graphDb(), SecurityContext.AUTH_DISABLED)); layers = exportPoints(tx, osm, spatial, topTen); layers = removeEmptyLayers(tx, layers); @@ -420,23 +421,19 @@ private SortedSet getTopTen(Map userIndex) { return topTen; } - private Map collectUserChangesetData(Node usersNode) { + private Map collectUserChangesetData(Iterable userNodes) { Map userIndex = new HashMap<>(); - for (Relationship r : usersNode.getRelationships(Direction.OUTGOING, OSMRelation.OSM_USER)) { - Node userNode = r.getEndNode(); + for (Node userNode : userNodes) { String name = (String) userNode.getProperty("name"); - User user = new User(userNode.getElementId(), name); userIndex.put(name, user); - for (Relationship ur : userNode.getRelationships(Direction.INCOMING, OSMRelation.USER)) { Node node = ur.getStartNode(); - if (node.hasProperty("changeset")) { + if (node.hasProperty(PROP_CHANGESET)) { user.changesets.add(node.getElementId()); } } } - return userIndex; } diff --git a/src/test/java/org/neo4j/gis/spatial/TestOSMImport.java b/src/test/java/org/neo4j/gis/spatial/TestOSMImport.java index 10572e40..f7c0c7ce 100644 --- a/src/test/java/org/neo4j/gis/spatial/TestOSMImport.java +++ b/src/test/java/org/neo4j/gis/spatial/TestOSMImport.java @@ -105,6 +105,8 @@ public void buildDataModel() { debugNode(n1); debugNode(n2); tx.commit(); + } catch (Exception e) { + throw new SpatialDatabaseException("Failed to check OSM layer:" + e.getMessage(), e); } try (Transaction tx = this.graphDb().beginTx()) { for (Node n : tx.getAllNodes()) { diff --git a/src/test/java/org/neo4j/gis/spatial/TestSpatial.java b/src/test/java/org/neo4j/gis/spatial/TestSpatial.java index 87be13e7..e6bb37f3 100644 --- a/src/test/java/org/neo4j/gis/spatial/TestSpatial.java +++ b/src/test/java/org/neo4j/gis/spatial/TestSpatial.java @@ -369,6 +369,8 @@ private void testSpatialIndex(String layerName) { System.out.println( "Total time for index test: " + 1.0 * (System.currentTimeMillis() - start) / 1000.0 + "s"); tx.commit(); + } catch (Exception e) { + throw new SpatialDatabaseException("Failed to run index test: " + e.getMessage(), e); } } diff --git a/src/test/java/org/neo4j/gis/spatial/pipes/GeoPipesDocTest.java b/src/test/java/org/neo4j/gis/spatial/pipes/GeoPipesDocTest.java index be7e8d13..cb51d50a 100644 --- a/src/test/java/org/neo4j/gis/spatial/pipes/GeoPipesDocTest.java +++ b/src/test/java/org/neo4j/gis/spatial/pipes/GeoPipesDocTest.java @@ -146,7 +146,7 @@ public void filter_by_cql_using_bbox() throws CQLException { */ @Test @Documented("search_within_geometry") - public void search_within_geometry() throws CQLException { + public void search_within_geometry() { // tag::search_within_geometry[] GeoPipeline pipeline = GeoPipeline .startWithinSearch(tx, osmLayer, @@ -211,12 +211,10 @@ public void translate_geometries() { // end::affine_transformation[] addImageSnippet(boxesLayer, pipeline, getTitle()); - GeoPipeline original = GeoPipeline.start(tx, osmLayer).copyDatabaseRecordProperties(tx).sort( - "name"); + GeoPipeline original = GeoPipeline.start(tx, osmLayer).copyDatabaseRecordProperties(tx).sort("name"); GeoPipeline translated = GeoPipeline.start(tx, osmLayer).applyAffineTransform( - AffineTransformation.translationInstance(10, 25)).copyDatabaseRecordProperties(tx).sort( - "name"); + AffineTransformation.translationInstance(10, 25)).copyDatabaseRecordProperties(tx).sort("name"); for (int k = 0; k < 2; k++) { Coordinate[] coords = original.next().getGeometry().getCoordinates(); @@ -680,7 +678,7 @@ public void compute_distance() throws ParseException { */ @Documented("unite_all") @Test - public void unite_all() { + public void unite_all() throws ParseException { // tag::unite_all[] GeoPipeline pipeline = GeoPipeline.start(tx, intersectionLayer).unionAll(); // end::unite_all[] @@ -690,8 +688,8 @@ public void unite_all() { .unionAll() .createWellKnownText(); - assertEquals("POLYGON ((2 5, 2 6, 4 6, 4 10, 10 10, 10 4, 6 4, 6 2, 5 2, 5 0, 0 0, 0 5, 2 5))", - pipeline.next().getProperty(tx, "WellKnownText")); + assertWKTGeometryEquals(intersectionLayer, pipeline, + "POLYGON ((2 5, 2 6, 4 6, 4 10, 10 10, 10 4, 6 4, 6 2, 5 2, 5 0, 0 0, 0 5, 2 5))"); try { pipeline.next(); @@ -714,7 +712,7 @@ public void unite_all() { */ @Documented("intersect_all") @Test - public void intersect_all() { + public void intersect_all() throws ParseException { // tag::intersect_all[] GeoPipeline pipeline = GeoPipeline.start(tx, intersectionLayer).intersectAll(); // end::intersect_all[] @@ -724,7 +722,7 @@ public void intersect_all() { .intersectAll() .createWellKnownText(); - assertEquals("POLYGON ((4 5, 5 5, 5 4, 4 4, 4 5))", pipeline.next().getProperty(tx, "WellKnownText")); + assertWKTGeometryEquals(intersectionLayer, pipeline, "POLYGON ((4 5, 5 5, 5 4, 4 4, 4 5))"); try { pipeline.next(); @@ -972,16 +970,14 @@ private static void load() throws Exception { equalLayer = (EditableLayerImpl) spatial.getOrCreateEditableLayer(tx, "equal"); equalLayer.setExtraPropertyNames(new String[]{"id", "name"}, tx); equalLayer.setCoordinateReferenceSystem(tx, DefaultEngineeringCRS.GENERIC_2D); - reader = new WKTReader(intersectionLayer.getGeometryFactory()); - equalLayer.add(tx, reader.read("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))"), - new String[]{"id", "name"}, new Object[]{1, "equal"}); - equalLayer.add(tx, reader.read("POLYGON ((0 0, 0.1 5, 5 5, 5 0, 0 0))"), - new String[]{"id", "name"}, new Object[]{2, "tolerance"}); - equalLayer.add(tx, reader.read("POLYGON ((0 5, 5 5, 5 0, 0 0, 0 5))"), - new String[]{"id", "name"}, new Object[]{3, - "different order"}); - equalLayer.add(tx, - reader.read("POLYGON ((0 0, 0 2, 0 4, 0 5, 5 5, 5 3, 5 2, 5 0, 0 0))"), + reader = new WKTReader(equalLayer.getGeometryFactory()); + equalLayer.add(tx, reader.read("POLYGON ((0 0, 0 5, 5 5, 5 0, 0 0))"), new String[]{"id", "name"}, + new Object[]{1, "equal"}); + equalLayer.add(tx, reader.read("POLYGON ((0 0, 0.1 5, 5 5, 5 0, 0 0))"), new String[]{"id", "name"}, + new Object[]{2, "tolerance"}); + equalLayer.add(tx, reader.read("POLYGON ((0 5, 5 5, 5 0, 0 0, 0 5))"), new String[]{"id", "name"}, + new Object[]{3, "different order"}); + equalLayer.add(tx, reader.read("POLYGON ((0 0, 0 2, 0 4, 0 5, 5 5, 5 3, 5 2, 5 0, 0 0))"), new String[]{"id", "name"}, new Object[]{4, "topo equal"}); linesLayer = (EditableLayerImpl) spatial.getOrCreateEditableLayer(tx, "lines"); @@ -994,17 +990,28 @@ private static void load() throws Exception { } @SuppressWarnings("SameParameterValue") - private static void loadTestOsmData(String layerName, int commitInterval) - throws Exception { + private static void loadTestOsmData(String layerName, int commitInterval) throws Exception { String osmPath = "./" + layerName; - System.out.println("\n=== Loading layer " + layerName + " from " - + osmPath + " ==="); + System.out.println("\n=== Loading layer " + layerName + " from " + osmPath + " ==="); OSMImporter importer = new OSMImporter(layerName); importer.setCharset(StandardCharsets.UTF_8); importer.importFile(db, osmPath); importer.reIndex(db, commitInterval); } + private void assertWKTGeometryEquals(EditableLayerImpl layer, GeoPipeline pipeline, String expectedWKT) + throws ParseException { + WKTReader reader = new WKTReader(layer.getGeometryFactory()); + Geometry expected = reader.read(expectedWKT); + Geometry actual = reader.read(pipeline.next().getProperty(tx, "WellKnownText").toString()); + assertEquals(expected.getGeometryType(), actual.getGeometryType(), "Expected matching geometry types"); + assertEquals(expected.getArea(), actual.getArea(), 0.000001, "Expected matching geometry areas"); + // JTS will handle different starting coordinates for matching geometries, so we check with JTS first, and only if that fails run the assertion to get the appropriate error message + if (!expected.equals(actual)) { + assertEquals(expected, actual, "Expected matching geometries"); + } + } + @BeforeEach public void setUp() { gen.get().setGraph(db); diff --git a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java index eef17156..b2f44a12 100644 --- a/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java +++ b/src/test/java/org/neo4j/gis/spatial/procedures/SpatialProceduresTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gis.spatial.procedures; +import static java.util.Collections.emptyMap; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.closeTo; @@ -30,24 +31,28 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import static org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME; import static org.neo4j.gis.spatial.Constants.LABEL_LAYER; import static org.neo4j.gis.spatial.Constants.PROP_GEOMENCODER; import static org.neo4j.gis.spatial.Constants.PROP_GEOMENCODER_CONFIG; import static org.neo4j.gis.spatial.Constants.PROP_LAYER; import static org.neo4j.gis.spatial.Constants.PROP_LAYER_CLASS; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_NODE_ID; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_RELATION_ID; +import static org.neo4j.gis.spatial.osm.OSMModel.PROP_WAY_ID; import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Consumer; import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -58,10 +63,15 @@ import org.neo4j.gis.spatial.Layer; import org.neo4j.gis.spatial.SpatialDatabaseService; import org.neo4j.gis.spatial.SpatialRelationshipTypes; +import org.neo4j.gis.spatial.index.Envelope; import org.neo4j.gis.spatial.index.IndexManager; +import org.neo4j.gis.spatial.osm.OSMRelation; import org.neo4j.gis.spatial.utilities.ReferenceNodes; +import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.ResourceIterator; import org.neo4j.graphdb.Result; import org.neo4j.graphdb.Transaction; @@ -121,9 +131,9 @@ public static void testCallFails(GraphDatabaseService db, String call, Map { int numLeft = count; while (numLeft > 0) { - assertTrue(res.hasNext(), + Assertions.assertTrue(res.hasNext(), "Expected " + count + " results but found only " + (count - numLeft)); res.next(); numLeft--; @@ -242,9 +252,9 @@ public void add_node_point_layer() { Node node = createNode("MATCH (n:Point) WITH n CALL spatial.addNode('points',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.bbox('points',{longitude:15.0,latitude:60.0},{longitude:15.3, latitude:60.2})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); testCall(db, "CALL spatial.withinDistance('points',{longitude:15.0,latitude:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -254,17 +264,19 @@ public void add_node_and_search_bbox_and_distance() { "CREATE (n:Node {lat:60.1,lon:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0},{lon:15.3, lat:60.2})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); testCall(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test // This tests issue https://github.com/neo4j-contrib/spatial/issues/298 public void add_node_point_layer_and_search_multiple_points_precision() { execute("CALL spatial.addPointLayer('bar')"); - execute("create (n:Point) set n={latitude: 52.2029252, longitude: 0.0905302} with n call spatial.addNode('bar', n) yield node return node"); - execute("create (n:Point) set n={latitude: 52.202925, longitude: 0.090530} with n call spatial.addNode('bar', n) yield node return node"); + execute( + "create (n:Point) set n={latitude: 52.2029252, longitude: 0.0905302} with n call spatial.addNode('bar', n) yield node return node"); + execute( + "create (n:Point) set n={latitude: 52.202925, longitude: 0.090530} with n call spatial.addNode('bar', n) yield node return node"); // long countLow = execute("call spatial.withinDistance('bar', {latitude:52.202925,longitude:0.0905302}, 100) YIELD node RETURN node"); // assertThat("Expected two nodes when using low precision", countLow, equalTo(2L)); long countHigh = execute( @@ -279,9 +291,9 @@ public void add_node_and_search_bbox_and_distance_geohash() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0},{lon:15.3, lat:60.2})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); testCall(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -291,9 +303,9 @@ public void add_node_and_search_bbox_and_distance_zorder() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0},{lon:15.3, lat:60.2})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); testCall(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -303,22 +315,24 @@ public void add_node_and_search_bbox_and_distance_hilbert() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0},{lon:15.3, lat:60.2})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); testCall(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test // This tests issue https://github.com/neo4j-contrib/spatial/issues/298 public void add_node_point_layer_and_search_multiple_points_precision_geohash() { execute("CALL spatial.addPointLayerGeohash('bar')"); - execute("create (n:Point) set n={latitude: 52.2029252, longitude: 0.0905302} with n call spatial.addNode('bar', n) yield node return node"); - execute("create (n:Point) set n={latitude: 52.202925, longitude: 0.090530} with n call spatial.addNode('bar', n) yield node return node"); + execute( + "create (n:Point) set n={latitude: 52.2029252, longitude: 0.0905302} with n call spatial.addNode('bar', n) yield node return node"); + execute( + "create (n:Point) set n={latitude: 52.202925, longitude: 0.090530} with n call spatial.addNode('bar', n) yield node return node"); // long countLow = execute("call spatial.withinDistance('bar', {latitude:52.202925,longitude:0.0905302}, 100) YIELD node RETURN node"); // assertEquals("Expected two nodes when using low precision", countLow, equalTo(2L)); long countHigh = execute( "call spatial.withinDistance('bar', {latitude:52.2029252,longitude:0.0905302}, 100) YIELD node RETURN node"); - assertEquals(2L, countHigh, "Expected two nodes when using high precision"); + Assertions.assertEquals(2L, countHigh, "Expected two nodes when using high precision"); } // @@ -419,9 +433,16 @@ private void executeWrite(String call) { } private Node createNode(String call, String column) { + return createNode(call, null, column); + } + + private Node createNode(String call, Map params, String column) { + if (params == null) { + params = emptyMap(); + } Node node; try (Transaction tx = db.beginTx()) { - ResourceIterator nodes = tx.execute(call).columnAs(column); + ResourceIterator nodes = tx.execute(call, params).columnAs(column); node = (Node) nodes.next(); nodes.close(); tx.commit(); @@ -455,13 +476,13 @@ private Object executeObject(String call, Map params, String col @Test public void create_a_pointlayer_with_x_and_y() { testCall(db, "CALL spatial.addPointLayerXY('geom','lon','lat')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); } @Test public void create_a_pointlayer_with_config() { testCall(db, "CALL spatial.addPointLayerWithConfig('geom','lon:lat')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); } @Test @@ -469,10 +490,10 @@ public void create_a_pointlayer_with_config_on_existing_wkt_layer() { execute("CALL spatial.addWKTLayer('geom','wkt')"); try { testCall(db, "CALL spatial.addPointLayerWithConfig('geom','lon:lat')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); - fail("Expected exception to be thrown"); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + Assertions.fail("Expected exception to be thrown"); } catch (Exception e) { - assertTrue(e.getMessage().contains("Cannot create existing layer")); + Assertions.assertTrue(e.getMessage().contains("Cannot create existing layer")); } } @@ -481,41 +502,41 @@ public void create_a_pointlayer_with_config_on_existing_osm_layer() { execute("CALL spatial.addLayer('geom','OSM','')"); try { testCall(db, "CALL spatial.addPointLayerWithConfig('geom','lon:lat')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); - fail("Expected exception to be thrown"); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + Assertions.fail("Expected exception to be thrown"); } catch (Exception e) { - assertTrue(e.getMessage().contains("Cannot create existing layer")); + Assertions.assertTrue(e.getMessage().contains("Cannot create existing layer")); } } @Test public void create_a_pointlayer_with_rtree() { testCall(db, "CALL spatial.addPointLayer('geom')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); } @Test public void create_a_pointlayer_with_geohash() { testCall(db, "CALL spatial.addPointLayerGeohash('geom')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); } @Test public void create_a_pointlayer_with_zorder() { testCall(db, "CALL spatial.addPointLayerZOrder('geom')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); } @Test public void create_a_pointlayer_with_hilbert() { testCall(db, "CALL spatial.addPointLayerHilbert('geom')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); } @Test public void create_and_delete_a_pointlayer_with_rtree() { testCall(db, "CALL spatial.addPointLayer('geom')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); testCallCount(db, "CALL spatial.layers()", null, 1); execute("CALL spatial.removeLayer('geom')"); testCallCount(db, "CALL spatial.layers()", null, 0); @@ -524,7 +545,7 @@ public void create_and_delete_a_pointlayer_with_rtree() { @Test public void create_and_delete_a_pointlayer_with_geohash() { testCall(db, "CALL spatial.addPointLayerGeohash('geom')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); testCallCount(db, "CALL spatial.layers()", null, 1); execute("CALL spatial.removeLayer('geom')"); testCallCount(db, "CALL spatial.layers()", null, 0); @@ -533,7 +554,7 @@ public void create_and_delete_a_pointlayer_with_geohash() { @Test public void create_and_delete_a_pointlayer_with_zorder() { testCall(db, "CALL spatial.addPointLayerZOrder('geom')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); testCallCount(db, "CALL spatial.layers()", null, 1); execute("CALL spatial.removeLayer('geom')"); testCallCount(db, "CALL spatial.layers()", null, 0); @@ -542,7 +563,7 @@ public void create_and_delete_a_pointlayer_with_zorder() { @Test public void create_and_delete_a_pointlayer_with_hilbert() { testCall(db, "CALL spatial.addPointLayerHilbert('geom')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); testCallCount(db, "CALL spatial.layers()", null, 1); execute("CALL spatial.removeLayer('geom')"); testCallCount(db, "CALL spatial.layers()", null, 0); @@ -552,11 +573,11 @@ public void create_and_delete_a_pointlayer_with_hilbert() { public void create_a_simple_pointlayer_using_named_encoder() { testCall(db, "CALL spatial.addLayerWithEncoder('geom','SimplePointEncoder','')", (r) -> { Node node = dump((Node) r.get("node")); - assertEquals("geom", node.getProperty("layer")); - assertEquals("org.neo4j.gis.spatial.encoders.SimplePointEncoder", + Assertions.assertEquals("geom", node.getProperty("layer")); + Assertions.assertEquals("org.neo4j.gis.spatial.encoders.SimplePointEncoder", node.getProperty("geomencoder")); - assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty("layer_class")); - assertFalse(node.hasProperty(PROP_GEOMENCODER_CONFIG)); + Assertions.assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty("layer_class")); + Assertions.assertFalse(node.hasProperty(PROP_GEOMENCODER_CONFIG)); }); } @@ -564,11 +585,11 @@ public void create_a_simple_pointlayer_using_named_encoder() { public void create_a_simple_pointlayer_using_named_and_configured_encoder() { testCall(db, "CALL spatial.addLayerWithEncoder('geom','SimplePointEncoder','x:y:mbr')", (r) -> { Node node = dump((Node) r.get("node")); - assertEquals("geom", node.getProperty(PROP_LAYER)); - assertEquals("org.neo4j.gis.spatial.encoders.SimplePointEncoder", + Assertions.assertEquals("geom", node.getProperty(PROP_LAYER)); + Assertions.assertEquals("org.neo4j.gis.spatial.encoders.SimplePointEncoder", node.getProperty(PROP_GEOMENCODER)); - assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); - assertEquals("x:y:mbr", node.getProperty(PROP_GEOMENCODER_CONFIG)); + Assertions.assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); + Assertions.assertEquals("x:y:mbr", node.getProperty(PROP_GEOMENCODER_CONFIG)); }); } @@ -576,11 +597,11 @@ public void create_a_simple_pointlayer_using_named_and_configured_encoder() { public void create_a_native_pointlayer_using_named_encoder() { testCall(db, "CALL spatial.addLayerWithEncoder('geom','NativePointEncoder','')", (r) -> { Node node = dump((Node) r.get("node")); - assertEquals("geom", node.getProperty(PROP_LAYER)); - assertEquals("org.neo4j.gis.spatial.encoders.NativePointEncoder", + Assertions.assertEquals("geom", node.getProperty(PROP_LAYER)); + Assertions.assertEquals("org.neo4j.gis.spatial.encoders.NativePointEncoder", node.getProperty(PROP_GEOMENCODER)); - assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); - assertFalse(node.hasProperty(PROP_GEOMENCODER_CONFIG)); + Assertions.assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); + Assertions.assertFalse(node.hasProperty(PROP_GEOMENCODER_CONFIG)); }); } @@ -588,11 +609,11 @@ public void create_a_native_pointlayer_using_named_encoder() { public void create_a_native_pointlayer_using_named_and_configured_encoder() { testCall(db, "CALL spatial.addLayerWithEncoder('geom','NativePointEncoder','pos:mbr')", (r) -> { Node node = dump((Node) r.get("node")); - assertEquals("geom", node.getProperty(PROP_LAYER)); - assertEquals("org.neo4j.gis.spatial.encoders.NativePointEncoder", + Assertions.assertEquals("geom", node.getProperty(PROP_LAYER)); + Assertions.assertEquals("org.neo4j.gis.spatial.encoders.NativePointEncoder", node.getProperty(PROP_GEOMENCODER)); - assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); - assertEquals("pos:mbr", node.getProperty(PROP_GEOMENCODER_CONFIG)); + Assertions.assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); + Assertions.assertEquals("pos:mbr", node.getProperty(PROP_GEOMENCODER_CONFIG)); }); } @@ -600,11 +621,11 @@ public void create_a_native_pointlayer_using_named_and_configured_encoder() { public void create_a_native_pointlayer_using_named_and_configured_encoder_with_cartesian() { testCall(db, "CALL spatial.addLayerWithEncoder('geom','NativePointEncoder','pos:mbr:Cartesian')", (r) -> { Node node = dump((Node) r.get("node")); - assertEquals("geom", node.getProperty(PROP_LAYER)); - assertEquals("org.neo4j.gis.spatial.encoders.NativePointEncoder", + Assertions.assertEquals("geom", node.getProperty(PROP_LAYER)); + Assertions.assertEquals("org.neo4j.gis.spatial.encoders.NativePointEncoder", node.getProperty(PROP_GEOMENCODER)); - assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); - assertEquals("pos:mbr:Cartesian", node.getProperty(PROP_GEOMENCODER_CONFIG)); + Assertions.assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); + Assertions.assertEquals("pos:mbr:Cartesian", node.getProperty(PROP_GEOMENCODER_CONFIG)); }); } @@ -612,18 +633,18 @@ public void create_a_native_pointlayer_using_named_and_configured_encoder_with_c public void create_a_native_pointlayer_using_named_and_configured_encoder_with_geographic() { testCall(db, "CALL spatial.addLayerWithEncoder('geom','NativePointEncoder','pos:mbr:WGS-84')", (r) -> { Node node = dump((Node) r.get("node")); - assertEquals("geom", node.getProperty(PROP_LAYER)); - assertEquals("org.neo4j.gis.spatial.encoders.NativePointEncoder", + Assertions.assertEquals("geom", node.getProperty(PROP_LAYER)); + Assertions.assertEquals("org.neo4j.gis.spatial.encoders.NativePointEncoder", node.getProperty(PROP_GEOMENCODER)); - assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); - assertEquals("pos:mbr:WGS-84", node.getProperty(PROP_GEOMENCODER_CONFIG)); + Assertions.assertEquals("org.neo4j.gis.spatial.SimplePointLayer", node.getProperty(PROP_LAYER_CLASS)); + Assertions.assertEquals("pos:mbr:WGS-84", node.getProperty(PROP_GEOMENCODER_CONFIG)); }); } @Test public void create_a_wkt_layer_using_know_format() { testCall(db, "CALL spatial.addLayer('geom','WKT',null)", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); } @Test @@ -633,8 +654,8 @@ public void list_layer_names() { execute("CALL spatial.addWKT('geom',$wkt)", map("wkt", wkt)); testCall(db, "CALL spatial.layers()", (r) -> { - assertEquals("geom", r.get("name")); - assertEquals("EditableLayer(name='geom', encoder=WKTGeometryEncoder(geom='wkt', bbox='bbox'))", + Assertions.assertEquals("geom", r.get("name")); + Assertions.assertEquals("EditableLayer(name='geom', encoder=WKTGeometryEncoder(geom='wkt', bbox='bbox'))", r.get("signature")); }); } @@ -687,19 +708,19 @@ public void list_spatial_procedures() { for (String key : procs.keySet()) { System.out.println(key + ": " + procs.get(key)); } - assertEquals("spatial.procedures() :: (name :: STRING, signature :: STRING)", + Assertions.assertEquals("spatial.procedures() :: (name :: STRING, signature :: STRING)", procs.get("spatial.procedures")); - assertEquals("spatial.layers() :: (name :: STRING, signature :: STRING)", + Assertions.assertEquals("spatial.layers() :: (name :: STRING, signature :: STRING)", procs.get("spatial.layers")); - assertEquals("spatial.layer(name :: STRING) :: (node :: NODE)", procs.get("spatial.layer")); - assertEquals( + Assertions.assertEquals("spatial.layer(name :: STRING) :: (node :: NODE)", procs.get("spatial.layer")); + Assertions.assertEquals( "spatial.addLayer(name :: STRING, type :: STRING, encoderConfig :: STRING) :: (node :: NODE)", procs.get("spatial.addLayer")); - assertEquals("spatial.addNode(layerName :: STRING, node :: NODE) :: (node :: NODE)", + Assertions.assertEquals("spatial.addNode(layerName :: STRING, node :: NODE) :: (node :: NODE)", procs.get("spatial.addNode")); - assertEquals("spatial.addWKT(layerName :: STRING, geometry :: STRING) :: (node :: NODE)", + Assertions.assertEquals("spatial.addWKT(layerName :: STRING, geometry :: STRING) :: (node :: NODE)", procs.get("spatial.addWKT")); - assertEquals("spatial.intersects(layerName :: STRING, geometry :: ANY) :: (node :: NODE)", + Assertions.assertEquals("spatial.intersects(layerName :: STRING, geometry :: ANY) :: (node :: NODE)", procs.get("spatial.intersects")); }); } @@ -715,34 +736,34 @@ public void list_layer_types() { for (String key : procs.keySet()) { System.out.println(key + ": " + procs.get(key)); } - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='SimplePoint', geometryEncoder=SimplePointEncoder, layerClass=SimplePointLayer, index=LayerRTreeIndex, crs='WGS84(DD)', defaultConfig='longitude:latitude')", procs.get("simplepoint")); - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='NativePoint', geometryEncoder=NativePointEncoder, layerClass=SimplePointLayer, index=LayerRTreeIndex, crs='WGS84(DD)', defaultConfig='location')", procs.get("nativepoint")); - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='WKT', geometryEncoder=WKTGeometryEncoder, layerClass=EditableLayerImpl, index=LayerRTreeIndex, crs='WGS84(DD)', defaultConfig='geometry')", procs.get("wkt")); - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='WKB', geometryEncoder=WKBGeometryEncoder, layerClass=EditableLayerImpl, index=LayerRTreeIndex, crs='WGS84(DD)', defaultConfig='geometry')", procs.get("wkb")); - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='Geohash', geometryEncoder=SimplePointEncoder, layerClass=SimplePointLayer, index=LayerGeohashPointIndex, crs='WGS84(DD)', defaultConfig='longitude:latitude')", procs.get("geohash")); - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='ZOrder', geometryEncoder=SimplePointEncoder, layerClass=SimplePointLayer, index=LayerZOrderPointIndex, crs='WGS84(DD)', defaultConfig='longitude:latitude')", procs.get("zorder")); - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='Hilbert', geometryEncoder=SimplePointEncoder, layerClass=SimplePointLayer, index=LayerHilbertPointIndex, crs='WGS84(DD)', defaultConfig='longitude:latitude')", procs.get("hilbert")); - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='NativeGeohash', geometryEncoder=NativePointEncoder, layerClass=SimplePointLayer, index=LayerGeohashPointIndex, crs='WGS84(DD)', defaultConfig='location')", procs.get("nativegeohash")); - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='NativeZOrder', geometryEncoder=NativePointEncoder, layerClass=SimplePointLayer, index=LayerZOrderPointIndex, crs='WGS84(DD)', defaultConfig='location')", procs.get("nativezorder")); - assertEquals( + Assertions.assertEquals( "RegisteredLayerType(name='NativeHilbert', geometryEncoder=NativePointEncoder, layerClass=SimplePointLayer, index=LayerHilbertPointIndex, crs='WGS84(DD)', defaultConfig='location')", procs.get("nativehilbert")); }); @@ -755,7 +776,7 @@ public void find_layer() { execute("CALL spatial.addWKT('geom',$wkt)", map("wkt", wkt)); testCall(db, "CALL spatial.layer('geom')", - (r) -> assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); + (r) -> Assertions.assertEquals("geom", (dump((Node) r.get("node"))).getProperty("layer"))); testCallFails(db, "CALL spatial.layer('badname')", null, "No such layer 'badname'"); } @@ -821,9 +842,9 @@ public void add_a_node_to_multiple_different_indexes_for_both_simple_and_native_ String layerName = (encoder + indexType).toLowerCase(); testCall(db, "MATCH (node:Node) RETURN node", r -> assertEquals(node, r.get("node"))); testCall(db, "MATCH (n:Node) WITH n CALL spatial.addNode('" + layerName + "',n) YIELD node RETURN node", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); testCall(db, "CALL spatial.withinDistance('" + layerName + "',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } } for (String encoder : encoders) { @@ -843,7 +864,7 @@ public void testDistanceNode() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -853,7 +874,7 @@ public void testDistanceNodeWithGeohashIndex() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -863,7 +884,7 @@ public void testDistanceNodeGeohash() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -873,7 +894,7 @@ public void testDistanceNodeZOrder() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -883,7 +904,7 @@ public void testDistanceNodeHilbert() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -919,11 +940,11 @@ public void add_two_nodes_to_the_spatial_layer() { tx.commit(); } testResult(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", res -> { - assertTrue(res.hasNext()); - assertEquals(node1, ((Node) res.next().get("node")).getElementId()); - assertTrue(res.hasNext()); - assertEquals(node2, ((Node) res.next().get("node")).getElementId()); - assertFalse(res.hasNext()); + Assertions.assertTrue(res.hasNext()); + Assertions.assertEquals(node1, ((Node) res.next().get("node")).getElementId()); + Assertions.assertTrue(res.hasNext()); + Assertions.assertEquals(node2, ((Node) res.next().get("node")).getElementId()); + Assertions.assertFalse(res.hasNext()); }); try (Transaction tx = db.beginTx()) { Node node = (Node) tx.execute("MATCH (node) WHERE elementId(node) = $nodeId RETURN node", @@ -935,9 +956,9 @@ public void add_two_nodes_to_the_spatial_layer() { tx.commit(); } testResult(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", res -> { - assertTrue(res.hasNext()); - assertEquals(node2, ((Node) res.next().get("node")).getElementId()); - assertFalse(res.hasNext()); + Assertions.assertTrue(res.hasNext()); + Assertions.assertEquals(node2, ((Node) res.next().get("node")).getElementId()); + Assertions.assertFalse(res.hasNext()); }); try (Transaction tx = db.beginTx()) { Result removeResult = tx.execute("CALL spatial.removeNode.byId('geom',$nodeId) YIELD nodeId RETURN nodeId", @@ -947,7 +968,7 @@ public void add_two_nodes_to_the_spatial_layer() { tx.commit(); } testResult(db, "CALL spatial.withinDistance('geom',{lon:15.0,lat:60.0},100)", - res -> assertFalse(res.hasNext())); + res -> Assertions.assertFalse(res.hasNext())); } @Test @@ -1139,7 +1160,7 @@ public void import_osm_and_add_geometry() { Node node = createNode( "CALL spatial.addWKT('geom', 'POINT(6.3740429666 50.93676351666)') YIELD node RETURN node", "node"); testCall(db, "CALL spatial.withinDistance('geom',{lon:6.3740429666,lat:50.93676351666},100)", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); testCallCount(db, "CALL spatial.withinDistance('geom',{lon:6.3740429666,lat:50.93676351666},100)", null, 1); testCallCount(db, "CALL spatial.withinDistance('geom',{lon:6.3740429666,lat:50.93676351666},10000)", null, 218); } @@ -1195,7 +1216,7 @@ public void import_osm_and_polygons_withinDistance() { assertThat("Geometry should contain coordinates", map, hasKey("coordinates")); assertThat("Geometry should not be a point", map.get("type"), not(equalTo("Point"))); } else { - fail("Geometry should be either a point or a Map containing coordinates"); + Assertions.fail("Geometry should be either a point or a Map containing coordinates"); } } }); @@ -1210,33 +1231,279 @@ private void testCountQuery(String name, String query, long count, String column } long start = System.currentTimeMillis(); testResult(db, query, params, res -> { - assertTrue(res.hasNext(), "Expected a single result"); + Assertions.assertTrue(res.hasNext(), "Expected a single result"); long c = (Long) res.next().get(column); - assertFalse(res.hasNext(), "Expected a single result"); - assertEquals(count, c, "Expected count of " + count + " nodes but got " + c); + assertFalse(res.hasNext(), "Expected a single result"); + assertEquals(count, c, "Expected count of " + count + " nodes but got " + c); } ); System.out.println(name + " query took " + (System.currentTimeMillis() - start) + "ms - " + params); } + private Node addPointLayerXYWithNode(String name, double x, double y) { + execute("CALL spatial.addPointLayerXY($name,'lon','lat')", map("name", name)); + Node node = createNode( + "CREATE (n:Node {lat:$lat,lon:$lon}) WITH n CALL spatial.addNode($name,n) YIELD node RETURN node", + map("name", name, "lat", y, "lon", x), "node"); + return node; + } + + private Node addPointLayerWithNode(String name, double x, double y) { + execute("CALL spatial.addPointLayer($name)", map("name", name)); + Node node = createNode( + "CREATE (n:Node {latitude:$lat,longitude:$lon}) WITH n CALL spatial.addNode($name,n) YIELD node RETURN node", + map("name", name, "lat", y, "lon", x), "node"); + return node; + } + + private Node addPointLayerGeohashWithNode(String name, double x, double y) { + execute("CALL spatial.addPointLayerGeohash($name)", map("name", name)); + Node node = createNode( + "CREATE (n:Node {latitude:$lat,longitude:$lon}) WITH n CALL spatial.addNode($name,n) YIELD node RETURN node", + map("name", name, "lat", y, "lon", x), "node"); + return node; + } + + private Node addWKTLayerWithPointNode(String name, double x, double y) { + execute("CALL spatial.addWKTLayer($name,'wkt')", map("name", name)); + Node node = createNode("CALL spatial.addWKT($name,$wkt)", + map("name", name, "wkt", String.format("POINT (%f %f)", x, y)), "node"); + return node; + } + @Test - public void find_geometries_in_a_bounding_box_short() { - execute("CALL spatial.addPointLayerXY('geom','lon','lat')"); + public void merge_layers_of_identical_type() { + Node nodeA = addPointLayerWithNode("geomA", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeA, r.get("node"))); + Node nodeB = addPointLayerWithNode("geomB", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeB, r.get("node"))); + Node nodeC = addPointLayerWithNode("geomC", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeC, r.get("node"))); + + testCall(db, "CALL spatial.merge.into($nameB,$nameC)", map("nameB", "geomB", "nameC", "geomC"), + r -> Assertions.assertEquals(1L, r.get("count"))); + testCallCount(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 1); + testCallCount(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 2); + testCallCount(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + + testCall(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "geomA", "nameB", "geomB"), + r -> Assertions.assertEquals(2L, r.get("count"))); + testCallCount(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 3); + testCallCount(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + testCallCount(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + } + + @Test + public void merge_layers_of_similar_type() { + Node nodeA = addPointLayerXYWithNode("geomA", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeA, r.get("node"))); + Node nodeB = addPointLayerWithNode("geomB", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeB, r.get("node"))); + Node nodeC = addPointLayerGeohashWithNode("geomC", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeC, r.get("node"))); + + testCall(db, "CALL spatial.merge.into($nameB,$nameC)", map("nameB", "geomB", "nameC", "geomC"), + r -> Assertions.assertEquals(1L, r.get("count"))); + testCallCount(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 1); + testCallCount(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 2); + testCallCount(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + + testCall(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "geomA", "nameB", "geomB"), + r -> Assertions.assertEquals(2L, r.get("count"))); + testCallCount(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 3); + testCallCount(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + testCallCount(db, "CALL spatial.bbox('geomC',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", null, 0); + } + + @Test + public void fail_to_merge_layers_of_different_type() { + Node nodeA = addPointLayerXYWithNode("geomA", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeA, r.get("node"))); + Node nodeB = addWKTLayerWithPointNode("geomB", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeB, r.get("node"))); + testCallFails(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "geomA", "nameB", "geomB"), + "layer classes are not compatible"); + } + + @Test + public void fail_to_merge_non_OSM_into_OSM() { + execute("CALL spatial.addLayer('osm','OSM','')"); + testCountQuery("importOSMToLayerAndAddGeometry", "CALL spatial.importOSMToLayer($name,'map.osm')", 55, "count", + map("name", "osm")); + testCallCount(db, "CALL spatial.layers()", null, 1); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", + map("name", "osm", "lon", 15.2, "lat", 60.1), 0); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", + map("name", "osm"), 217); + + // Adding a point to the layer Node node = createNode( - "CREATE (n:Node {lat:60.1,lon:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", - "node"); + "CALL spatial.addWKT($name,$wkt)", + map("name", "osm", "wkt", String.format("POINT (%f %f)", 15.2, 60.1)), "node"); + testCall(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", + map("name", "osm", "lon", 15.2, "lat", 60.1), r -> Assertions.assertEquals(node, r.get("node"))); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", + map("name", "osm", "lon", 15.2, "lat", 60.1), 1); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", + map("name", "osm"), 217); + + Node nodeA = addPointLayerXYWithNode("geomA", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomA',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeA, r.get("node"))); + Node nodeB = addWKTLayerWithPointNode("geomB", 15.2, 60.1); + testCall(db, "CALL spatial.bbox('geomB',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", + r -> Assertions.assertEquals(nodeB, r.get("node"))); + testCallFails(db, "CALL spatial.merge.into($name,$nameA)", map("name", "osm", "nameA", "geomA"), + "Cannot merge non-OSM layer into OSM layer"); + testCallFails(db, "CALL spatial.merge.into($name,$nameB)", map("name", "osm", "nameB", "geomB"), + "Cannot merge non-OSM layer into OSM layer"); + } + + private static class NodeIdRecorder implements Consumer { + + ArrayList nodeIds = new ArrayList<>(); + HashMap osmIds = new HashMap<>(); + HashMap> envelopes = new HashMap<>(); + + @Override + public void accept(Result result) { + while (result.hasNext()) { + Node geom = (Node) result.next().get("node"); + double[] bbox = (double[]) geom.getProperty("bbox"); + Envelope env = new Envelope(bbox[0], bbox[1], bbox[2], bbox[3]); + Relationship geomRel = geom.getSingleRelationship(OSMRelation.GEOM, Direction.INCOMING); + if (geomRel == null) { + System.out.printf("Geometry node %s has no attached model node%n", geom.getElementId()); + } else { + Node node = geomRel.getStartNode(); + nodeIds.add(node.getElementId()); + ArrayList envNodes = envelopes.computeIfAbsent(env, k -> new ArrayList<>()); + envNodes.add(node.getElementId()); + Map properties = node.getAllProperties(); + boolean found = false; + for (String osmIdKey : new String[]{PROP_NODE_ID, PROP_WAY_ID, PROP_RELATION_ID}) { + if (!found) { + if (properties.containsKey(osmIdKey)) { + found = true; + long osmId = (Long) node.getProperty(osmIdKey); + osmIds.put(osmId, node.getElementId()); + } + } + } + if (!found) { + StringBuilder sb = new StringBuilder(); + for (Label label : node.getLabels()) { + if (!sb.isEmpty()) { + sb.append(","); + } + sb.append(label.name()); + } + System.out.printf("Found no OSM id in node %s (%s)%n", node.getElementId(), sb); + } + } + } + } + + public void debug(String name) { + System.out.printf("Results for '%s':%n", name); + System.out.printf("\t%d node ids%n", nodeIds.size()); + System.out.printf("\t%d OSM ids%n", osmIds.size()); + System.out.printf("\t%d envelops%n", envelopes.size()); + for (Envelope env : envelopes.keySet()) { + List ids = envelopes.get(env); + if (ids.size() > 1) { + System.out.printf("\t\t%d entries found in %s%n", ids.size(), env); + } + } + } + + public void shouldContain(NodeIdRecorder expected) { + ArrayList missing = new ArrayList<>(); + ArrayList found = new ArrayList<>(); + for (long bid : expected.osmIds.keySet()) { + if (!this.osmIds.containsKey(bid)) { + //System.out.printf("Failed to find expected node %d in results%n", bid); + missing.add(bid); + } else { + found.add(bid); + } + } + System.out.printf("There were %d/%d found nodes%n", found.size(), this.nodeIds.size()); + System.out.printf("There were %d/%d missing nodes%n", missing.size(), this.nodeIds.size()); + } + } + + @Test + public void should_not_fail_to_merge_OSM_into_very_similar_OSM() { + for (String name : new String[]{"osmA", "osmB"}) { + execute("CALL spatial.addLayer($name,'OSM','')", map("name", name)); + testCountQuery("should_not_fail_to_merge_OSM_into_very_similar_OSM.importOSMToLayer('" + name + "')", + "CALL spatial.importOSMToLayer($name,'map.osm')", 55, "count", map("name", name)); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", + map("name", name, "lon", 15.2, "lat", 60.1), 0); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", + map("name", name), 217); + } + testCallCount(db, "CALL spatial.layers()", null, 2); + + // Adding a point to the second layer + Node node = createNode("CALL spatial.addWKT($name,$wkt)", + map("name", "osmB", "wkt", String.format("POINT (%f %f)", 15.2, 60.1)), "node"); + testCall(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", + map("name", "osmB", "lon", 15.2, "lat", 60.1), r -> Assertions.assertEquals(node, r.get("node"))); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", + map("name", "osmB", "lon", 15.2, "lat", 60.1), 1); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", + map("name", "osmB"), 217); + NodeIdRecorder bnodesFound = new NodeIdRecorder(); + testResult(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", + map("name", "osmB"), bnodesFound); + bnodesFound.debug("withinDistance('osmB')"); + + // Assert that osmA does not have the extra node + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", + map("name", "osmB", "lon", 15.2, "lat", 60.1), 1); + + // try merge osmB into osmA - should succeed, but does not yet + testCall(db, "CALL spatial.merge.into($nameA,$nameB)", map("nameA", "osmA", "nameB", "osmB"), + r -> Assertions.assertEquals(1L, r.get("count"))); + + // Extra debugging to find differences + NodeIdRecorder anodesFound = new NodeIdRecorder(); + testResult(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", + map("name", "osmA"), anodesFound); + anodesFound.debug("withinDistance('osmA'+'osmB')"); + anodesFound.shouldContain(bnodesFound); + + // Here we should assert that osmA now does have the extra node + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", + map("name", "osmA"), 217); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:6.3740429666,lat:50.93676351666},1000)", + map("name", "osmB"), 0); + testCallCount(db, "CALL spatial.withinDistance($name,{lon:$lon,lat:$lat},100)", + map("name", "osmA", "lon", 15.2, "lat", 60.1), 1); + } + + @Test + public void find_geometries_in_a_bounding_box_short() { + Node node = addPointLayerXYWithNode("geom", 15.2, 60.1); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test public void find_geometries_in_a_bounding_box() { - execute("CALL spatial.addPointLayer('geom')"); - Node node = createNode( - "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", - "node"); + Node node = addPointLayerWithNode("geom", 15.2, 60.1); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -1246,17 +1513,14 @@ public void find_geometries_in_a_polygon() { "UNWIND [{name:'a',latitude:60.1,longitude:15.2},{name:'b',latitude:60.3,longitude:15.5}] as point CREATE (n:Node) SET n += point WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node.name as name"); String polygon = "POLYGON((15.3 60.2, 15.3 60.4, 15.7 60.4, 15.7 60.2, 15.3 60.2))"; testCall(db, "CALL spatial.intersects('geom','" + polygon + "') YIELD node RETURN node.name as name", - r -> assertEquals("b", r.get("name"))); + r -> Assertions.assertEquals("b", r.get("name"))); } @Test public void find_geometries_in_a_bounding_box_geohash() { - execute("CALL spatial.addPointLayerGeohash('geom')"); - Node node = createNode( - "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", - "node"); + Node node = addPointLayerGeohashWithNode("geom", 15.2, 60.1); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -1266,7 +1530,7 @@ public void find_geometries_in_a_polygon_geohash() { "UNWIND [{name:'a',latitude:60.1,longitude:15.2},{name:'b',latitude:60.3,longitude:15.5}] as point CREATE (n:Node) SET n += point WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node.name as name"); String polygon = "POLYGON((15.3 60.2, 15.3 60.4, 15.7 60.4, 15.7 60.2, 15.3 60.2))"; testCall(db, "CALL spatial.intersects('geom','" + polygon + "') YIELD node RETURN node.name as name", - r -> assertEquals("b", r.get("name"))); + r -> Assertions.assertEquals("b", r.get("name"))); } @Test @@ -1276,7 +1540,7 @@ public void find_geometries_in_a_bounding_box_zorder() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -1286,7 +1550,7 @@ public void find_geometries_in_a_polygon_zorder() { "UNWIND [{name:'a',latitude:60.1,longitude:15.2},{name:'b',latitude:60.3,longitude:15.5}] as point CREATE (n:Node) SET n += point WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node.name as name"); String polygon = "POLYGON((15.3 60.2, 15.3 60.4, 15.7 60.4, 15.7 60.2, 15.3 60.2))"; testCall(db, "CALL spatial.intersects('geom','" + polygon + "') YIELD node RETURN node.name as name", - r -> assertEquals("b", r.get("name"))); + r -> Assertions.assertEquals("b", r.get("name"))); } @Test @@ -1296,7 +1560,7 @@ public void find_geometries_in_a_bounding_box_hilbert() { "CREATE (n:Node {latitude:60.1,longitude:15.2}) WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node", "node"); testCall(db, "CALL spatial.bbox('geom',{lon:15.0,lat:60.0}, {lon:15.3, lat:61.0})", - r -> assertEquals(node, r.get("node"))); + r -> Assertions.assertEquals(node, r.get("node"))); } @Test @@ -1306,13 +1570,13 @@ public void find_geometries_in_a_polygon_hilbert() { "UNWIND [{name:'a',latitude:60.1,longitude:15.2},{name:'b',latitude:60.3,longitude:15.5}] as point CREATE (n:Node) SET n += point WITH n CALL spatial.addNode('geom',n) YIELD node RETURN node.name as name"); String polygon = "POLYGON((15.3 60.2, 15.3 60.4, 15.7 60.4, 15.7 60.2, 15.3 60.2))"; testCall(db, "CALL spatial.intersects('geom','" + polygon + "') YIELD node RETURN node.name as name", - r -> assertEquals("b", r.get("name"))); + r -> Assertions.assertEquals("b", r.get("name"))); } @Test public void create_a_WKT_layer() { testCall(db, "CALL spatial.addWKTLayer('geom','wkt')", - r -> assertEquals("wkt", dump(((Node) r.get("node"))).getProperty("geomencoder_config"))); + r -> Assertions.assertEquals("wkt", dump(((Node) r.get("node"))).getProperty("geomencoder_config"))); } private static Node dump(Node n) { @@ -1327,7 +1591,7 @@ public void add_a_WKT_geometry_to_a_layer() { execute("CALL spatial.addWKTLayer('geom','wkt')"); testCall(db, "CALL spatial.addWKT('geom',$wkt)", map("wkt", lineString), - r -> assertEquals(lineString, dump(((Node) r.get("node"))).getProperty("wkt"))); + r -> Assertions.assertEquals(lineString, dump(((Node) r.get("node"))).getProperty("wkt"))); } @Test @@ -1336,7 +1600,7 @@ public void find_geometries_close_to_a_point_wkt() { execute("CALL spatial.addLayer('geom','WKT','wkt')"); execute("CALL spatial.addWKT('geom',$wkt)", map("wkt", lineString)); testCall(db, "CALL spatial.closest('geom',{lon:15.2, lat:60.1}, 1.0)", - r -> assertEquals(lineString, (dump((Node) r.get("node"))).getProperty("wkt"))); + r -> Assertions.assertEquals(lineString, (dump((Node) r.get("node"))).getProperty("wkt"))); } @Test