From d0f5504dd04592f27bdccd118e9298c46ef741ff Mon Sep 17 00:00:00 2001 From: Taylor Smock Date: Tue, 19 Mar 2024 14:58:26 -0600 Subject: [PATCH] Initial support for Mapillary point clouds Signed-off-by: Taylor Smock --- .../plugins/mapillary/MapillaryPlugin.java | 5 + .../actions/MapillaryPointCloudAction.java | 65 +++++ .../pointcloud/PointCloudCamera.java | 6 + .../mapillary/pointcloud/PointCloudColor.java | 5 + .../pointcloud/PointCloudLatLonAlt.java | 7 + .../mapillary/pointcloud/PointCloudPoint.java | 5 + .../pointcloud/PointCloudReconstruction.java | 8 + .../mapillary/pointcloud/PointCloudShot.java | 8 + .../mapillary/pointcloud/PointCloudXYZ.java | 5 + .../mapillary/pointcloud/ProjectionType.java | 6 + .../MapillaryPointCloudImageSource.java | 104 ++++++++ .../pointcloud/MapillaryPointCloudLayer.java | 248 ++++++++++++++++++ .../pointcloud/MapillaryPointCloudLoader.java | 55 ++++ .../MapillaryPointCloudTileJob.java | 93 +++++++ .../mapillary/io/PointCloudParser.java | 156 +++++++++++ .../spi/preferences/ApiKeyReader.java | 6 +- 16 files changed, 781 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/actions/MapillaryPointCloudAction.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudCamera.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudColor.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudLatLonAlt.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudPoint.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudReconstruction.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudShot.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudXYZ.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/ProjectionType.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudImageSource.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudLayer.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudLoader.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudTileJob.java create mode 100644 src/main/java/org/openstreetmap/josm/plugins/mapillary/io/PointCloudParser.java diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/MapillaryPlugin.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/MapillaryPlugin.java index 8bf68a0e5..751275795 100644 --- a/src/main/java/org/openstreetmap/josm/plugins/mapillary/MapillaryPlugin.java +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/MapillaryPlugin.java @@ -24,6 +24,7 @@ import org.openstreetmap.josm.plugins.mapillary.actions.MapPointObjectLayerAction; import org.openstreetmap.josm.plugins.mapillary.actions.MapillaryDownloadAction; import org.openstreetmap.josm.plugins.mapillary.actions.MapillaryExportAction; +import org.openstreetmap.josm.plugins.mapillary.actions.MapillaryPointCloudAction; import org.openstreetmap.josm.plugins.mapillary.actions.MapillaryZoomAction; import org.openstreetmap.josm.plugins.mapillary.data.mapillary.VectorDataSelectionListener; import org.openstreetmap.josm.plugins.mapillary.data.mapillary.smartedit.IgnoredObjects; @@ -104,6 +105,10 @@ public MapillaryPlugin(PluginInformation info) { mapPointObjectLayerAction.updateEnabledState(); destroyables.add(mapPointObjectLayerAction); + MapillaryPointCloudAction mapillaryPointCloudAction = new MapillaryPointCloudAction(); + MainMenu.add(menu.imagerySubMenu, mapillaryPointCloudAction, true); // FIXME expert until stable + destroyables.add(mapillaryPointCloudAction); + // TODO remove in destroy (not currently possible) RequestProcessor.addRequestHandlerClass("photo", MapillaryRemoteControl.class); RequestProcessor.addRequestHandlerClass("mapillaryfilter", MapillaryFilterRemoteControl.class); diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/actions/MapillaryPointCloudAction.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/actions/MapillaryPointCloudAction.java new file mode 100644 index 000000000..6e6db0803 --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/actions/MapillaryPointCloudAction.java @@ -0,0 +1,65 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.actions; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.awt.event.ActionEvent; +import java.awt.event.KeyEvent; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; + +import javax.swing.JOptionPane; + +import org.openstreetmap.josm.actions.JosmAction; +import org.openstreetmap.josm.gui.MainApplication; +import org.openstreetmap.josm.gui.Notification; +import org.openstreetmap.josm.plugins.mapillary.MapillaryPlugin; +import org.openstreetmap.josm.plugins.mapillary.gui.layer.MapillaryLayer; +import org.openstreetmap.josm.plugins.mapillary.gui.layer.pointcloud.MapillaryPointCloudLayer; +import org.openstreetmap.josm.plugins.mapillary.oauth.OAuthUtils; +import org.openstreetmap.josm.plugins.mapillary.spi.preferences.MapillaryConfig; +import org.openstreetmap.josm.plugins.mapillary.utils.MapillaryImageUtils; +import org.openstreetmap.josm.tools.ImageProvider; +import org.openstreetmap.josm.tools.Shortcut; + +/** + * An action to add point clouds as aerial imagery + */ +public class MapillaryPointCloudAction extends JosmAction { + /** + * Create a new action + */ + public MapillaryPointCloudAction() { + super(tr("Mapillary Point Cloud (experimental)"), + new ImageProvider(MapillaryPlugin.LOGO).setSize(ImageProvider.ImageSizes.DEFAULT), + tr("Open Mapillary Point Cloud layer"), Shortcut.registerShortcut("mapillary:pointcloud", + tr("Mapillary Point Cloud"), KeyEvent.CHAR_UNDEFINED, Shortcut.NONE), + false, "mapillary:pointcloud", true); + } + + @Override + public void actionPerformed(ActionEvent e) { + final var selected = MapillaryLayer.getInstance().getImage(); + try { + var obj = OAuthUtils + .getWithHeader(URI.create(MapillaryConfig.getUrls().getImageInformation(selected.getUniqueId(), + MapillaryImageUtils.ImageProperties.SFM_CLUSTER))) + .getJsonObject(MapillaryImageUtils.ImageProperties.SFM_CLUSTER.toString()); + if (obj.containsKey("id") && obj.containsKey("url")) { + MainApplication.getLayerManager() + .addLayer(new MapillaryPointCloudLayer(obj.getString("id"), obj.getString("url"), selected)); + } else { + new Notification(tr("Could not find point cloud for Mapillary image {0}", selected.getUniqueId())) + .setIcon(JOptionPane.ERROR_MESSAGE).show(); + } + } catch (IOException ioException) { + throw new UncheckedIOException(ioException); + } + } + + @Override + protected void updateEnabledState() { + setEnabled(MapillaryLayer.hasInstance() && MapillaryLayer.getInstance().getImage() != null || true); + } +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudCamera.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudCamera.java new file mode 100644 index 000000000..6a9dbd8c9 --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudCamera.java @@ -0,0 +1,6 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud; + +public record PointCloudCamera(ProjectionType projectionType, int width, int height, double focal, double k1, + double k2) { +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudColor.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudColor.java new file mode 100644 index 000000000..09f9cb08b --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudColor.java @@ -0,0 +1,5 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud; + +public record PointCloudColor(int r, int g, int b) { +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudLatLonAlt.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudLatLonAlt.java new file mode 100644 index 000000000..7e68c2ceb --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudLatLonAlt.java @@ -0,0 +1,7 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud; + +import org.openstreetmap.josm.data.coor.ILatLon; + +public record PointCloudLatLonAlt(double lat, double lon, double altitude) implements ILatLon { +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudPoint.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudPoint.java new file mode 100644 index 000000000..1f606062b --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudPoint.java @@ -0,0 +1,5 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud; + +public record PointCloudPoint(PointCloudXYZ coordinates, PointCloudColor color) { +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudReconstruction.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudReconstruction.java new file mode 100644 index 000000000..a82b0d7da --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudReconstruction.java @@ -0,0 +1,8 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud; + +import java.util.Map; + +public record PointCloudReconstruction(PointCloudLatLonAlt referenceLatLonAlt, Map cameras, + Map shots, Map points) { +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudShot.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudShot.java new file mode 100644 index 000000000..84c0f1191 --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudShot.java @@ -0,0 +1,8 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud; + +import java.time.Instant; + +public record PointCloudShot(String cameraId, PointCloudXYZ rotation, PointCloudXYZ translation, + PointCloudXYZ gpsPosition, double gpsDop, int orientation, Instant captureTime) { +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudXYZ.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudXYZ.java new file mode 100644 index 000000000..3c296d138 --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/PointCloudXYZ.java @@ -0,0 +1,5 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud; + +public record PointCloudXYZ(double x, double y, double z) { +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/ProjectionType.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/ProjectionType.java new file mode 100644 index 000000000..8bc6efc4d --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/data/mapillary/pointcloud/ProjectionType.java @@ -0,0 +1,6 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud; + +public enum ProjectionType { + PERSPECTIVE, BROWN, FISHEYE, EQUIRECTANGULAR +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudImageSource.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudImageSource.java new file mode 100644 index 000000000..a067a0ecf --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudImageSource.java @@ -0,0 +1,104 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.gui.layer.pointcloud; + +import static org.openstreetmap.gui.jmapviewer.OsmMercator.MERCATOR_256; + +import java.awt.Point; + +import org.openstreetmap.gui.jmapviewer.Coordinate; +import org.openstreetmap.gui.jmapviewer.OsmMercator; +import org.openstreetmap.gui.jmapviewer.Projected; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.TileRange; +import org.openstreetmap.gui.jmapviewer.TileXY; +import org.openstreetmap.gui.jmapviewer.interfaces.ICoordinate; +import org.openstreetmap.gui.jmapviewer.interfaces.IProjected; +import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource; +import org.openstreetmap.josm.data.coor.ILatLon; +import org.openstreetmap.josm.data.imagery.ImageryInfo; + +/** + * The source for creating tiles for Mapillary Point Clouds + */ +public class MapillaryPointCloudImageSource extends AbstractTMSTileSource { + + private final ILatLon origin; + + public MapillaryPointCloudImageSource(ImageryInfo info, ILatLon origin) { + super(info); + this.origin = origin; + } + + @Override + public double getDistance(double lat1, double lon1, double lat2, double lon2) { + return osmMercator().getDistance(lat1, lon1, lat2, lon2); + } + + @Override + public Point latLonToXY(double lat, double lon, int zoom) { + return new Point((int) Math.round(osmMercator().lonToX(lon, zoom)), + (int) Math.round(osmMercator().latToY(lat, zoom))); + } + + @Override + public ICoordinate xyToLatLon(int x, int y, int zoom) { + return new Coordinate(osmMercator().yToLat(y, zoom), osmMercator().xToLon(x, zoom)); + } + + @Override + public TileXY latLonToTileXY(double lat, double lon, int zoom) { + return new TileXY(osmMercator().lonToX(lon, zoom) / getTileSize(), + osmMercator().latToY(lat, zoom) / getTileSize()); + } + + @Override + public ICoordinate tileXYToLatLon(int x, int y, int zoom) { + return new Coordinate(osmMercator().yToLat((long) y * getTileSize(), zoom), + osmMercator().xToLon((long) x * getTileSize(), zoom)); + } + + @Override + public IProjected tileXYtoProjected(int x, int y, int zoom) { + final var mercatorWidth = 2 * Math.PI * OsmMercator.EARTH_RADIUS; + final var f = mercatorWidth * getTileSize() / osmMercator().getMaxPixels(zoom); + return new Projected(f * x - mercatorWidth / 2, -(f * y - mercatorWidth / 2)); + } + + @Override + public TileXY projectedToTileXY(IProjected p, int zoom) { + final var mercatorWidth = 2 * Math.PI * OsmMercator.EARTH_RADIUS; + final var f = mercatorWidth * getTileSize() / osmMercator().getMaxPixels(zoom); + return new TileXY((p.getEast() + mercatorWidth / 2) / f, (-p.getNorth() + mercatorWidth / 2) / f); + } + + @Override + public boolean isInside(Tile inner, Tile outer) { + final int dz = inner.getZoom() - outer.getZoom(); + if (dz < 0) + return false; + return outer.getXtile() == inner.getXtile() >> dz && outer.getYtile() == inner.getYtile() >> dz; + } + + @Override + public TileRange getCoveringTileRange(Tile tile, int newZoom) { + if (newZoom <= tile.getZoom()) { + final int dz = tile.getZoom() - newZoom; + final var xy = new TileXY(tile.getXtile() >> dz, tile.getYtile() >> dz); + return new TileRange(xy, xy, newZoom); + } else { + final int dz = newZoom - tile.getZoom(); + final var t1 = new TileXY(tile.getXtile() << dz, tile.getYtile() << dz); + final var t2 = new TileXY(t1.getX() + (1 << dz) - 1, t1.getY() + (1 << dz) - 1); + return new TileRange(t1, t2, newZoom); + } + } + + @Override + public String getServerCRS() { + return "EPSG:3857"; + } + + private static OsmMercator osmMercator() { + return MERCATOR_256; + } +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudLayer.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudLayer.java new file mode 100644 index 000000000..87b347d78 --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudLayer.java @@ -0,0 +1,248 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.gui.layer.pointcloud; + +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.geom.AffineTransform; +import java.awt.geom.Point2D; +import java.awt.geom.Rectangle2D; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.LongStream; +import java.util.zip.InflaterInputStream; + +import javax.swing.Action; + +import jakarta.json.Json; +import jakarta.json.JsonObject; +import org.openstreetmap.josm.data.Bounds; +import org.openstreetmap.josm.data.coor.ILatLon; +import org.openstreetmap.josm.data.imagery.ImageryInfo; +import org.openstreetmap.josm.data.osm.INode; +import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; +import org.openstreetmap.josm.gui.MainApplication; +import org.openstreetmap.josm.gui.MapView; +import org.openstreetmap.josm.gui.layer.ImageryLayer; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.MapillaryNode; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud.PointCloudReconstruction; +import org.openstreetmap.josm.plugins.mapillary.gui.layer.MapillaryLayer; +import org.openstreetmap.josm.plugins.mapillary.io.PointCloudParser; +import org.openstreetmap.josm.plugins.mapillary.oauth.OAuthUtils; +import org.openstreetmap.josm.plugins.mapillary.spi.preferences.IMapillaryUrls; +import org.openstreetmap.josm.plugins.mapillary.spi.preferences.MapillaryConfig; +import org.openstreetmap.josm.plugins.mapillary.utils.MapillaryImageUtils; +import org.openstreetmap.josm.tools.HttpClient; +import org.openstreetmap.josm.tools.Logging; + +/** + * Show Mapillary Point Clouds as aerial imagery + */ +public class MapillaryPointCloudLayer extends ImageryLayer { + private final String id; + private final String url; + private final ILatLon origin; + private List reconstructions; + + /** + * Create a new point cloud layer + * + * @param id The id of the point cloud + * @param url The URL of the point cloud + * @param origin A point to use for determining the origin of the layer + */ + public MapillaryPointCloudLayer(String id, String url, ILatLon origin) { + super(new ImageryInfo("Mapillary Point Cloud: " + id, url)); + this.id = id; + this.url = url; + this.origin = origin; + } + + @Override + protected Action getAdjustAction() { + return null; + } + + @Override + protected List getOffsetMenuEntries() { + return List.of(); + } + + @Override + public String getToolTipText() { + return "Point Cloud: " + this.id; + } + + @Override + public void visitBoundingBox(BoundingXYVisitor v) { + + } + + @Override + public Action[] getMenuEntries() { + return new Action[0]; + } + + @Override + public void paint(Graphics2D g, MapView mv, Bounds bbox) { + if (reconstructions == null) { + reconstructions = Collections.emptyList(); + MainApplication.worker.execute(() -> this.getData(bbox)); + } + // "gps_position": [ + // -731.7142190674425, + // -373.7361228710761, + // 1399.7760358098894 + // ], + final var currentOrigin = mv.getPoint(this.origin); + final var xyzToPixel = 1.5 / mv.getScale(); + for (var reconstruction : reconstructions) { + // Perform world translation + final Point2D reference; + if (reconstruction.referenceLatLonAlt() != null) { + reference = mv.getPoint(reconstruction.referenceLatLonAlt()); + g.setTransform(AffineTransform.getTranslateInstance(reference.getX(), reference.getY())); + g.setColor(Color.RED); + g.drawRect(-2, -2, 4, 4); + } else { + // Find the potential image(s) + final var potentialImages = reconstruction.shots().values().stream() + .filter(s -> s.captureTime().equals(((MapillaryNode) this.origin).getInstant())).toList(); + // For now, just handle the "simple" case where we have a single possible image + if (potentialImages.size() == 1) { + final var xyz = potentialImages.get(0).gpsPosition(); + g.setTransform( + AffineTransform.getTranslateInstance(currentOrigin.x - xyz.z(), currentOrigin.y - xyz.x())); + reference = new Point2D.Double(currentOrigin.x - xyz.z(), currentOrigin.y - xyz.x()); + } else { + reference = null; + } + } + final Rectangle2D bounds = new Rectangle2D.Double(); + for (var point : reconstruction.points().values().stream() + .sorted(Comparator.comparingDouble(d -> d.coordinates().z())).toList()) { + final var point2d = new Point2D.Double(xyzToPixel * point.coordinates().x(), + -xyzToPixel * point.coordinates().y()); + g.setColor(new Color(point.color().r(), point.color().g(), point.color().b())); + g.drawRect((int) point2d.x, (int) point2d.y, 1, 1); + bounds.add(point2d); + } + if (reference != null && Logging.isTraceEnabled() && false) { + g.setColor(Color.RED); + g.draw(bounds); + g.drawLine((int) bounds.getMinX(), (int) bounds.getMinY(), 0, 0); + } + } + g.setTransform(AffineTransform.getTranslateInstance(0, 0)); + } + + private static JsonObject cacheSfmKey(INode... images) { + final var imageIds = Arrays.stream(images).mapToLong(INode::getUniqueId).toArray(); + final var sfmCluster = MapillaryImageUtils.ImageProperties.SFM_CLUSTER; + final JsonObject json; + final URI tUrl; + if (imageIds.length == 1) { + tUrl = URI.create(MapillaryConfig.getUrls().getImageInformation(images[0].getId(), sfmCluster)); + } else { + Map queryFields = new HashMap<>(2); + queryFields.put(IMapillaryUrls.FIELDS, sfmCluster.toString()); + queryFields.put("image_ids", + LongStream.of(imageIds).mapToObj(Long::toString).collect(Collectors.joining(","))); + tUrl = URI.create( + MapillaryConfig.getUrls().getBaseMetaDataUrl() + "images" + IMapillaryUrls.queryString(queryFields)); + } + try { + json = OAuthUtils.getWithHeader(tUrl); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + // Stick the data in the objects so we don't have to fetch it again later. + for (JsonObject obj : json.getJsonArray("data").getValuesAs(JsonObject.class)) { + if (obj.containsKey(sfmCluster.toString())) { + final var id = Long.parseLong(obj.getString(MapillaryImageUtils.ImageProperties.ID.toString())); + Arrays.stream(images).filter(i -> i.getUniqueId() == id).findFirst().ifPresent( + image -> image.put(sfmCluster.toString(), obj.getJsonObject(sfmCluster.toString()).toString())); + } + } + return json; + } + + private void getData(Bounds bbox) { + if (true) { + /* The collection of reconstructions to show when we finish */ + final var tReconstructions = new ArrayList(); + /* Reconstructions we have already fetched */ + final var currentReconstructions = new HashMap>(); + /* Images that we have already fetched */ + final var fetched = new HashSet(); + final var searchNodes = MapillaryLayer.getInstance().getData().searchNodes(bbox.toBBox()); + cacheSfmKey(searchNodes.stream() + .filter(node -> !node.hasKey(MapillaryImageUtils.ImageProperties.SFM_CLUSTER.toString())) + .toArray(INode[]::new)); + for (var selected : searchNodes) { + if (fetched.contains(selected.getUniqueId()) || selected.getUniqueId() <= 0) { + continue; + } + final var sfmJson = getSfmCluster(selected); + if (sfmJson == null || !sfmJson.containsKey("url") || !sfmJson.containsKey("id")) { + continue; // No clue what is causing this on the backend. Skip for now. + } + fetched.add(selected.getUniqueId()); + + final var tUrl = sfmJson.getString("url"); + final var id = sfmJson.getString("id"); + if (currentReconstructions.containsKey(id)) { + continue; + } + final var cReconstructions = parseReconstruction(tUrl); + tReconstructions.addAll(cReconstructions); + this.reconstructions = List.copyOf(tReconstructions); + currentReconstructions.put(id, cReconstructions); + } + tReconstructions.trimToSize(); + this.reconstructions = Collections.unmodifiableList(tReconstructions); + } else { + this.reconstructions = parseReconstruction(this.url); + } + } + + private static JsonObject getSfmCluster(INode selected) { + final var sfmCluster = MapillaryImageUtils.ImageProperties.SFM_CLUSTER; + if (!selected.hasKey(sfmCluster.toString())) { + return cacheSfmKey(selected); + } + if (selected.hasKey(sfmCluster.toString())) { + try (var reader = Json.createReader( + new ByteArrayInputStream(selected.get(sfmCluster.toString()).getBytes(StandardCharsets.UTF_8)))) { + return reader.readObject(); + } + } + return Json.createObjectBuilder().build(); + } + + private static List parseReconstruction(String url) { + try { + final var client = HttpClient.create(URI.create(url).toURL()); + final var response = client.connect(); + try (var iis = new InflaterInputStream(response.getContent())) { + return PointCloudParser.parse(iis); + } finally { + client.disconnect(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudLoader.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudLoader.java new file mode 100644 index 000000000..5b83b044a --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudLoader.java @@ -0,0 +1,55 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.gui.layer.pointcloud; + +import java.util.Collection; +import java.util.HashSet; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.commons.jcs3.access.behavior.ICacheAccess; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; +import org.openstreetmap.josm.data.cache.CacheEntry; +import org.openstreetmap.josm.data.imagery.TMSCachedTileLoader; +import org.openstreetmap.josm.data.imagery.TileJobOptions; + +public class MapillaryPointCloudLoader implements TileLoader { + /** The executor for fetching the tiles. */ + private static final ThreadPoolExecutor EXECUTOR = TMSCachedTileLoader + .getNewThreadPoolExecutor("mapillary:pointcloud"); + /** The current jobs for the loader */ + private final Collection jobs = new HashSet<>(); + /** The cache for downloaded tiles */ + private final ICacheAccess cache; + /** The options for the tile loader */ + private final TileJobOptions options; + /** The listener for when jobs finish */ + private final TileLoaderListener listener; + + /** + * Create a new tile loader + * + * @param listener The listener to call when tiles are finished loading + * @param cache cache instance that we will work on + * @param options options of the request + */ + public MapillaryPointCloudLoader(TileLoaderListener listener, ICacheAccess cache, + TileJobOptions options) { + this.options = options; + this.cache = cache; + this.listener = listener; + } + + @Override + public TileJob createTileLoaderJob(Tile tile) { + var job = new MapillaryPointCloudTileJob(EXECUTOR, listener, cache, options, tile); + jobs.add(job); + return job; + } + + @Override + public void cancelOutstandingTasks() { + this.jobs.clear(); + } +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudTileJob.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudTileJob.java new file mode 100644 index 000000000..32347b0f5 --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/layer/pointcloud/MapillaryPointCloudTileJob.java @@ -0,0 +1,93 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.gui.layer.pointcloud; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.zip.InflaterInputStream; + +import org.apache.commons.jcs3.access.behavior.ICacheAccess; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; +import org.openstreetmap.josm.data.cache.CacheEntry; +import org.openstreetmap.josm.data.imagery.TileJobOptions; +import org.openstreetmap.josm.plugins.mapillary.io.PointCloudParser; +import org.openstreetmap.josm.tools.HttpClient; +import org.openstreetmap.josm.tools.JosmRuntimeException; + +/** + * A job to load a tile from a point cloud + */ +public class MapillaryPointCloudTileJob implements TileJob { + private final ThreadPoolExecutor executor; + private final TileLoaderListener listener; + private final ICacheAccess cache; + private final TileJobOptions options; + private final Tile tile; + + /** + * A job to load a tile from a point cloud + * + * @param executor that will be executing the jobs + * @param listener The listener to notify when a tile is loaded + * @param cache cache instance that we will work on + * @param options options of the request + * @param tile The tile for the request + */ + public MapillaryPointCloudTileJob(ThreadPoolExecutor executor, TileLoaderListener listener, + ICacheAccess cache, TileJobOptions options, Tile tile) { + this.executor = executor; + this.listener = listener; + this.cache = cache; + this.options = options; + this.tile = tile; + } + + @Override + public void submit() { + submit(false); + } + + @Override + public void submit(boolean force) { + this.executor.submit(this); + } + + @Override + public void run() { + final var id = this.tile.getTileSource().getId(); + final CacheEntry cacheEntry; + synchronized (id) { // Yes, we want to synchronize on the id, which **should** be the same memory object + cacheEntry = cache.get(id, () -> getCacheEntry(id)); + } + if (cacheEntry == null) { + this.listener.tileLoadingFinished(this.tile, false); + } else { + try (var iis = new InflaterInputStream(new ByteArrayInputStream(cacheEntry.getContent()))) { + final var reconstruction = PointCloudParser.parse(iis); + throw new UnsupportedOperationException("TODO: paint this somehow"); + } catch (IOException e) { + // This should never happen since we are using a ByteArrayInputStream + throw new UncheckedIOException(e); + } + // this.listener.tileLoadingFinished(this.tile, true); + } + } + + private static CacheEntry getCacheEntry(String url) { + try { + final var client = HttpClient.create(URI.create(url).toURL()); + try { + final var response = client.connect(); + return new CacheEntry(response.getContent().readAllBytes()); + } finally { + client.disconnect(); + } + } catch (IOException e) { + throw new JosmRuntimeException(e); + } + } +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/io/PointCloudParser.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/io/PointCloudParser.java new file mode 100644 index 000000000..5d38b81c9 --- /dev/null +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/io/PointCloudParser.java @@ -0,0 +1,156 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.plugins.mapillary.io; + +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import jakarta.json.Json; +import jakarta.json.JsonArray; +import jakarta.json.JsonNumber; +import jakarta.json.JsonObject; +import jakarta.json.stream.JsonParser; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud.PointCloudCamera; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud.PointCloudColor; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud.PointCloudLatLonAlt; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud.PointCloudPoint; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud.PointCloudReconstruction; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud.PointCloudShot; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud.PointCloudXYZ; +import org.openstreetmap.josm.plugins.mapillary.data.mapillary.pointcloud.ProjectionType; + +public final class PointCloudParser { + public static List parse(InputStream inputStream) { + try (var parser = Json.createParser(inputStream)) { + return switch (parser.next()) { + case START_OBJECT -> Collections.singletonList(parseReconstruction(parser)); + case START_ARRAY -> Collections.unmodifiableList(parseReconstructions(parser)); + default -> Collections.emptyList(); + }; + } + } + + private static List parseReconstructions(JsonParser parser) { + var list = new ArrayList(); + while (parser.hasNext()) { + if (parser.next() == JsonParser.Event.START_OBJECT) { + list.add(parseReconstruction(parser)); + } else if (parser.currentEvent() == JsonParser.Event.START_ARRAY) { + parser.skipArray(); + } + } + list.trimToSize(); + return list; + } + + private static PointCloudReconstruction parseReconstruction(JsonParser parser) { + final var cameras = new HashMap(); + final var shots = new HashMap(); + final var points = new HashMap(); + PointCloudLatLonAlt referenceLLA = null; + String lastKey = ""; + while (parser.hasNext()) { + if (parser.next() == JsonParser.Event.KEY_NAME) { + lastKey = parser.getString(); + switch (lastKey) { + case "cameras" -> parseCameras(parser, cameras); + case "shots" -> parseShots(parser, shots); + case "points" -> parsePoints(parser, points); + case "reference_lla" -> referenceLLA = parseReferenceLatLonAltitude(parser); + case "biases", "rig_cameras", "rig_instances" -> { + /* Do nothing */ } + default -> { + /* Do nothing */ } // skip this entry + } + } else if (parser.currentEvent() == JsonParser.Event.START_OBJECT) { + parser.skipObject(); + } else if (parser.currentEvent() == JsonParser.Event.END_OBJECT) { + break; + } + } + return new PointCloudReconstruction(referenceLLA, Collections.unmodifiableMap(cameras), + Collections.unmodifiableMap(shots), Collections.unmodifiableMap(points)); + } + + private static PointCloudLatLonAlt parseReferenceLatLonAltitude(JsonParser parser) { + if (parser.next() == JsonParser.Event.START_OBJECT) { + final var obj = parser.getObject(); + return new PointCloudLatLonAlt(obj.getJsonNumber("latitude").doubleValue(), + obj.getJsonNumber("longitude").doubleValue(), obj.getJsonNumber("altitude").doubleValue()); + } + throw new IllegalStateException("Unknown state for json"); + } + + private static void parseCameras(JsonParser parser, Map cameras) { + while (parser.hasNext()) { + if (parser.next() == JsonParser.Event.KEY_NAME) { + final var cameraId = parser.getString(); + parser.next(); + cameras.put(cameraId, parseCamera(parser.getObject())); + } else if (parser.currentEvent() == JsonParser.Event.END_OBJECT) { + return; + } + } + } + + private static PointCloudCamera parseCamera(JsonObject object) { + final var perspective = switch (object.getString("projection_type")) { + case "brown" -> ProjectionType.BROWN; + case "fisheye" -> ProjectionType.FISHEYE; + case "equirectangular", "spherical" -> ProjectionType.EQUIRECTANGULAR; + case "perspective" -> ProjectionType.PERSPECTIVE; + default -> + throw new IllegalArgumentException("Unknown projection_type: " + object.getString("projection_type")); + }; + return new PointCloudCamera(perspective, object.getInt("width"), object.getInt("height"), + object.containsKey("focal") ? object.getJsonNumber("focal").doubleValue() : Double.NaN, + object.containsKey("k1") ? object.getJsonNumber("k1").doubleValue() : Double.NaN, + object.containsKey("k2") ? object.getJsonNumber("k2").doubleValue() : Double.NaN); + } + + private static void parseShots(JsonParser parser, Map shots) { + while (parser.hasNext()) { + if (parser.next() == JsonParser.Event.KEY_NAME) { + final var shotId = parser.getString(); + parser.next(); + shots.put(shotId, parseShot(parser.getObject())); + } else if (parser.currentEvent() == JsonParser.Event.END_OBJECT) { + return; + } + } + } + + private static PointCloudShot parseShot(JsonObject object) { + return new PointCloudShot(object.getString("camera"), parseXYZ(object.getJsonArray("rotation")), + parseXYZ(object.getJsonArray("translation")), parseXYZ(object.getJsonArray("gps_position")), + object.getJsonNumber("gps_dop").doubleValue(), object.getJsonNumber("orientation").intValue(), + Instant.ofEpochSecond(object.getJsonNumber("capture_time").longValue())); + } + + private static void parsePoints(JsonParser parser, Map points) { + while (parser.hasNext()) { + if (parser.next() == JsonParser.Event.KEY_NAME) { + final var pointId = parser.getString(); + parser.next(); + points.put(pointId, parsePoint(parser.getObject())); + } else if (parser.currentEvent() == JsonParser.Event.END_OBJECT) { + return; + } + } + } + + private static PointCloudPoint parsePoint(JsonObject object) { + final var rgb = object.getJsonArray("color").getValuesAs(JsonNumber.class); + return new PointCloudPoint(parseXYZ(object.getJsonArray("coordinates")), + new PointCloudColor(rgb.get(0).intValue(), rgb.get(1).intValue(), rgb.get(2).intValue())); + } + + private static PointCloudXYZ parseXYZ(JsonArray array) { + return new PointCloudXYZ(array.getJsonNumber(0).doubleValue(), array.getJsonNumber(1).doubleValue(), + array.getJsonNumber(2).doubleValue()); + } +} diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/spi/preferences/ApiKeyReader.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/spi/preferences/ApiKeyReader.java index 4c576b79e..07b48ddc9 100644 --- a/src/main/java/org/openstreetmap/josm/plugins/mapillary/spi/preferences/ApiKeyReader.java +++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/spi/preferences/ApiKeyReader.java @@ -24,7 +24,11 @@ private ApiKeyReader() { static String readValue(final String key) { // Prefer system property (this is something that can be changed fairly easily) if (System.getProperty(key) != null) { - return System.getProperty(key); + return Utils.strip(System.getProperty(key), "\""); + } + // Then environment variables + if (System.getenv(key) != null) { + return Utils.strip(System.getenv(key), "\""); } // Then check if there was something stored in JOSM preferences if (Config.getPref() != null && !Utils.isBlank(Config.getPref().get("mapillary.api." + key))) {