Skip to content

Commit

Permalink
Merge pull request #977 from WildMeOrg/873_opensearch_indexing_annota…
Browse files Browse the repository at this point in the history 
…tions

873 opensearch indexing annotations
  • Loading branch information
@holmbergius
holmbergius authored Jan 11, 2025
2 parents ae7cf1f + 5516a01 commit 6a24192
Show file tree
Hide file tree
Showing 10 changed files with 263 additions and 119 deletions.
94 changes: 88 additions & 6 deletions src/main/java/org/ecocean/Annotation.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@

package org.ecocean;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.awt.Rectangle;
import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -23,7 +24,7 @@
import org.json.JSONArray;
import org.json.JSONObject;

public class Annotation implements java.io.Serializable {
public class Annotation extends Base implements java.io.Serializable {
public Annotation() {}
private String id;
private static final String[][] VALID_VIEWPOINTS = new String[][] {
Expand Down Expand Up @@ -51,13 +52,14 @@ public Annotation() {}
// TODO: was this made obsolete by ACM and friends?
private boolean matchAgainst = false;

// TODO: can these (thru mediaAsset) be removed now that there Features?
// TODO: can these (thru mediaAsset) be removed now that there Features?
private int x;
private int y;
private int width;
private int height;
private float[] transformMatrix;
private double theta;
private long version = System.currentTimeMillis();

// quality indicates the fidelity of the annotation, e.g. the overall image quality of a picture.
// This is useful e.g. for researchers who want to account for a bias where "better" images are
Expand Down Expand Up @@ -115,6 +117,80 @@ public Annotation(String species, ArrayList<Feature> f, String iaClass) {
this.iaClass = iaClass;
}

@Override public String opensearchIndexName() { return "annotation"; }

@Override public long getVersion() {
return version;
}

public long setVersion() {
version = System.currentTimeMillis();
return version;
}

public JSONObject opensearchMapping() {
JSONObject map = super.opensearchMapping();
JSONObject keywordType = new org.json.JSONObject("{\"type\": \"keyword\"}");

/*
JSONObject keywordNormalType = new org.json.JSONObject(
"{\"type\": \"keyword\", \"normalizer\": \"wildbook_keyword_normalizer\"}");
*/

// "id" is done in Base
map.put("viewpoint", keywordType);
map.put("iaClass", keywordType);
map.put("acmId", keywordType);
map.put("encounterId", keywordType);
map.put("encounterSubmitterId", keywordType);
map.put("encounterLocationId", keywordType);
map.put("encounterTaxonomy", keywordType);

// all case-insensitive keyword-ish types
// map.put("fubar", keywordNormalType);

return map;
}

public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd)
throws IOException, JsonProcessingException {
super.opensearchDocumentSerializer(jgen, myShepherd);

jgen.writeStringField("acmId", this.getAcmId());
jgen.writeStringField("viewpoint", this.getViewpoint());
jgen.writeStringField("iaClass", this.getIAClass());
MediaAsset ma = this.getMediaAsset();
if (ma != null) {
jgen.writeNumberField("mediaAssetId", ma.getId());
}
Encounter enc = this.findEncounter(myShepherd);
if (enc != null) {
jgen.writeStringField("encounterId", enc.getId());
jgen.writeStringField("encounterSubmitterId", enc.getSubmitterID());
jgen.writeStringField("encounterLocationId", enc.getLocationID());
jgen.writeStringField("encounterTaxonomy", enc.getTaxonomyString());
}
}

@Override public String getAllVersionsSql() {
return "SELECT \"ID\", \"VERSION\" AS version FROM \"ANNOTATION\" ORDER BY version";
}

@Override public Base getById(Shepherd myShepherd, String id) {
return myShepherd.getAnnotation(id);
}

// comment cruft only needed for Base class
@Override public String getComments() {
return null;
}

@Override public void setComments(final String comments) {
}

@Override public void addComments(final String newComments) {
}

// this is for use *only* to migrate old-world Annotations to new-world
public Feature migrateToFeatures() {
Feature f;
Expand All @@ -140,6 +216,7 @@ public Feature migrateToFeatures() {

public void setAcmId(String id) {
this.acmId = id;
this.setVersion();
}

public String getAcmId() {
Expand All @@ -156,11 +233,13 @@ public ArrayList<Feature> getFeatures() {

public void setFeatures(ArrayList<Feature> f) {
features = f;
this.setVersion();
}

public void addFeature(Feature f) {
if (features == null) features = new ArrayList<Feature>();
if (!features.contains(f)) features.add(f);
this.setVersion();
}

public String getId() {
Expand All @@ -169,6 +248,7 @@ public String getId() {

public void setId(String id) {
this.id = id;
this.setVersion();
}

public Double getQuality() {
Expand Down Expand Up @@ -245,7 +325,7 @@ public boolean isTrivial() {

if (ma == null) return false;
for (Feature ft : getFeatures()) {
if (ft.isUnity()) return true;
if (ft.isUnity()) return true;
}
return (!needsTransform() && (getWidth() == (int)ma.getWidth()) &&
(getHeight() == (int)ma.getHeight()));
Expand Down Expand Up @@ -343,6 +423,7 @@ public static String[] getViewpointAndNeighbors(String vp) {

public void setViewpoint(String v) {
viewpoint = v;
this.setVersion();
}

// note! this can block and take a while if IA has yet to compute the viewpoint!
Expand Down Expand Up @@ -460,6 +541,7 @@ public String getIAClass() {

public void setIAClass(String iaClass) {
this.iaClass = iaClass;
this.setVersion();
}

public boolean hasIAClass() {
Expand Down Expand Up @@ -515,6 +597,7 @@ public boolean getMatchAgainst() {

public void setMatchAgainst(boolean b) {
matchAgainst = b;
this.setVersion();
}

public String getIdentificationStatus() {
Expand All @@ -523,6 +606,7 @@ public String getIdentificationStatus() {

public void setIdentificationStatus(String status) {
this.identificationStatus = status;
this.setVersion();
}

// if this cannot determine a bounding box, then we return null
Expand Down Expand Up @@ -851,7 +935,6 @@ private String getMatchingSetFilterFromParameters(JSONObject taskParams) {
}
}
}

List<String> expandedLocationIds = LocationID.expandIDs(rawLocationIds);
String locFilter = "";
if (expandedLocationIds.size() > 0) {
Expand Down Expand Up @@ -1060,7 +1143,6 @@ the one case that is still to be considered is when (theoretically) detection *i
myShepherd.getContext()));
}
return newEnc;

}

public Annotation revertToTrivial(Shepherd myShepherd)
Expand Down
71 changes: 71 additions & 0 deletions src/main/java/org/ecocean/Base.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,77 @@ public static Map<String, Long> getAllVersions(Shepherd myShepherd, String sql)
return rtn;
}

// these two methods are kinda hacky needs for opensearchSyncIndex (e.g. the fact
// they are not static)
public abstract Base getById(Shepherd myShepherd, String id);

public abstract String getAllVersionsSql();

// contains some reflection; not pretty, but gets the job done
public static int[] opensearchSyncIndex(Shepherd myShepherd, Class cls, int stopAfter)
throws IOException {
int[] rtn = new int[2];
Object tmpObj = null;

try {
tmpObj = cls.newInstance();
} catch (Exception ex) {
throw new IOException("FAIL: " + ex);
}
Base baseObj = (Base)tmpObj;
String indexName = baseObj.opensearchIndexName();
if (OpenSearch.indexingActive()) {
System.out.println("Base.opensearchSyncIndex(" + indexName +
") skipped due to indexingActive()");
rtn[0] = -1;
rtn[1] = -1;
return rtn;
}
OpenSearch.setActiveIndexingBackground();
OpenSearch os = new OpenSearch();
List<List<String> > changes = os.resolveVersions(getAllVersions(myShepherd,
baseObj.getAllVersionsSql()), os.getAllVersions(indexName));
if (changes.size() != 2) throw new IOException("invalid resolveVersions results");
List<String> needIndexing = changes.get(0);
List<String> needRemoval = changes.get(1);
rtn[0] = needIndexing.size();
rtn[1] = needRemoval.size();
System.out.println("Base.opensearchSyncIndex(" + indexName + "): stopAfter=" + stopAfter +
", needIndexing=" + rtn[0] + ", needRemoval=" + rtn[1]);
int ct = 0;
for (String id : needIndexing) {
Base obj = baseObj.getById(myShepherd, id);
try {
if (obj != null) os.index(indexName, obj);
} catch (Exception ex) {
System.out.println("Base.opensearchSyncIndex(" + indexName + "): index failed " +
obj + " => " + ex.toString());
ex.printStackTrace();
}
if (ct % 500 == 0)
System.out.println("Base.opensearchSyncIndex(" + indexName + ") needIndexing: " +
ct + "/" + rtn[0]);
ct++;
if ((stopAfter > 0) && (ct > stopAfter)) {
System.out.println("Base.opensearchSyncIndex(" + indexName +
") breaking due to stopAfter");
break;
}
}
System.out.println("Base.opensearchSyncIndex(" + indexName + ") finished needIndexing");
ct = 0;
for (String id : needRemoval) {
os.delete(indexName, id);
if (ct % 500 == 0)
System.out.println("Base.opensearchSyncIndex(" + indexName + ") needRemoval: " +
ct + "/" + rtn[1]);
ct++;
}
System.out.println("Base.opensearchSyncIndex(" + indexName + ") finished needRemoval");
OpenSearch.unsetActiveIndexingBackground();
return rtn;
}

public static Base createFromApi(JSONObject payload, List<File> files, Shepherd myShepherd)
throws ApiException {
throw new ApiException("not yet supported");
Expand Down
75 changes: 8 additions & 67 deletions src/main/java/org/ecocean/Encounter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4326,15 +4326,17 @@ public static boolean opensearchAccess(org.json.JSONObject doc, User user,
return false;
}

@Override public long getVersion() {
return Util.getVersionFromModified(modified);
@Override public Base getById(Shepherd myShepherd, String id) {
return myShepherd.getEncounter(id);
}

public static Map<String, Long> getAllVersions(Shepherd myShepherd) {
String sql =
"SELECT \"CATALOGNUMBER\", CAST(COALESCE(EXTRACT(EPOCH FROM CAST(\"MODIFIED\" AS TIMESTAMP))*1000,-1) AS BIGINT) AS version FROM \"ENCOUNTER\" ORDER BY version";
@Override public String getAllVersionsSql() {
return
"SELECT \"CATALOGNUMBER\", CAST(COALESCE(EXTRACT(EPOCH FROM CAST(\"MODIFIED\" AS TIMESTAMP))*1000,-1) AS BIGINT) AS version FROM \"ENCOUNTER\" ORDER BY version";
}

return getAllVersions(myShepherd, sql);
@Override public long getVersion() {
return Util.getVersionFromModified(modified);
}

public org.json.JSONObject opensearchMapping() {
Expand Down Expand Up @@ -4382,66 +4384,6 @@ public org.json.JSONObject opensearchMapping() {
return map;
}

public static int[] opensearchSyncIndex(Shepherd myShepherd)
throws IOException {
return opensearchSyncIndex(myShepherd, 0);
}

public static int[] opensearchSyncIndex(Shepherd myShepherd, int stopAfter)
throws IOException {
int[] rtn = new int[2];

if (OpenSearch.indexingActive()) {
System.out.println("Encounter.opensearchSyncIndex() skipped due to indexingActive()");
rtn[0] = -1;
rtn[1] = -1;
return rtn;
}
OpenSearch.setActiveIndexingBackground();
String indexName = "encounter";
OpenSearch os = new OpenSearch();
List<List<String> > changes = os.resolveVersions(getAllVersions(myShepherd),
os.getAllVersions(indexName));
if (changes.size() != 2) throw new IOException("invalid resolveVersions results");
List<String> needIndexing = changes.get(0);
List<String> needRemoval = changes.get(1);
rtn[0] = needIndexing.size();
rtn[1] = needRemoval.size();
System.out.println("Encounter.opensearchSyncIndex(): stopAfter=" + stopAfter +
", needIndexing=" + rtn[0] + ", needRemoval=" + rtn[1]);
int ct = 0;
for (String id : needIndexing) {
Encounter enc = myShepherd.getEncounter(id);
try {
if (enc != null) os.index(indexName, enc);
} catch (Exception ex) {
System.out.println("Encounter.opensearchSyncIndex(): index failed " + enc + " => " +
ex.toString());
ex.printStackTrace();
}
if (ct % 500 == 0)
System.out.println("Encounter.opensearchSyncIndex needIndexing: " + ct + "/" +
rtn[0]);
ct++;
if ((stopAfter > 0) && (ct > stopAfter)) {
System.out.println("Encounter.opensearchSyncIndex() breaking due to stopAfter");
break;
}
}
System.out.println("Encounter.opensearchSyncIndex() finished needIndexing");
ct = 0;
for (String id : needRemoval) {
os.delete(indexName, id);
if (ct % 500 == 0)
System.out.println("Encounter.opensearchSyncIndex needRemoval: " + ct + "/" +
rtn[1]);
ct++;
}
System.out.println("Encounter.opensearchSyncIndex() finished needRemoval");
OpenSearch.unsetActiveIndexingBackground();
return rtn;
}

public static Base createFromApi(org.json.JSONObject payload, List<File> files,
Shepherd myShepherd)
throws ApiException {
Expand Down Expand Up @@ -4704,5 +4646,4 @@ public void sendCreationEmails(Shepherd myShepherd, String langCode) {
myShepherd.rollbackDBTransaction();
}
}

}
Loading

0 comments on commit 6a24192

Please sign in to comment.