Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deep handle mongo ID_FIELD when useObjectId #309

Merged
merged 5 commits into from
Nov 8, 2023
Merged

Deep handle mongo ID_FIELD when useObjectId #309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
aca97d7
Deep encode key when use object id
liuchong Oct 12, 2023
931355c
Use deepEncodeKeyWhenUseObjectId only in query or filter
liuchong Oct 12, 2023
49adb45
Fix deepEncodeKeyWhenUseObjectId should not use on bulk INSERT
liuchong Oct 24, 2023
9053850
Add unit tests for deepEncodeKeyWhenUseObjectId for MongoClient
liuchong Oct 24, 2023
469603c
Handle Map and List for deepEncodeKeyWhenUseObjectId
liuchong Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 71 additions & 17 deletions src/main/java/io/vertx/ext/mongo/impl/MongoClientImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,9 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -207,7 +209,7 @@ public Future<Void> close() {
requireNonNull(options, OPTIONS_CANNOT_BE_NULL);

MongoCollection<JsonObject> coll = getCollection(collection, options.getWriteOption());
Bson bquery = wrap(encodeKeyWhenUseObjectId(query));
Bson bquery = wrap(deepEncodeKeyWhenUseObjectId(query));
Bson bupdate = wrap(encodeKeyWhenUseObjectId(generateIdIfNeeded(query, update, options)));

com.mongodb.client.model.UpdateOptions updateOptions = new com.mongodb.client.model.UpdateOptions().upsert(options.isUpsert());
Expand Down Expand Up @@ -246,7 +248,7 @@ public Future<Void> close() {
requireNonNull(options, OPTIONS_CANNOT_BE_NULL);

MongoCollection<JsonObject> coll = getCollection(collection, options.getWriteOption());
Bson bquery = wrap(encodeKeyWhenUseObjectId(query));
Bson bquery = wrap(deepEncodeKeyWhenUseObjectId(query));
List<Bson> bpipeline = new ArrayList<>(pipeline.size());
for (int i=0 ; i<pipeline.size() ; i++) {
bpipeline.add(wrap(pipeline.getJsonObject(i)));
Expand Down Expand Up @@ -305,7 +307,7 @@ private JsonObject generateIdIfNeeded(JsonObject query, JsonObject update, Updat
requireNonNull(options, OPTIONS_CANNOT_BE_NULL);

MongoCollection<JsonObject> coll = getCollection(collection, options.getWriteOption());
Bson bquery = wrap(encodeKeyWhenUseObjectId(query));
Bson bquery = wrap(deepEncodeKeyWhenUseObjectId(query));
com.mongodb.client.model.ReplaceOptions replaceOptions = new com.mongodb.client.model.ReplaceOptions().upsert(options.isUpsert());
if (options.getHint() != null) {
replaceOptions.hint(wrap(options.getHint()));
Expand All @@ -332,7 +334,7 @@ public Future<List<JsonObject>> findWithOptions(String collection, JsonObject qu
requireNonNull(query, QUERY_CANNOT_BE_NULL);

Promise<List<JsonObject>> promise = vertx.promise();
doFind(collection, encodeKeyWhenUseObjectId(query), options)
doFind(collection, deepEncodeKeyWhenUseObjectId(query), options)
.subscribe(new MappingAndBufferingSubscriber<>(this::decodeKeyWhenUseObjectId, promise));
return promise.future();
}
Expand All @@ -355,7 +357,7 @@ public ReadStream<JsonObject> findBatchWithOptions(String collection, JsonObject
requireNonNull(collection, COLLECTION_CANNOT_BE_NULL);
requireNonNull(query, QUERY_CANNOT_BE_NULL);

JsonObject encodedQuery = encodeKeyWhenUseObjectId(query);
JsonObject encodedQuery = deepEncodeKeyWhenUseObjectId(query);

Bson bquery = wrap(encodedQuery);
Bson bfields = wrap(fields);
Expand All @@ -377,7 +379,7 @@ public ReadStream<JsonObject> findBatchWithOptions(String collection, JsonObject
requireNonNull(findOptions, FIND_OPTIONS_CANNOT_BE_NULL);
requireNonNull(updateOptions, "update options cannot be null");

JsonObject encodedQuery = encodeKeyWhenUseObjectId(query);
JsonObject encodedQuery = deepEncodeKeyWhenUseObjectId(query);

Bson bquery = wrap(encodedQuery);
Bson bupdate = wrap(update);
Expand Down Expand Up @@ -428,7 +430,7 @@ public ReadStream<JsonObject> findBatchWithOptions(String collection, JsonObject
requireNonNull(findOptions, FIND_OPTIONS_CANNOT_BE_NULL);
requireNonNull(updateOptions, "update options cannot be null");

JsonObject encodedQuery = encodeKeyWhenUseObjectId(query);
JsonObject encodedQuery = deepEncodeKeyWhenUseObjectId(query);

Bson bquery = wrap(encodedQuery);
FindOneAndReplaceOptions foarOptions = new FindOneAndReplaceOptions();
Expand Down Expand Up @@ -470,7 +472,7 @@ public ReadStream<JsonObject> findBatchWithOptions(String collection, JsonObject
requireNonNull(query, QUERY_CANNOT_BE_NULL);
requireNonNull(findOptions, FIND_OPTIONS_CANNOT_BE_NULL);

JsonObject encodedQuery = encodeKeyWhenUseObjectId(query);
JsonObject encodedQuery = deepEncodeKeyWhenUseObjectId(query);

Bson bquery = wrap(encodedQuery);
FindOneAndDeleteOptions foadOptions = new FindOneAndDeleteOptions();
Expand Down Expand Up @@ -504,7 +506,7 @@ public Future<Long> countWithOptions(String collection, JsonObject query, CountO
requireNonNull(collection, COLLECTION_CANNOT_BE_NULL);
requireNonNull(query, QUERY_CANNOT_BE_NULL);

Bson bquery = wrap(encodeKeyWhenUseObjectId(query));
Bson bquery = wrap(deepEncodeKeyWhenUseObjectId(query));
MongoCollection<JsonObject> coll = getCollection(collection);
Promise<Long> promise = vertx.promise();
Publisher<Long> countPublisher = countOptions != null
Expand All @@ -525,7 +527,7 @@ public Future<Long> countWithOptions(String collection, JsonObject query, CountO
requireNonNull(query, QUERY_CANNOT_BE_NULL);

MongoCollection<JsonObject> coll = getCollection(collection, writeOption);
Bson bquery = wrap(encodeKeyWhenUseObjectId(query));
Bson bquery = wrap(deepEncodeKeyWhenUseObjectId(query));
Promise<DeleteResult> promise = vertx.promise();
coll.deleteMany(bquery).subscribe(new SingleResultSubscriber<>(promise));
return promise.future().map(Utils::toMongoClientDeleteResult);
Expand All @@ -542,7 +544,7 @@ public Future<Long> countWithOptions(String collection, JsonObject query, CountO
requireNonNull(query, QUERY_CANNOT_BE_NULL);

MongoCollection<JsonObject> coll = getCollection(collection, writeOption);
Bson bquery = wrap(encodeKeyWhenUseObjectId(query));
Bson bquery = wrap(deepEncodeKeyWhenUseObjectId(query));
Promise<DeleteResult> promise = vertx.promise();
coll.deleteOne(bquery).subscribe(new SingleResultSubscriber<>(promise));
return promise.future().map(Utils::toMongoClientDeleteResult);
Expand Down Expand Up @@ -571,7 +573,7 @@ private List<WriteModel<JsonObject>> convertBulkOperations(List<BulkOperation> o
for (BulkOperation bulkOperation : operations) {
switch (bulkOperation.getType()) {
case DELETE:
Bson bsonFilter = toBson(encodeKeyWhenUseObjectId(bulkOperation.getFilter()));
Bson bsonFilter = toBson(deepEncodeKeyWhenUseObjectId(bulkOperation.getFilter()));
DeleteOptions deleteOptions = new DeleteOptions();
if (bulkOperation.getHint() != null) {
deleteOptions.hint(toBson(bulkOperation.getHint()));
Expand Down Expand Up @@ -602,11 +604,11 @@ private List<WriteModel<JsonObject>> convertBulkOperations(List<BulkOperation> o
if (bulkOperation.getHintString() != null && !bulkOperation.getHintString().isEmpty()) {
replaceOptions.hintString(bulkOperation.getHintString());
}
result.add(new ReplaceOneModel<>(toBson(encodeKeyWhenUseObjectId(bulkOperation.getFilter())), bulkOperation.getDocument(),
result.add(new ReplaceOneModel<>(toBson(deepEncodeKeyWhenUseObjectId(bulkOperation.getFilter())), bulkOperation.getDocument(),
replaceOptions.upsert(bulkOperation.isUpsert())));
break;
case UPDATE:
Bson filter = toBson(encodeKeyWhenUseObjectId(bulkOperation.getFilter()));
Bson filter = toBson(deepEncodeKeyWhenUseObjectId(bulkOperation.getFilter()));
Bson document = toBson(encodeKeyWhenUseObjectId(bulkOperation.getDocument()));
com.mongodb.client.model.UpdateOptions updateOptions = new com.mongodb.client.model.UpdateOptions()
.upsert(bulkOperation.isUpsert());
Expand Down Expand Up @@ -872,7 +874,7 @@ private DistinctPublisher<?> findDistinctValuesWithQuery(String collection, Stri
requireNonNull(fieldName, FIELD_NAME_CANNOT_BE_NULL);
requireNonNull(query, QUERY_CANNOT_BE_NULL);

JsonObject encodedQuery = encodeKeyWhenUseObjectId(query);
JsonObject encodedQuery = deepEncodeKeyWhenUseObjectId(query);

Bson bquery = wrap(encodedQuery);

Expand Down Expand Up @@ -905,8 +907,60 @@ private AggregatePublisher<JsonObject> doAggregate(final String collection, fina
return aggregate;
}


JsonArray deepEncodeKeyWhenUseObjectId(JsonArray arr) {
if(!useObjectId) return arr;

JsonArray newArr = new JsonArray(new ArrayList<>(arr.size()));

for (Object item : arr) {
if (item instanceof JsonArray) {
newArr.add(deepEncodeKeyWhenUseObjectId((JsonArray) item));
} else if(item instanceof List) {
newArr.add(deepEncodeKeyWhenUseObjectId(new JsonArray((List) item)));
} else if (item instanceof JsonObject) {
newArr.add(deepEncodeKeyWhenUseObjectId((JsonObject) item));
} else if (item instanceof Map) {
newArr.add(deepEncodeKeyWhenUseObjectId(new JsonObject((Map) item)));
} else {
newArr.add(item);
}
}

return newArr;
}

JsonObject deepEncodeKeyWhenUseObjectId(JsonObject json) {
if(!useObjectId) return json;

JsonObject newJson = new JsonObject(new LinkedHashMap<>(json.size()));

for (Map.Entry<String, Object> entry : json) {
String key = entry.getKey();
Object value = entry.getValue();
if (key.equals(ID_FIELD)
&& value instanceof String
&& ObjectId.isValid((String) value)) {
newJson.put(key, new JsonObject().put(JsonObjectCodec.OID_FIELD, value));
} else if (value instanceof JsonObject) {
newJson.put(key, deepEncodeKeyWhenUseObjectId((JsonObject) value));
} else if (value instanceof Map) {
newJson.put(key, deepEncodeKeyWhenUseObjectId(new JsonObject((Map) value)));
} else if (value instanceof JsonArray) {
newJson.put(key, deepEncodeKeyWhenUseObjectId((JsonArray) value));
} else if (value instanceof List) {
newJson.put(key, deepEncodeKeyWhenUseObjectId(new JsonArray((List) value)));
} else {
newJson.put(key, value);
}
}

return newJson;
}

JsonObject encodeKeyWhenUseObjectId(JsonObject json) {
tsegismont marked this conversation as resolved.
Show resolved Hide resolved
if (!useObjectId) return json;
if (!useObjectId)
return json;

Object idString = json.getValue(ID_FIELD, null);
if (idString instanceof String && ObjectId.isValid((String) idString)) {
Expand All @@ -932,7 +986,7 @@ private JsonObject decodeKeyWhenUseObjectId(JsonObject json) {

private FindPublisher<JsonObject> doFind(String collection, JsonObject query, FindOptions options) {
MongoCollection<JsonObject> coll = getCollection(collection);
Bson bquery = wrap(encodeKeyWhenUseObjectId(query));
Bson bquery = wrap(deepEncodeKeyWhenUseObjectId(query));
FindPublisher<JsonObject> find = coll.find(bquery, JsonObject.class);
if (options.getLimit() != -1) {
find.limit(options.getLimit());
Expand Down
89 changes: 89 additions & 0 deletions src/test/java/io/vertx/ext/mongo/MongoClientWithObjectIdTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package io.vertx.ext.mongo;

import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import org.bson.types.ObjectId;
import org.junit.Test;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;

import static io.vertx.ext.mongo.WriteOption.ACKNOWLEDGED;
Expand Down Expand Up @@ -77,6 +81,32 @@ public void testFindOneReturnsStringId() throws Exception {
await();
}

@Test
public void testFindOneWithNestedQueryReturnsStringId() throws Exception {
String collection = randomCollection();
mongoClient.createCollection(collection).onComplete(onSuccess(res -> {
JsonObject orig = createDoc();
JsonObject doc = orig.copy();
String objectId = getObjectId(doc);
JsonObject query = JsonObject.of("$and", JsonArray.of(
JsonObject.of("foo", "bar"),
JsonObject.of("_id", objectId)));
mongoClient.insert(collection, doc).onComplete(onSuccess(id -> {
// no auto-generated objectId from mongo
assertNull(id);
mongoClient.findOne(collection, query, null).onComplete(onSuccess(obj -> {
assertTrue(obj.containsKey("_id"));
assertTrue(obj.getValue("_id") instanceof String);
obj.remove("_id");
// nested "_id" will not be modified when insert
assertEquals(orig, obj);
testComplete();
}));
}));
}));
await();
}

@Test
public void testFindOneReturnsNothing() throws Exception {
String collection = randomCollection();
Expand Down Expand Up @@ -116,6 +146,65 @@ public void testFindReturnsStringId() throws Exception {
await();
}


@Test
public void testFindWithNestedQueryReturnsStringId() throws Exception {
String collection = randomCollection();
mongoClient.createCollection(collection).onComplete(onSuccess(res -> {
JsonObject orig = createDoc();
JsonObject doc = orig.copy();
String objectId = getObjectId(doc);
JsonObject query = JsonObject.of("$and", JsonArray.of(
JsonObject.of("foo", "bar"),
JsonObject.of("_id", objectId)));
mongoClient.insert(collection, doc).onComplete(onSuccess(id -> {
// no auto-generated objectId from mongo
assertNull(id);
mongoClient.find(collection, query).onComplete(onSuccess(list -> {
assertTrue(list.size() == 1);
JsonObject obj = list.get(0);
assertTrue(obj.containsKey("_id"));
assertTrue(obj.getValue("_id") instanceof String);
obj.remove("_id");
// nested "_id" will not be modified when insert
assertEquals(orig, obj);
testComplete();
}));
}));
}));
await();
}

@Test
public void testFindWithNestedQueryWithListMapReturnsStringId() throws Exception {
String collection = randomCollection();
mongoClient.createCollection(collection).onComplete(onSuccess(res -> {
JsonObject orig = createDoc();
JsonObject doc = orig.copy();
String objectId = getObjectId(doc);
Map<String, String> m1 = new HashMap<>();
m1.put("foo", "bar");
Map<String, String> m2 = new HashMap<>();
m2.put("_id", objectId);
JsonObject query = JsonObject.of("$and", Arrays.asList(m1, m2));
mongoClient.insert(collection, doc).onComplete(onSuccess(id -> {
// no auto-generated objectId from mongo
assertNull(id);
mongoClient.find(collection, query).onComplete(onSuccess(list -> {
assertTrue(list.size() == 1);
JsonObject obj = list.get(0);
assertTrue(obj.containsKey("_id"));
assertTrue(obj.getValue("_id") instanceof String);
obj.remove("_id");
// nested "_id" will not be modified when insert
assertEquals(orig, obj);
testComplete();
}));
}));
}));
await();
}

@Test
@Override
public void testInsertPreexistingObjectID() throws Exception {
Expand Down
33 changes: 32 additions & 1 deletion src/test/java/io/vertx/ext/mongo/MongoTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,14 @@ protected JsonObject createDoc() {
.put("myarr", new JsonArray()
.add("blah")
.add(true)
.add(312)));
.add(312)))
.put("nested_id1", new JsonObject()
.put("_id", new ObjectId().toHexString()))
.put("nested_id2", new JsonArray()
.add(new JsonObject()
.put("_id", new ObjectId().toHexString()))
.add(new JsonObject()
.put("_id", new ObjectId().toHexString())));
}

protected JsonObject createDoc(int num) {
Expand Down Expand Up @@ -210,4 +217,28 @@ protected JsonObject createDocWithAmbiguitiesDependingOnLocale(int num) {
.put("longval", 123456789L).put("dblval", 1.23);
}

// WARN: try to getObjectId from doc will generate new objectId on doc if not exists
protected String getObjectId(JsonObject doc) {
Object idVal = doc.getValue("_id");

// auto generate when not exists
if(idVal == null) {
String _id = new ObjectId().toHexString();
doc.put("_id", JsonObject.of("$oid", _id));
return _id;
}

// return string
if(idVal instanceof String) {
return (String) idVal;
}

// return $oid from ObjectId object
if(idVal instanceof JsonObject) {
return ((JsonObject) idVal).getString("$oid");
}

return null;
}

}
Loading