From 300592f8ce8773a656d4e73a19e9288cbae6f3bc Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 14 Feb 2024 19:31:25 +0530 Subject: [PATCH 01/15] feat: Add an api to add bulk import users --- .../java/io/supertokens/inmemorydb/Start.java | 12 +- .../inmemorydb/config/SQLiteConfig.java | 4 + .../inmemorydb/queries/BulkImportQueries.java | 68 +++ .../inmemorydb/queries/GeneralQueries.java | 6 + .../io/supertokens/webserver/Webserver.java | 3 + .../api/bulkimport/AddBulkImportUsers.java | 124 ++++++ .../apis/AddBulkImportUsersTest.java | 408 ++++++++++++++++++ 7 files changed, 624 insertions(+), 1 deletion(-) create mode 100644 src/main/java/io/supertokens/inmemorydb/queries/BulkImportQueries.java create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index 129adee22..2cb372551 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -26,6 +26,8 @@ import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.dashboard.DashboardSearchTags; import io.supertokens.pluginInterface.dashboard.DashboardSessionInfo; import io.supertokens.pluginInterface.dashboard.DashboardUser; @@ -102,7 +104,7 @@ public class Start implements SessionSQLStorage, EmailPasswordSQLStorage, EmailVerificationSQLStorage, ThirdPartySQLStorage, JWTRecipeSQLStorage, PasswordlessSQLStorage, UserMetadataSQLStorage, UserRolesSQLStorage, UserIdMappingStorage, UserIdMappingSQLStorage, MultitenancyStorage, MultitenancySQLStorage, TOTPSQLStorage, ActiveUsersStorage, - DashboardSQLStorage, AuthRecipeSQLStorage { + DashboardSQLStorage, AuthRecipeSQLStorage, BulkImportStorage { private static final Object appenderLock = new Object(); private static final String APP_ID_KEY_NAME = "app_id"; @@ -2952,4 +2954,12 @@ public UserIdMapping[] getUserIdMapping_Transaction(TransactionConnection con, A } } + @Override + public void addBulkImportUsers(AppIdentifier appIdentifier, ArrayList users) throws StorageQueryException { + try { + BulkImportQueries.insertBulkImportUsers(this, users); + } catch (SQLException e) { + throw new StorageQueryException(e); + } + } } diff --git a/src/main/java/io/supertokens/inmemorydb/config/SQLiteConfig.java b/src/main/java/io/supertokens/inmemorydb/config/SQLiteConfig.java index 25fd59c61..9ce5cf561 100644 --- a/src/main/java/io/supertokens/inmemorydb/config/SQLiteConfig.java +++ b/src/main/java/io/supertokens/inmemorydb/config/SQLiteConfig.java @@ -156,4 +156,8 @@ public String getDashboardUsersTable() { public String getDashboardSessionsTable() { return "dashboard_user_sessions"; } + + public String getBulkImportUsersTable() { + return "bulk_import_users"; + } } diff --git a/src/main/java/io/supertokens/inmemorydb/queries/BulkImportQueries.java b/src/main/java/io/supertokens/inmemorydb/queries/BulkImportQueries.java new file mode 100644 index 000000000..aeb6fe9e6 --- /dev/null +++ b/src/main/java/io/supertokens/inmemorydb/queries/BulkImportQueries.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + + +package io.supertokens.inmemorydb.queries; + +import io.supertokens.inmemorydb.config.Config; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; + +import static io.supertokens.inmemorydb.PreparedStatementValueSetter.NO_OP_SETTER; +import static io.supertokens.inmemorydb.QueryExecutorTemplate.update; + +import java.sql.SQLException; +import java.util.ArrayList; + +import io.supertokens.inmemorydb.Start; + +public class BulkImportQueries { + static String getQueryToCreateBulkImportUsersTable(Start start) { + return "CREATE TABLE IF NOT EXISTS " + Config.getConfig(start).getBulkImportUsersTable() + " (" + + "id CHAR(36) PRIMARY KEY," + + "raw_data TEXT NOT NULL," + + "status VARCHAR(128) NOT NULL DEFAULT 'NEW'," + + "error_msg TEXT," + + "created_at TIMESTAMP DEFAULT (strftime('%s', 'now'))," + + "updated_at TIMESTAMP DEFAULT (strftime('%s', 'now'))" + + " );"; + } + + public static String getQueryToCreateStatusUpdatedAtIndex(Start start) { + return "CREATE INDEX IF NOT EXISTS bulk_import_users_status_updated_at_index ON " + + Config.getConfig(start).getBulkImportUsersTable() + " (status, updated_at)"; + } + + public static void insertBulkImportUsers(Start start, ArrayList users) + throws SQLException, StorageQueryException { + StringBuilder queryBuilder = new StringBuilder( + "INSERT INTO " + Config.getConfig(start).getBulkImportUsersTable() + " (id, raw_data) VALUES "); + for (BulkImportUser user : users) { + queryBuilder.append("('") + .append(user.id) + .append("', '") + .append(user.toString()) + .append("')"); + + if (user != users.get(users.size() - 1)) { + queryBuilder.append(","); + } + } + queryBuilder.append(";"); + update(start, queryBuilder.toString(), NO_OP_SETTER); + } +} + diff --git a/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java b/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java index c645f2f7a..4ac2f32a9 100644 --- a/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java +++ b/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java @@ -407,6 +407,12 @@ public static void createTablesIfNotExists(Start start, Main main) throws SQLExc update(start, TOTPQueries.getQueryToCreateUsedCodesExpiryTimeIndex(start), NO_OP_SETTER); } + if (!doesTableExists(start, Config.getConfig(start).getBulkImportUsersTable())) { + getInstance(main).addState(CREATING_NEW_TABLE, null); + update(start, BulkImportQueries.getQueryToCreateBulkImportUsersTable(start), NO_OP_SETTER); + // index: + update(start, BulkImportQueries.getQueryToCreateStatusUpdatedAtIndex(start), NO_OP_SETTER); + } } diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 8763843e2..1664fffe2 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -27,6 +27,7 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifierWithStorage; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; +import io.supertokens.webserver.api.bulkimport.AddBulkImportUsers; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -259,6 +260,8 @@ private void setupRoutes() { addAPI(new RequestStatsAPI(main)); + addAPI(new AddBulkImportUsers(main)); + StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java b/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java new file mode 100644 index 000000000..0f98470a1 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import io.supertokens.Main; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +public class AddBulkImportUsers extends WebserverAPI { + private static final int MAX_USERS_TO_ADD = 10000; + + public AddBulkImportUsers(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/add-users"; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); + + if (users.size() > MAX_USERS_TO_ADD) { + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.addProperty("error", "You can only add 1000 users at a time."); + super.sendJsonResponse(400, errorResponseJson, resp); + return; + } + + AppIdentifier appIdentifier = null; + try { + appIdentifier = getTenantIdentifierFromRequest(req).toAppIdentifier(); + } catch (ServletException e) { + throw new ServletException(e); + } + + TenantConfig[] allTenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + ArrayList validTenantIds = new ArrayList<>(); + Arrays.stream(allTenantConfigs) + .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); + + JsonArray errorsJson = new JsonArray(); + ArrayList usersToAdd = new ArrayList<>(); + + for (int i = 0; i < users.size(); i++) { + try { + usersToAdd.add(new BulkImportUser(users.get(i).getAsJsonObject(), validTenantIds, null)); + } catch (InvalidBulkImportDataException e) { + JsonObject errorObj = new JsonObject(); + + JsonArray errors = e.errors.stream() + .map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + + errorObj.addProperty("index", i); + errorObj.add("errors", errors); + + errorsJson.add(errorObj); + } catch (Exception e) { + JsonObject errorObj = new JsonObject(); + errorObj.addProperty("index", i); + errorObj.addProperty("errors", "An unknown error occurred"); + errorsJson.add(errorObj); + } + } + + if (errorsJson.size() > 0) { + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.addProperty("error", + "Data has missing or invalid fields. Please check the users field for more details."); + errorResponseJson.add("users", errorsJson); + super.sendJsonResponse(400, errorResponseJson, resp); + return; + } + + try { + AppIdentifierWithStorage appIdentifierWithStorage = getAppIdentifierWithStorage(req); + BulkImportStorage storage = appIdentifierWithStorage.getBulkImportStorage(); + storage.addBulkImportUsers(appIdentifierWithStorage, usersToAdd); + } catch (Exception e) { + throw new ServletException(e); + } + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + super.sendJsonResponse(200, result, resp); + + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java new file mode 100644 index 000000000..abc5d1aa0 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.UUID; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.ProcessState; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + +public class AddBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + public String getResponseMessageFromError(String response) { + return response.substring(response.indexOf("Message: ") + "Message: ".length()); + } + + @Test + public void shouldThrow400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + String genericErrMsg = "Data has missing or invalid fields. Please check the users field for more details."; + + // users is required in the json body + { + // CASE 1: users field is not present + try { + JsonObject request = new JsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "Field name 'users' is invalid in JSON input"); + } + // CASE 2: users field type in incorrect + try { + JsonObject request = new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "Field name 'users' is invalid in JSON input"); + } + } + // loginMethod array is required in the user object + { + // CASE 1: loginMethods field is not present + try { + JsonObject request = new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); + } + // CASE 2: loginMethods field type in incorrect + try { + JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); + } + // CASE 3: loginMethods array is empty + try { + JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); + } + } + // Invalid field type of non required fields outside loginMethod + { + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"roles\":{},\"totp\":{}}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"roles should be of type array of string.\",\"totp should be of type array of object.\",\"loginMethods is required.\"]}]}"); + } + } + // Invalid field type of non required fields inside loginMethod + { + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantId\":[],\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantId should be of type string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type number for a loginMethod\"]}]}"); + } + } + // Invalid recipeId + { + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); + } + } + // Invalid field type in emailpassword recipe + { + // CASE 1: email, passwordHash and hashingAlgorithm are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); + } + // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":[],\"passwordHash\":[],\"hashingAlgorithm\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); + } + // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"somehash\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); + } + } + // Invalid field type in thirdparty recipe + { + // CASE 1: email, thirdPartyId and thirdPartyUserId are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); + } + // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\",\"email\":[],\"thirdPartyId\":[],\"thirdPartyUserId\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); + } + } + // Invalid field type in passwordless recipe + { + // CASE 1: email and phoneNumber are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + } + // CASE 2: email and phoneNumber field type is incorrect + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":[],\"phoneNumber\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\"]}]}"); + } + } + // No two loginMethods can have isPrimary as true + { + // CASE 1: email, passwordHash and hashingAlgorithm are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"somehash\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); + } + } + // Can't import than 10000 users at a time + { + try { + JsonObject request = generateUsersJson(10001); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"You can only add 1000 users at a time.\"}"); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + JsonObject request = generateUsersJson(10000); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + public static JsonObject generateUsersJson(int numberOfUsers) { + JsonObject userJsonObject = new JsonObject(); + JsonParser parser = new JsonParser(); + + JsonArray usersArray = new JsonArray(); + for (int i = 0; i < numberOfUsers; i++) { + JsonObject user = new JsonObject(); + + user.addProperty("externalUserId", UUID.randomUUID().toString()); + user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); + user.add("roles", parser.parse("[\"role1\", \"role2\"]")); + user.add("totp", parser.parse("[{\"secretKey\":\"secretKey\",\"period\":0,\"skew\":0,\"deviceName\":\"deviceName\"}]")); + + String email = "johndoe+" + i + "@gmail.com"; + + JsonArray loginMethodsArray = new JsonArray(); + loginMethodsArray.add(createEmailLoginMethod(email)); + loginMethodsArray.add(createThirdPartyLoginMethod(email)); + loginMethodsArray.add(createPasswordlessLoginMethod(email)); + user.add("loginMethods", loginMethodsArray); + + usersArray.add(user); + } + + userJsonObject.add("users", usersArray); + return userJsonObject; + } + + private static JsonObject createEmailLoginMethod(String email) { + JsonObject loginMethod = new JsonObject(); + loginMethod.addProperty("tenantId", "public"); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "emailpassword"); + loginMethod.addProperty("passwordHash", "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); + loginMethod.addProperty("hashingAlgorithm", "argon2"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", true); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createThirdPartyLoginMethod(String email) { + JsonObject loginMethod = new JsonObject(); + loginMethod.addProperty("tenantId", "somethingelse"); + loginMethod.addProperty("recipeId", "thirdparty"); + loginMethod.addProperty("email", email); + loginMethod.addProperty("thirdPartyId", "google"); + loginMethod.addProperty("thirdPartyUserId", "112618388912586834161"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createPasswordlessLoginMethod(String email) { + JsonObject loginMethod = new JsonObject(); + loginMethod.addProperty("tenantId", "public"); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "passwordless"); + loginMethod.addProperty("phoneNumber", "+91-9999999999"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } +} From bb7d84e063f51525c37656113f9f8f5c4ae1bb59 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Fri, 16 Feb 2024 00:48:15 +0530 Subject: [PATCH 02/15] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 79 +++++++++ .../BulkImportUserPaginationContainer.java | 32 ++++ .../BulkImportUserPaginationToken.java | 45 +++++ .../io/supertokens/webserver/Webserver.java | 2 + .../api/bulkimport/AddBulkImportUsers.java | 76 +++++---- .../api/bulkimport/GetBulkImportUsers.java | 104 ++++++++++++ .../apis/AddBulkImportUsersTest.java | 55 +++++- .../apis/GetBulkImportUsersTest.java | 158 ++++++++++++++++++ 8 files changed, 520 insertions(+), 31 deletions(-) create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImport.java create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java new file mode 100644 index 000000000..4b8f55509 --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.utils.Utils; + +import com.google.gson.JsonObject; + +import java.util.ArrayList; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class BulkImport { + + public static final int GET_USERS_PAGINATION_LIMIT = 500; + public static final int GET_USERS_DEFAULT_LIMIT = 100; + + public static void addUsers(AppIdentifierWithStorage appIdentifierWithStorage, ArrayList users) + throws StorageQueryException, TenantOrAppNotFoundException { + while (true) { + try { + appIdentifierWithStorage.getBulkImportStorage().addBulkImportUsers(appIdentifierWithStorage, users); + break; + } catch (io.supertokens.pluginInterface.bulkimport.exceptions.DuplicateUserIdException ignored) { + // We re-generate the user id for every user and retry + for (BulkImportUser user : users) { + user.id = Utils.getUUID(); + } + } + } + } + + public static BulkImportUserPaginationContainer getUsers(AppIdentifierWithStorage appIdentifierWithStorage, + @Nonnull Integer limit, @Nullable String status, @Nullable String paginationToken) + throws StorageQueryException, BulkImportUserPaginationToken.InvalidTokenException, + TenantOrAppNotFoundException { + JsonObject[] users; + + if (paginationToken == null) { + users = appIdentifierWithStorage.getBulkImportStorage() + .getBulkImportUsers(appIdentifierWithStorage, limit + 1, status, null); + } else { + BulkImportUserPaginationToken tokenInfo = BulkImportUserPaginationToken.extractTokenInfo(paginationToken); + users = appIdentifierWithStorage.getBulkImportStorage() + .getBulkImportUsers(appIdentifierWithStorage, limit + 1, status, tokenInfo.bulkImportUserId); + } + + String nextPaginationToken = null; + int maxLoop = users.length; + if (users.length == limit + 1) { + maxLoop = limit; + nextPaginationToken = new BulkImportUserPaginationToken(users[limit].get("id").getAsString()) + .generateToken(); + } + + JsonObject[] resultUsers = new JsonObject[maxLoop]; + System.arraycopy(users, 0, resultUsers, 0, maxLoop); + return new BulkImportUserPaginationContainer(resultUsers, nextPaginationToken); + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java new file mode 100644 index 000000000..a9e4d644c --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.google.gson.JsonObject; + +public class BulkImportUserPaginationContainer { + public final JsonObject[] users; + public final String nextPaginationToken; + + public BulkImportUserPaginationContainer(@Nonnull JsonObject[] users, @Nullable String nextPaginationToken) { + this.users = users; + this.nextPaginationToken = nextPaginationToken; + } +} \ No newline at end of file diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java new file mode 100644 index 000000000..d66d02041 --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import java.util.Base64; + +public class BulkImportUserPaginationToken { + public final String bulkImportUserId; + + public BulkImportUserPaginationToken(String bulkImportUserId) { + this.bulkImportUserId = bulkImportUserId; + } + + public static BulkImportUserPaginationToken extractTokenInfo(String token) throws InvalidTokenException { + try { + String bulkImportUserId = new String(Base64.getDecoder().decode(token)); + return new BulkImportUserPaginationToken(bulkImportUserId); + } catch (Exception e) { + throw new InvalidTokenException(); + } + } + + public String generateToken() { + return new String(Base64.getEncoder().encode((this.bulkImportUserId).getBytes())); + } + + public static class InvalidTokenException extends Exception { + + private static final long serialVersionUID = 6289026174830695478L; + } +} diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 1664fffe2..6f9c30ef1 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -28,6 +28,7 @@ import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; import io.supertokens.webserver.api.bulkimport.AddBulkImportUsers; +import io.supertokens.webserver.api.bulkimport.GetBulkImportUsers; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -261,6 +262,7 @@ private void setupRoutes() { addAPI(new RequestStatsAPI(main)); addAPI(new AddBulkImportUsers(main)); + addAPI(new GetBulkImportUsers(main)); StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java b/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java index 0f98470a1..0e5d3df46 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java @@ -17,13 +17,18 @@ package io.supertokens.webserver.api.bulkimport; import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlag; import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.InputParser; import io.supertokens.webserver.WebserverAPI; import jakarta.servlet.ServletException; @@ -55,31 +60,27 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); - if (users.size() > MAX_USERS_TO_ADD) { + if (users.size() <= 0 || users.size() > MAX_USERS_TO_ADD) { JsonObject errorResponseJson = new JsonObject(); - errorResponseJson.addProperty("error", "You can only add 1000 users at a time."); - super.sendJsonResponse(400, errorResponseJson, resp); - return; + String errorMsg = users.size() <= 0 ? "You need to add at least one user." + : "You can only add 10000 users at a time."; + errorResponseJson.addProperty("error", errorMsg); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); } - AppIdentifier appIdentifier = null; - try { - appIdentifier = getTenantIdentifierFromRequest(req).toAppIdentifier(); - } catch (ServletException e) { - throw new ServletException(e); - } - - TenantConfig[] allTenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); - ArrayList validTenantIds = new ArrayList<>(); - Arrays.stream(allTenantConfigs) - .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); + AppIdentifier appIdentifier = getTenantIdentifierFromRequest(req).toAppIdentifier(); JsonArray errorsJson = new JsonArray(); ArrayList usersToAdd = new ArrayList<>(); for (int i = 0; i < users.size(); i++) { try { - usersToAdd.add(new BulkImportUser(users.get(i).getAsJsonObject(), validTenantIds, null)); + BulkImportUser user = new BulkImportUser(users.get(i).getAsJsonObject(), null); + usersToAdd.add(user); + + for (BulkImportUser.LoginMethod loginMethod : user.loginMethods) { + validateTenantId(appIdentifier, loginMethod.tenantId, loginMethod.recipeId); + } } catch (InvalidBulkImportDataException e) { JsonObject errorObj = new JsonObject(); @@ -89,12 +90,6 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S errorObj.addProperty("index", i); errorObj.add("errors", errors); - - errorsJson.add(errorObj); - } catch (Exception e) { - JsonObject errorObj = new JsonObject(); - errorObj.addProperty("index", i); - errorObj.addProperty("errors", "An unknown error occurred"); errorsJson.add(errorObj); } } @@ -104,21 +99,46 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S errorResponseJson.addProperty("error", "Data has missing or invalid fields. Please check the users field for more details."); errorResponseJson.add("users", errorsJson); - super.sendJsonResponse(400, errorResponseJson, resp); - return; + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); } try { AppIdentifierWithStorage appIdentifierWithStorage = getAppIdentifierWithStorage(req); - BulkImportStorage storage = appIdentifierWithStorage.getBulkImportStorage(); - storage.addBulkImportUsers(appIdentifierWithStorage, usersToAdd); - } catch (Exception e) { + BulkImport.addUsers(appIdentifierWithStorage, usersToAdd); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { throw new ServletException(e); } JsonObject result = new JsonObject(); result.addProperty("status", "OK"); super.sendJsonResponse(200, result, resp); + } + + private void validateTenantId(AppIdentifier appIdentifier, String tenantId, String recipeId) + throws InvalidBulkImportDataException, ServletException { + if (tenantId == null || tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID)) { + return; + } + + try { + if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .noneMatch(t -> t == EE_FEATURES.MULTI_TENANCY)) { + throw new InvalidBulkImportDataException(new ArrayList<>( + Arrays.asList("Multitenancy must be enabled before importing users to a different tenant."))); + } + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new ServletException(e); + } + TenantConfig[] allTenantConfigs = Multitenancy + .getAllTenantsForApp(appIdentifier, main); + ArrayList validTenantIds = new ArrayList<>(); + Arrays.stream(allTenantConfigs) + .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); + + if (!validTenantIds.contains(tenantId)) { + throw new InvalidBulkImportDataException( + new ArrayList<>(Arrays.asList("Invalid tenantId: " + tenantId + " for " + recipeId + " recipe."))); + } } } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java b/src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java new file mode 100644 index 000000000..f349ac55b --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import java.io.IOException; +import java.util.Arrays; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.bulkimport.BulkImportUserPaginationToken; +import io.supertokens.output.Logging; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.utils.Utils; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class GetBulkImportUsers extends WebserverAPI { + public GetBulkImportUsers(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/users"; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + String status = InputParser.getQueryParamOrThrowError(req, "status", true); + String paginationToken = InputParser.getQueryParamOrThrowError(req, "paginationToken", true); + Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); + + if (limit != null) { + if (limit > BulkImport.GET_USERS_PAGINATION_LIMIT) { + throw new ServletException( + new BadRequestException("Max limit allowed is " + BulkImport.GET_USERS_PAGINATION_LIMIT)); + } else if (limit < 1) { + throw new ServletException(new BadRequestException("limit must a positive integer with min value 1")); + } + } else { + limit = BulkImport.GET_USERS_DEFAULT_LIMIT; + } + + if (status != null + && !Arrays.asList("NEW", "PROCESSING", "FAILED").contains(status)) { + throw new ServletException(new BadRequestException( + "Invalid value for status. Pass one of NEW, PROCESSING or, FAILED!")); + } + + AppIdentifierWithStorage appIdentifierWithStorage = null; + + try { + appIdentifierWithStorage = this.getAppIdentifierWithStorage(req); + } catch (TenantOrAppNotFoundException e) { + throw new ServletException(e); + } + + try { + BulkImportUserPaginationContainer users = BulkImport.getUsers(appIdentifierWithStorage, limit, status, + paginationToken); + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + + JsonArray usersJson = new JsonArray(); + for (JsonObject user : users.users) { + usersJson.add(user); + } + result.add("users", usersJson); + + if (users.nextPaginationToken != null) { + result.addProperty("nextPaginationToken", users.nextPaginationToken); + } + super.sendJsonResponse(200, result, resp); + } catch (BulkImportUserPaginationToken.InvalidTokenException e) { + Logging.debug(main, null, Utils.exceptionStacktraceToString(e)); + throw new ServletException(new BadRequestException("invalid pagination token")); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index abc5d1aa0..06fcd7096 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -33,6 +33,8 @@ import com.google.gson.JsonParser; import io.supertokens.ProcessState; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; @@ -285,6 +287,40 @@ public void shouldThrow400Error() throws Exception { "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\"]}]}"); } } + // Validate tenantId + { + // CASE 1: Different tenantId when multitenancy is not enabled + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantId\":\"invalid\",\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); + } + // CASE 2: Different tenantId when multitenancy is enabled + try { + FeatureFlagTestContent.getInstance(process.getProcess()) + .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{EE_FEATURES.MULTI_TENANCY}); + + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantId\":\"invalid\",\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); + } + } // No two loginMethods can have isPrimary as true { // CASE 1: email, passwordHash and hashingAlgorithm are not present @@ -301,7 +337,20 @@ public void shouldThrow400Error() throws Exception { "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); } } - // Can't import than 10000 users at a time + // Can't import less than 1 user at a time + { + try { + JsonObject request = generateUsersJson(0); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"You need to add at least one user.\"}"); + } + } + // Can't import more than 10000 users at a time { try { JsonObject request = generateUsersJson(10001); @@ -311,7 +360,7 @@ public void shouldThrow400Error() throws Exception { } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); assertEquals(400, e.statusCode); - assertEquals(responseString, "{\"error\":\"You can only add 1000 users at a time.\"}"); + assertEquals(responseString, "{\"error\":\"You can only add 10000 users at a time.\"}"); } } @@ -383,7 +432,7 @@ private static JsonObject createEmailLoginMethod(String email) { private static JsonObject createThirdPartyLoginMethod(String email) { JsonObject loginMethod = new JsonObject(); - loginMethod.addProperty("tenantId", "somethingelse"); + loginMethod.addProperty("tenantId", "public"); loginMethod.addProperty("recipeId", "thirdparty"); loginMethod.addProperty("email", email); loginMethod.addProperty("thirdPartyId", "google"); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java new file mode 100644 index 000000000..7dbba3b04 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.ProcessState; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + +public class GetBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + try { + Map params = new HashMap<>(); + params.put("status", "INVALID_STATUS"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: Invalid value for status. Pass one of NEW, PROCESSING or, FAILED!", + e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("limit", "0"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: limit must a positive integer with min value 1", + e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("limit", "501"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Max limit allowed is 500", e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("paginationToken", "invalid_token"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: invalid pagination token", e.getMessage()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // Create a bulk import user to test the GET API + String rawData = "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}"; + { + JsonObject request = new JsonParser().parse(rawData).getAsJsonObject(); + JsonObject res = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assert res.get("status").getAsString().equals("OK"); + } + + Map params = new HashMap<>(); + JsonObject response = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + JsonArray bulkImportUsers = response.get("users").getAsJsonArray(); + assertEquals(1, bulkImportUsers.size()); + JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); + bulkImportUserJson.get("status").getAsString().equals("NEW"); + bulkImportUserJson.get("raw_data").getAsString().equals(rawData); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } +} From 32625d4ba2edfc865467034b6a7d02e3eb3de295 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 20 Feb 2024 12:09:30 +0530 Subject: [PATCH 03/15] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 29 ++- .../BulkImportUserPaginationContainer.java | 8 +- .../BulkImportUserPaginationToken.java | 16 +- .../bulkimport/BulkImportUserUtils.java | 167 ++++++++++++++++++ .../InvalidBulkImportDataException.java | 33 ++++ .../api/bulkimport/AddBulkImportUsers.java | 42 ++++- .../api/bulkimport/GetBulkImportUsers.java | 30 ++-- .../apis/AddBulkImportUsersTest.java | 51 +++++- .../apis/GetBulkImportUsersTest.java | 4 +- 9 files changed, 333 insertions(+), 47 deletions(-) create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java create mode 100644 src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 4b8f55509..0f95b9cef 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -16,15 +16,16 @@ package io.supertokens.bulkimport; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUserInfo; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.utils.Utils; -import com.google.gson.JsonObject; -import java.util.ArrayList; +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -34,12 +35,12 @@ public class BulkImport { public static final int GET_USERS_PAGINATION_LIMIT = 500; public static final int GET_USERS_DEFAULT_LIMIT = 100; - public static void addUsers(AppIdentifierWithStorage appIdentifierWithStorage, ArrayList users) + public static void addUsers(AppIdentifierWithStorage appIdentifierWithStorage, List users) throws StorageQueryException, TenantOrAppNotFoundException { while (true) { try { - appIdentifierWithStorage.getBulkImportStorage().addBulkImportUsers(appIdentifierWithStorage, users); - break; + appIdentifierWithStorage.getBulkImportStorage().addBulkImportUsers(appIdentifierWithStorage, users); + break; } catch (io.supertokens.pluginInterface.bulkimport.exceptions.DuplicateUserIdException ignored) { // We re-generate the user id for every user and retry for (BulkImportUser user : users) { @@ -50,10 +51,9 @@ public static void addUsers(AppIdentifierWithStorage appIdentifierWithStorage, A } public static BulkImportUserPaginationContainer getUsers(AppIdentifierWithStorage appIdentifierWithStorage, - @Nonnull Integer limit, @Nullable String status, @Nullable String paginationToken) - throws StorageQueryException, BulkImportUserPaginationToken.InvalidTokenException, - TenantOrAppNotFoundException { - JsonObject[] users; + @Nonnull Integer limit, @Nullable BulkImportUserStatus status, @Nullable String paginationToken) + throws StorageQueryException, BulkImportUserPaginationToken.InvalidTokenException { + List users; if (paginationToken == null) { users = appIdentifierWithStorage.getBulkImportStorage() @@ -65,15 +65,14 @@ public static BulkImportUserPaginationContainer getUsers(AppIdentifierWithStorag } String nextPaginationToken = null; - int maxLoop = users.length; - if (users.length == limit + 1) { + int maxLoop = users.size(); + if (users.size() == limit + 1) { maxLoop = limit; - nextPaginationToken = new BulkImportUserPaginationToken(users[limit].get("id").getAsString()) - .generateToken(); + BulkImportUserInfo user = users.get(limit); + nextPaginationToken = new BulkImportUserPaginationToken(user.id, user.createdAt).generateToken(); } - JsonObject[] resultUsers = new JsonObject[maxLoop]; - System.arraycopy(users, 0, resultUsers, 0, maxLoop); + List resultUsers = users.subList(0, maxLoop); return new BulkImportUserPaginationContainer(resultUsers, nextPaginationToken); } } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java index a9e4d644c..2993a83a7 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java @@ -16,16 +16,18 @@ package io.supertokens.bulkimport; +import java.util.List; + import javax.annotation.Nonnull; import javax.annotation.Nullable; -import com.google.gson.JsonObject; +import io.supertokens.pluginInterface.bulkimport.BulkImportUserInfo; public class BulkImportUserPaginationContainer { - public final JsonObject[] users; + public final List users; public final String nextPaginationToken; - public BulkImportUserPaginationContainer(@Nonnull JsonObject[] users, @Nullable String nextPaginationToken) { + public BulkImportUserPaginationContainer(@Nonnull List users, @Nullable String nextPaginationToken) { this.users = users; this.nextPaginationToken = nextPaginationToken; } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java index d66d02041..0c24211a9 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java @@ -20,22 +20,30 @@ public class BulkImportUserPaginationToken { public final String bulkImportUserId; + public final long createdAt; - public BulkImportUserPaginationToken(String bulkImportUserId) { + public BulkImportUserPaginationToken(String bulkImportUserId, long timeJoined) { this.bulkImportUserId = bulkImportUserId; + this.createdAt = timeJoined; } public static BulkImportUserPaginationToken extractTokenInfo(String token) throws InvalidTokenException { try { - String bulkImportUserId = new String(Base64.getDecoder().decode(token)); - return new BulkImportUserPaginationToken(bulkImportUserId); + String decodedPaginationToken = new String(Base64.getDecoder().decode(token)); + String[] splitDecodedToken = decodedPaginationToken.split(";"); + if (splitDecodedToken.length != 2) { + throw new InvalidTokenException(); + } + String bulkImportUserId = splitDecodedToken[0]; + long timeJoined = Long.parseLong(splitDecodedToken[1]); + return new BulkImportUserPaginationToken(bulkImportUserId, timeJoined); } catch (Exception e) { throw new InvalidTokenException(); } } public String generateToken() { - return new String(Base64.getEncoder().encode((this.bulkImportUserId).getBytes())); + return new String(Base64.getEncoder().encode((this.bulkImportUserId + ";" + this.createdAt).getBytes())); } public static class InvalidTokenException extends Exception { diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java new file mode 100644 index 000000000..43915743a --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import java.util.ArrayList; +import java.util.List; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.EmailPasswordLoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.ThirdPartyLoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.PasswordlessLoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.utils.JsonValidatorUtils.ValueType; +import io.supertokens.utils.Utils; + +import static io.supertokens.pluginInterface.utils.JsonValidatorUtils.parseAndValidateField; +import static io.supertokens.pluginInterface.utils.JsonValidatorUtils.validateJsonFieldType; + +public class BulkImportUserUtils { + public static BulkImportUser createBulkImportUserFromJSON(JsonObject userData, String id) throws InvalidBulkImportDataException { + List errors = new ArrayList<>(); + + String externalUserId = parseAndValidateField(userData, "externalUserId", ValueType.STRING, false, String.class, errors, "."); + JsonObject userMetadata = parseAndValidateField(userData, "userMetadata", ValueType.OBJECT, false, JsonObject.class, errors, "."); + List userRoles = getParsedUserRoles(userData, errors); + List totpDevices = getParsedTotpDevices(userData, errors); + List loginMethods = getParsedLoginMethods(userData, errors); + + if (!errors.isEmpty()) { + throw new InvalidBulkImportDataException(errors); + } + return new BulkImportUser(id, externalUserId, userMetadata, userRoles, totpDevices, loginMethods); + } + + private static List getParsedUserRoles(JsonObject userData, List errors) { + JsonArray jsonUserRoles = parseAndValidateField(userData, "roles", ValueType.ARRAY_OF_STRING, + false, + JsonArray.class, errors, "."); + + if (jsonUserRoles == null) { + return null; + } + + List userRoles = new ArrayList<>(); + jsonUserRoles.forEach(role -> userRoles.add(role.getAsString().trim())); + return userRoles; + } + + private static List getParsedTotpDevices(JsonObject userData, List errors) { + JsonArray jsonTotpDevices = parseAndValidateField(userData, "totp", ValueType.ARRAY_OF_OBJECT, false, JsonArray.class, errors, "."); + if (jsonTotpDevices == null) { + return null; + } + + List totpDevices = new ArrayList<>(); + for (JsonElement jsonTotpDeviceEl : jsonTotpDevices) { + JsonObject jsonTotpDevice = jsonTotpDeviceEl.getAsJsonObject(); + + String secretKey = parseAndValidateField(jsonTotpDevice, "secretKey", ValueType.STRING, true, String.class, errors, " for a totp device."); + Number period = parseAndValidateField(jsonTotpDevice, "period", ValueType.NUMBER, true, Number.class, errors, " for a totp device."); + Number skew = parseAndValidateField(jsonTotpDevice, "skew", ValueType.NUMBER, true, Number.class, errors, " for a totp device."); + String deviceName = parseAndValidateField(jsonTotpDevice, "deviceName", ValueType.STRING, false, String.class, errors, " for a totp device."); + + if (period != null && period.intValue() < 1) { + errors.add("period should be > 0 for a totp device."); + } + if (skew != null && skew.intValue() < 0) { + errors.add("skew should be >= 0 for a totp device."); + } + + if(deviceName != null) { + deviceName = deviceName.trim(); + } + totpDevices.add(new TotpDevice(secretKey, period.intValue(), skew.intValue(), deviceName)); + } + return totpDevices; + } + + private static List getParsedLoginMethods(JsonObject userData, List errors) { + JsonArray jsonLoginMethods = parseAndValidateField(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, true, JsonArray.class, errors, "."); + + if (jsonLoginMethods == null) { + return new ArrayList<>(); + } + + if (jsonLoginMethods.size() == 0) { + errors.add("At least one loginMethod is required."); + return new ArrayList<>(); + } + + Boolean hasPrimaryLoginMethod = false; + + List loginMethods = new ArrayList<>(); + for (JsonElement jsonLoginMethod : jsonLoginMethods) { + JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); + + if (validateJsonFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN)) { + if (jsonLoginMethodObj.get("isPrimary").getAsBoolean()) { + if (hasPrimaryLoginMethod) { + errors.add("No two loginMethods can have isPrimary as true."); + } + hasPrimaryLoginMethod = true; + } + } + + String recipeId = parseAndValidateField(jsonLoginMethodObj, "recipeId", ValueType.STRING, true, String.class, errors, " for a loginMethod."); + String tenantId = parseAndValidateField(jsonLoginMethodObj, "tenantId", ValueType.STRING, false, String.class, errors, " for a loginMethod."); + Boolean isVerified = parseAndValidateField(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); + Boolean isPrimary = parseAndValidateField(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); + Number timeJoined = parseAndValidateField(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.NUMBER, false, Number.class, errors, " for a loginMethod"); + Long timeJoinedInMSSinceEpoch = timeJoined != null ? timeJoined.longValue() : 0; + + if ("emailpassword".equals(recipeId)) { + String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + String passwordHash = parseAndValidateField(jsonLoginMethodObj, "passwordHash", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + String hashingAlgorithm = parseAndValidateField(jsonLoginMethodObj, "hashingAlgorithm", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + + email = email != null ? Utils.normaliseEmail(email) : null; + hashingAlgorithm = hashingAlgorithm != null ? hashingAlgorithm.trim().toUpperCase() : null; + + EmailPasswordLoginMethod emailPasswordLoginMethod = new EmailPasswordLoginMethod(email, passwordHash, hashingAlgorithm); + loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, emailPasswordLoginMethod, null, null)); + } else if ("thirdparty".equals(recipeId)) { + String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + String thirdPartyId = parseAndValidateField(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + String thirdPartyUserId = parseAndValidateField(jsonLoginMethodObj, "thirdPartyUserId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + + email = email != null ? Utils.normaliseEmail(email) : null; + + ThirdPartyLoginMethod thirdPartyLoginMethod = new ThirdPartyLoginMethod(email, thirdPartyId, thirdPartyUserId); + loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, thirdPartyLoginMethod, null)); + } else if ("passwordless".equals(recipeId)) { + String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); + String phoneNumber = parseAndValidateField(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); + + email = email != null ? Utils.normaliseEmail(email) : null; + phoneNumber = Utils.normalizeIfPhoneNumber(phoneNumber); + + PasswordlessLoginMethod passwordlessLoginMethod = new PasswordlessLoginMethod(email, phoneNumber); + loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, null, passwordlessLoginMethod)); + } else if (recipeId != null) { + errors.add("Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!"); + } + } + return loginMethods; + } +} diff --git a/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java b/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java new file mode 100644 index 000000000..3fbcd8fbd --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport.exceptions; + +import java.util.List; + +public class InvalidBulkImportDataException extends Exception { + private static final long serialVersionUID = 1L; + public List errors; + + public InvalidBulkImportDataException(List errors) { + super("Data has missing or invalid fields. Please check the errors field for more details."); + this.errors = errors; + } + + public void addError(String error) { + this.errors.add(error); + } +} diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java b/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java index 0e5d3df46..524eb4dc8 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java @@ -18,17 +18,24 @@ import io.supertokens.Main; import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.config.CoreConfig; +import io.supertokens.emailpassword.PasswordHashingUtils; +import io.supertokens.emailpassword.exceptions.UnsupportedPasswordHashingFormatException; import io.supertokens.featureflag.EE_FEATURES; import io.supertokens.featureflag.FeatureFlag; import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.BadPermissionException; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; -import io.supertokens.pluginInterface.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.EmailPasswordLoginMethod; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.utils.Utils; import io.supertokens.webserver.InputParser; import io.supertokens.webserver.WebserverAPI; import jakarta.servlet.ServletException; @@ -68,20 +75,29 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); } - AppIdentifier appIdentifier = getTenantIdentifierFromRequest(req).toAppIdentifier(); + AppIdentifierWithStorage appIdentifierWithStorage; + try { + appIdentifierWithStorage = getAppIdentifierWithStorageFromRequestAndEnforcePublicTenant(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } JsonArray errorsJson = new JsonArray(); ArrayList usersToAdd = new ArrayList<>(); for (int i = 0; i < users.size(); i++) { try { - BulkImportUser user = new BulkImportUser(users.get(i).getAsJsonObject(), null); - usersToAdd.add(user); - + BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(users.get(i).getAsJsonObject(), Utils.getUUID()); + for (BulkImportUser.LoginMethod loginMethod : user.loginMethods) { - validateTenantId(appIdentifier, loginMethod.tenantId, loginMethod.recipeId); + validateTenantId(appIdentifierWithStorage, loginMethod.tenantId, loginMethod.recipeId); + + if (loginMethod.emailPasswordLoginMethod != null) { + validatePasswordHashingAlgorithm(appIdentifierWithStorage, loginMethod.emailPasswordLoginMethod); + } } - } catch (InvalidBulkImportDataException e) { + usersToAdd.add(user); + } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { JsonObject errorObj = new JsonObject(); JsonArray errors = e.errors.stream() @@ -103,7 +119,6 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S } try { - AppIdentifierWithStorage appIdentifierWithStorage = getAppIdentifierWithStorage(req); BulkImport.addUsers(appIdentifierWithStorage, usersToAdd); } catch (TenantOrAppNotFoundException | StorageQueryException e) { throw new ServletException(e); @@ -141,4 +156,15 @@ private void validateTenantId(AppIdentifier appIdentifier, String tenantId, Stri new ArrayList<>(Arrays.asList("Invalid tenantId: " + tenantId + " for " + recipeId + " recipe."))); } } + + private void validatePasswordHashingAlgorithm(AppIdentifier appIdentifier, EmailPasswordLoginMethod emailPasswordLoginMethod) throws InvalidBulkImportDataException, ServletException { + try { + CoreConfig.PASSWORD_HASHING_ALG passwordHashingAlgorithm = CoreConfig.PASSWORD_HASHING_ALG.valueOf(emailPasswordLoginMethod.hashingAlgorithm); + PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, emailPasswordLoginMethod.passwordHash, passwordHashingAlgorithm); + } catch (UnsupportedPasswordHashingFormatException | TenantOrAppNotFoundException e) { + throw new InvalidBulkImportDataException(new ArrayList<>(Arrays.asList(e.getMessage()))); + } catch (IllegalArgumentException e) { + throw new InvalidBulkImportDataException(new ArrayList<>(Arrays.asList("Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!"))); + } + } } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java b/src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java index f349ac55b..f89fe39c3 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java @@ -17,7 +17,6 @@ package io.supertokens.webserver.api.bulkimport; import java.io.IOException; -import java.util.Arrays; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -26,7 +25,10 @@ import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserPaginationContainer; import io.supertokens.bulkimport.BulkImportUserPaginationToken; +import io.supertokens.multitenancy.exception.BadPermissionException; import io.supertokens.output.Logging; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; +import io.supertokens.pluginInterface.bulkimport.BulkImportUserInfo; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; @@ -49,7 +51,7 @@ public String getPath() { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - String status = InputParser.getQueryParamOrThrowError(req, "status", true); + String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); String paginationToken = InputParser.getQueryParamOrThrowError(req, "paginationToken", true); Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); @@ -64,29 +66,31 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se limit = BulkImport.GET_USERS_DEFAULT_LIMIT; } - if (status != null - && !Arrays.asList("NEW", "PROCESSING", "FAILED").contains(status)) { - throw new ServletException(new BadRequestException( - "Invalid value for status. Pass one of NEW, PROCESSING or, FAILED!")); + BulkImportUserStatus status = null; + if (statusString != null) { + try { + status = BulkImportUserStatus.valueOf(statusString); + } catch (IllegalArgumentException e) { + throw new ServletException(new BadRequestException("Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!")); + } } AppIdentifierWithStorage appIdentifierWithStorage = null; try { - appIdentifierWithStorage = this.getAppIdentifierWithStorage(req); - } catch (TenantOrAppNotFoundException e) { + appIdentifierWithStorage = getAppIdentifierWithStorageFromRequestAndEnforcePublicTenant(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { throw new ServletException(e); } try { - BulkImportUserPaginationContainer users = BulkImport.getUsers(appIdentifierWithStorage, limit, status, - paginationToken); + BulkImportUserPaginationContainer users = BulkImport.getUsers(appIdentifierWithStorage, limit, status, paginationToken); JsonObject result = new JsonObject(); result.addProperty("status", "OK"); JsonArray usersJson = new JsonArray(); - for (JsonObject user : users.users) { - usersJson.add(user); + for (BulkImportUserInfo user : users.users) { + usersJson.add(user.toJsonObject()); } result.add("users", usersJson); @@ -97,7 +101,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se } catch (BulkImportUserPaginationToken.InvalidTokenException e) { Logging.debug(main, null, Utils.exceptionStacktraceToString(e)); throw new ServletException(new BadRequestException("invalid pagination token")); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { + } catch (StorageQueryException e) { throw new ServletException(e); } } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 06fcd7096..f96d5b485 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import java.util.HashMap; import java.util.UUID; import org.junit.AfterClass; @@ -382,9 +383,55 @@ public void shouldReturn200Response() throws Exception { JsonObject request = generateUsersJson(10000); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", "http://localhost:3567/bulk-import/add-users", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldNormaliseFields() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + JsonObject request = generateUsersJson(1); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/add-users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); + JsonObject getResponse = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + new HashMap<>(), 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", getResponse.get("status").getAsString()); + JsonArray bulkImportUsers = getResponse.get("users").getAsJsonArray(); + assertEquals(1, bulkImportUsers.size()); + + JsonParser parser = new JsonParser(); + JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); + JsonArray loginMethods = parser.parse(bulkImportUserJson.get("rawData").getAsString()).getAsJsonObject().getAsJsonArray("loginMethods"); + + for (int i = 0; i < loginMethods.size(); i++) { + JsonObject loginMethod = loginMethods.get(i).getAsJsonObject(); + if (loginMethod.has("email")) { + assertEquals("johndoe+1@gmail.com", loginMethod.get("hashingAlgorithm").getAsString()); + } + if (loginMethod.has("phoneNumber")) { + assertEquals("919999999999", loginMethod.get("phoneNumber").getAsString()); + } + if (loginMethod.has("hashingAlgorithm")) { + assertEquals("ARGON2", loginMethod.get("hashingAlgorithm").getAsString()); + } + } + process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } @@ -400,9 +447,9 @@ public static JsonObject generateUsersJson(int numberOfUsers) { user.addProperty("externalUserId", UUID.randomUUID().toString()); user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); user.add("roles", parser.parse("[\"role1\", \"role2\"]")); - user.add("totp", parser.parse("[{\"secretKey\":\"secretKey\",\"period\":0,\"skew\":0,\"deviceName\":\"deviceName\"}]")); + user.add("totp", parser.parse("[{\"secretKey\":\"secretKey\",\"period\": 30,\"skew\":1,\"deviceName\":\"deviceName\"}]")); - String email = "johndoe+" + i + "@gmail.com"; + String email = " johndoe+" + i + "@gmail.com "; JsonArray loginMethodsArray = new JsonArray(); loginMethodsArray.add(createEmailLoginMethod(email)); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java index 7dbba3b04..64539bee8 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -75,7 +75,7 @@ public void shouldReturn400Error() throws Exception { } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); assertEquals( - "Http error. Status Code: 400. Message: Invalid value for status. Pass one of NEW, PROCESSING or, FAILED!", + "Http error. Status Code: 400. Message: Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!", e.getMessage()); } @@ -150,7 +150,7 @@ public void shouldReturn200Response() throws Exception { assertEquals(1, bulkImportUsers.size()); JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); bulkImportUserJson.get("status").getAsString().equals("NEW"); - bulkImportUserJson.get("raw_data").getAsString().equals(rawData); + bulkImportUserJson.get("rawData").getAsString().equals(rawData); process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); From d5f0c677de93bc0cff42154fb8c40f0e89b87b12 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 20 Feb 2024 17:50:49 +0530 Subject: [PATCH 04/15] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 4 +- .../BulkImportUserPaginationToken.java | 8 +- .../bulkimport/BulkImportUserUtils.java | 215 +++++++++++------- .../supertokens/utils/JsonValidatorUtils.java | 117 ++++++++++ .../api/bulkimport/AddBulkImportUsers.java | 63 +---- .../apis/AddBulkImportUsersTest.java | 4 +- 6 files changed, 265 insertions(+), 146 deletions(-) create mode 100644 src/main/java/io/supertokens/utils/JsonValidatorUtils.java diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 0f95b9cef..234c13c3d 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -57,11 +57,11 @@ public static BulkImportUserPaginationContainer getUsers(AppIdentifierWithStorag if (paginationToken == null) { users = appIdentifierWithStorage.getBulkImportStorage() - .getBulkImportUsers(appIdentifierWithStorage, limit + 1, status, null); + .getBulkImportUsers(appIdentifierWithStorage, limit + 1, status, null, null); } else { BulkImportUserPaginationToken tokenInfo = BulkImportUserPaginationToken.extractTokenInfo(paginationToken); users = appIdentifierWithStorage.getBulkImportStorage() - .getBulkImportUsers(appIdentifierWithStorage, limit + 1, status, tokenInfo.bulkImportUserId); + .getBulkImportUsers(appIdentifierWithStorage, limit + 1, status, tokenInfo.bulkImportUserId, tokenInfo.createdAt); } String nextPaginationToken = null; diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java index 0c24211a9..8a492c2ca 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java @@ -22,9 +22,9 @@ public class BulkImportUserPaginationToken { public final String bulkImportUserId; public final long createdAt; - public BulkImportUserPaginationToken(String bulkImportUserId, long timeJoined) { + public BulkImportUserPaginationToken(String bulkImportUserId, long createdAt) { this.bulkImportUserId = bulkImportUserId; - this.createdAt = timeJoined; + this.createdAt = createdAt; } public static BulkImportUserPaginationToken extractTokenInfo(String token) throws InvalidTokenException { @@ -35,8 +35,8 @@ public static BulkImportUserPaginationToken extractTokenInfo(String token) throw throw new InvalidTokenException(); } String bulkImportUserId = splitDecodedToken[0]; - long timeJoined = Long.parseLong(splitDecodedToken[1]); - return new BulkImportUserPaginationToken(bulkImportUserId, timeJoined); + long createdAt = Long.parseLong(splitDecodedToken[1]); + return new BulkImportUserPaginationToken(bulkImportUserId, createdAt); } catch (Exception e) { throw new InvalidTokenException(); } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 43915743a..dc98dcef1 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -17,40 +17,56 @@ package io.supertokens.bulkimport; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import io.supertokens.Main; import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.config.CoreConfig; +import io.supertokens.emailpassword.PasswordHashingUtils; +import io.supertokens.emailpassword.exceptions.UnsupportedPasswordHashingFormatException; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlag; +import io.supertokens.multitenancy.Multitenancy; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.EmailPasswordLoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.ThirdPartyLoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.PasswordlessLoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; -import io.supertokens.pluginInterface.utils.JsonValidatorUtils.ValueType; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.utils.Utils; +import io.supertokens.utils.JsonValidatorUtils.ValueType; -import static io.supertokens.pluginInterface.utils.JsonValidatorUtils.parseAndValidateField; -import static io.supertokens.pluginInterface.utils.JsonValidatorUtils.validateJsonFieldType; +import static io.supertokens.utils.JsonValidatorUtils.parseAndValidateField; +import static io.supertokens.utils.JsonValidatorUtils.validateJsonFieldType; public class BulkImportUserUtils { - public static BulkImportUser createBulkImportUserFromJSON(JsonObject userData, String id) throws InvalidBulkImportDataException { + public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, JsonObject userData, String id) + throws InvalidBulkImportDataException, StorageQueryException, TenantOrAppNotFoundException { List errors = new ArrayList<>(); - String externalUserId = parseAndValidateField(userData, "externalUserId", ValueType.STRING, false, String.class, errors, "."); - JsonObject userMetadata = parseAndValidateField(userData, "userMetadata", ValueType.OBJECT, false, JsonObject.class, errors, "."); - List userRoles = getParsedUserRoles(userData, errors); + String externalUserId = parseAndValidateField(userData, "externalUserId", ValueType.STRING, false, String.class, + errors, "."); + JsonObject userMetadata = parseAndValidateField(userData, "userMetadata", ValueType.OBJECT, false, + JsonObject.class, errors, "."); + List userRoles = getParsedUserRoles(userData, errors); List totpDevices = getParsedTotpDevices(userData, errors); - List loginMethods = getParsedLoginMethods(userData, errors); + List loginMethods = getParsedLoginMethods(main, appIdentifier, userData, errors); if (!errors.isEmpty()) { throw new InvalidBulkImportDataException(errors); } return new BulkImportUser(id, externalUserId, userMetadata, userRoles, totpDevices, loginMethods); - } + } private static List getParsedUserRoles(JsonObject userData, List errors) { JsonArray jsonUserRoles = parseAndValidateField(userData, "roles", ValueType.ARRAY_OF_STRING, @@ -88,7 +104,7 @@ private static List getParsedTotpDevices(JsonObject userData, List= 0 for a totp device."); } - if(deviceName != null) { + if (deviceName != null) { deviceName = deviceName.trim(); } totpDevices.add(new TotpDevice(secretKey, period.intValue(), skew.intValue(), deviceName)); @@ -96,72 +112,115 @@ private static List getParsedTotpDevices(JsonObject userData, List getParsedLoginMethods(JsonObject userData, List errors) { - JsonArray jsonLoginMethods = parseAndValidateField(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, true, JsonArray.class, errors, "."); - - if (jsonLoginMethods == null) { - return new ArrayList<>(); - } - - if (jsonLoginMethods.size() == 0) { - errors.add("At least one loginMethod is required."); - return new ArrayList<>(); - } - - Boolean hasPrimaryLoginMethod = false; - - List loginMethods = new ArrayList<>(); - for (JsonElement jsonLoginMethod : jsonLoginMethods) { - JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); - - if (validateJsonFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN)) { - if (jsonLoginMethodObj.get("isPrimary").getAsBoolean()) { - if (hasPrimaryLoginMethod) { - errors.add("No two loginMethods can have isPrimary as true."); - } - hasPrimaryLoginMethod = true; - } - } - - String recipeId = parseAndValidateField(jsonLoginMethodObj, "recipeId", ValueType.STRING, true, String.class, errors, " for a loginMethod."); - String tenantId = parseAndValidateField(jsonLoginMethodObj, "tenantId", ValueType.STRING, false, String.class, errors, " for a loginMethod."); - Boolean isVerified = parseAndValidateField(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); - Boolean isPrimary = parseAndValidateField(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); - Number timeJoined = parseAndValidateField(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.NUMBER, false, Number.class, errors, " for a loginMethod"); - Long timeJoinedInMSSinceEpoch = timeJoined != null ? timeJoined.longValue() : 0; - - if ("emailpassword".equals(recipeId)) { - String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); - String passwordHash = parseAndValidateField(jsonLoginMethodObj, "passwordHash", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); - String hashingAlgorithm = parseAndValidateField(jsonLoginMethodObj, "hashingAlgorithm", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); - - email = email != null ? Utils.normaliseEmail(email) : null; - hashingAlgorithm = hashingAlgorithm != null ? hashingAlgorithm.trim().toUpperCase() : null; - - EmailPasswordLoginMethod emailPasswordLoginMethod = new EmailPasswordLoginMethod(email, passwordHash, hashingAlgorithm); - loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, emailPasswordLoginMethod, null, null)); - } else if ("thirdparty".equals(recipeId)) { - String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); - String thirdPartyId = parseAndValidateField(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); - String thirdPartyUserId = parseAndValidateField(jsonLoginMethodObj, "thirdPartyUserId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); - - email = email != null ? Utils.normaliseEmail(email) : null; - - ThirdPartyLoginMethod thirdPartyLoginMethod = new ThirdPartyLoginMethod(email, thirdPartyId, thirdPartyUserId); - loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, thirdPartyLoginMethod, null)); - } else if ("passwordless".equals(recipeId)) { - String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); - String phoneNumber = parseAndValidateField(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); - - email = email != null ? Utils.normaliseEmail(email) : null; - phoneNumber = Utils.normalizeIfPhoneNumber(phoneNumber); - - PasswordlessLoginMethod passwordlessLoginMethod = new PasswordlessLoginMethod(email, phoneNumber); - loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, null, passwordlessLoginMethod)); - } else if (recipeId != null) { - errors.add("Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!"); - } - } - return loginMethods; - } + private static List getParsedLoginMethods(Main main, AppIdentifier appIdentifier, JsonObject userData, List errors) + throws StorageQueryException, TenantOrAppNotFoundException { + JsonArray jsonLoginMethods = parseAndValidateField(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, true, JsonArray.class, errors, "."); + + if (jsonLoginMethods == null) { + return new ArrayList<>(); + } + + if (jsonLoginMethods.size() == 0) { + errors.add("At least one loginMethod is required."); + return new ArrayList<>(); + } + + boolean hasPrimaryLoginMethod = false; + + List loginMethods = new ArrayList<>(); + for (JsonElement jsonLoginMethod : jsonLoginMethods) { + JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); + + if (validateJsonFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN)) { + if (jsonLoginMethodObj.get("isPrimary").getAsBoolean()) { + if (hasPrimaryLoginMethod) { + errors.add("No two loginMethods can have isPrimary as true."); + } + hasPrimaryLoginMethod = true; + } + } + + String recipeId = parseAndValidateField(jsonLoginMethodObj, "recipeId", ValueType.STRING, true, String.class, errors, " for a loginMethod."); + String tenantId = parseAndValidateField(jsonLoginMethodObj, "tenantId", ValueType.STRING, false, String.class, errors, " for a loginMethod."); + Boolean isVerified = parseAndValidateField(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); + Boolean isPrimary = parseAndValidateField(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); + Number timeJoined = parseAndValidateField(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.NUMBER, false, Number.class, errors, " for a loginMethod"); + long timeJoinedInMSSinceEpoch = timeJoined != null ? timeJoined.longValue() : 0; + + validateTenantId(main, appIdentifier, tenantId, recipeId, errors); + + if ("emailpassword".equals(recipeId)) { + String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + String passwordHash = parseAndValidateField(jsonLoginMethodObj, "passwordHash", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + String hashingAlgorithm = parseAndValidateField(jsonLoginMethodObj, "hashingAlgorithm", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + + email = email != null ? Utils.normaliseEmail(email) : null; + hashingAlgorithm = hashingAlgorithm != null ? hashingAlgorithm.trim().toUpperCase() : null; + + EmailPasswordLoginMethod emailPasswordLoginMethod = new EmailPasswordLoginMethod(email, passwordHash, hashingAlgorithm); + + validatePasswordHashingAlgorithm(main, appIdentifier, emailPasswordLoginMethod, errors); + + loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, emailPasswordLoginMethod, null, null)); + } else if ("thirdparty".equals(recipeId)) { + String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + String thirdPartyId = parseAndValidateField(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + String thirdPartyUserId = parseAndValidateField(jsonLoginMethodObj, "thirdPartyUserId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + + email = email != null ? Utils.normaliseEmail(email) : null; + + ThirdPartyLoginMethod thirdPartyLoginMethod = new ThirdPartyLoginMethod(email, thirdPartyId, thirdPartyUserId); + loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, thirdPartyLoginMethod, null)); + } else if ("passwordless".equals(recipeId)) { + String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); + String phoneNumber = parseAndValidateField(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); + + email = email != null ? Utils.normaliseEmail(email) : null; + phoneNumber = Utils.normalizeIfPhoneNumber(phoneNumber); + + PasswordlessLoginMethod passwordlessLoginMethod = new PasswordlessLoginMethod(email, phoneNumber); + loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, null, passwordlessLoginMethod)); + } else if (recipeId != null) { + errors.add("Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!"); + } + } + return loginMethods; + } + + private static void validateTenantId(Main main, AppIdentifier appIdentifier, String tenantId, String recipeId, List errors) + throws StorageQueryException, TenantOrAppNotFoundException { + if (tenantId == null || tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID)) { + return; + } + + if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .noneMatch(t -> t == EE_FEATURES.MULTI_TENANCY)) { + errors.add("Multitenancy must be enabled before importing users to a different tenant."); + return; + } + + TenantConfig[] allTenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + ArrayList validTenantIds = new ArrayList<>(); + Arrays.stream(allTenantConfigs) + .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); + + if (!validTenantIds.contains(tenantId)) { + errors.add("Invalid tenantId: " + tenantId + " for " + recipeId + " recipe."); + } + } + + private static void validatePasswordHashingAlgorithm(Main main, AppIdentifier appIdentifier, EmailPasswordLoginMethod emailPasswordLoginMethod, List errors) throws TenantOrAppNotFoundException { + if(emailPasswordLoginMethod.hashingAlgorithm == null || emailPasswordLoginMethod.passwordHash == null) { + return; + } + + try { + CoreConfig.PASSWORD_HASHING_ALG passwordHashingAlgorithm = CoreConfig.PASSWORD_HASHING_ALG.valueOf(emailPasswordLoginMethod.hashingAlgorithm); + PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, emailPasswordLoginMethod.passwordHash, passwordHashingAlgorithm); + } catch (UnsupportedPasswordHashingFormatException e) { + errors.add(e.getMessage()); + } catch (IllegalArgumentException e) { + errors.add("Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!"); + } + } } diff --git a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java new file mode 100644 index 000000000..e8ee6d569 --- /dev/null +++ b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + + package io.supertokens.utils; + + import java.util.ArrayList; + import java.util.List; + + import com.google.gson.JsonArray; + import com.google.gson.JsonElement; + import com.google.gson.JsonObject; + + public class JsonValidatorUtils { + @SuppressWarnings("unchecked") + public static T parseAndValidateField(JsonObject jsonObject, String key, ValueType expectedType, + boolean isRequired, Class targetType, List errors, String errorSuffix) { + if (jsonObject.has(key)) { + if (validateJsonFieldType(jsonObject, key, expectedType)) { + T value; + switch (expectedType) { + case STRING: + value = (T) jsonObject.get(key).getAsString(); + break; + case NUMBER: + value = (T) jsonObject.get(key).getAsNumber(); + break; + case BOOLEAN: + Boolean boolValue = jsonObject.get(key).getAsBoolean(); + value = (T) boolValue; + break; + case OBJECT: + value = (T) jsonObject.get(key).getAsJsonObject(); + break; + case ARRAY_OF_OBJECT, ARRAY_OF_STRING: + value = (T) jsonObject.get(key).getAsJsonArray(); + break; + default: + value = null; + break; + } + if (value != null) { + return targetType.cast(value); + } else { + errors.add(key + " should be of type " + getTypeForErrorMessage(expectedType) + errorSuffix); + } + } else { + errors.add(key + " should be of type " + getTypeForErrorMessage(expectedType) + errorSuffix); + } + } else if (isRequired) { + errors.add(key + " is required" + errorSuffix); + } + return null; + } + + public enum ValueType { + STRING, + NUMBER, + BOOLEAN, + OBJECT, + ARRAY_OF_STRING, + ARRAY_OF_OBJECT + } + + private static String getTypeForErrorMessage(ValueType type) { + return switch (type) { + case STRING -> "string"; + case NUMBER -> "number"; + case BOOLEAN -> "boolean"; + case OBJECT -> "object"; + case ARRAY_OF_STRING -> "array of string"; + case ARRAY_OF_OBJECT -> "array of object"; + }; + } + + public static boolean validateJsonFieldType(JsonObject jsonObject, String key, ValueType expectedType) { + if (jsonObject.has(key)) { + return switch (expectedType) { + case STRING -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isString() + && !jsonObject.get(key).getAsString().isEmpty(); + case NUMBER -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isNumber(); + case BOOLEAN -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isBoolean(); + case OBJECT -> jsonObject.get(key).isJsonObject(); + case ARRAY_OF_OBJECT, ARRAY_OF_STRING -> jsonObject.get(key).isJsonArray() + && validateArrayElements(jsonObject.getAsJsonArray(key), expectedType); + default -> false; + }; + } + return false; + } + + public static boolean validateArrayElements(JsonArray array, ValueType expectedType) { + List elements = new ArrayList<>(); + array.forEach(elements::add); + + return switch (expectedType) { + case ARRAY_OF_OBJECT -> elements.stream().allMatch(JsonElement::isJsonObject); + case ARRAY_OF_STRING -> + elements.stream().allMatch(el -> el.isJsonPrimitive() && el.getAsJsonPrimitive().isString() + && !el.getAsString().isEmpty()); + default -> false; + }; + } + } + \ No newline at end of file diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java b/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java index 524eb4dc8..8239cd0ac 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java @@ -19,21 +19,10 @@ import io.supertokens.Main; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserUtils; -import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; -import io.supertokens.config.CoreConfig; -import io.supertokens.emailpassword.PasswordHashingUtils; -import io.supertokens.emailpassword.exceptions.UnsupportedPasswordHashingFormatException; -import io.supertokens.featureflag.EE_FEATURES; -import io.supertokens.featureflag.FeatureFlag; -import io.supertokens.multitenancy.Multitenancy; import io.supertokens.multitenancy.exception.BadPermissionException; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.EmailPasswordLoginMethod; import io.supertokens.pluginInterface.exceptions.StorageQueryException; -import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; -import io.supertokens.pluginInterface.multitenancy.TenantConfig; -import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.utils.Utils; import io.supertokens.webserver.InputParser; @@ -44,7 +33,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -87,15 +75,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S for (int i = 0; i < users.size(); i++) { try { - BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(users.get(i).getAsJsonObject(), Utils.getUUID()); - - for (BulkImportUser.LoginMethod loginMethod : user.loginMethods) { - validateTenantId(appIdentifierWithStorage, loginMethod.tenantId, loginMethod.recipeId); - - if (loginMethod.emailPasswordLoginMethod != null) { - validatePasswordHashingAlgorithm(appIdentifierWithStorage, loginMethod.emailPasswordLoginMethod); - } - } + BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifierWithStorage, users.get(i).getAsJsonObject(), Utils.getUUID()); usersToAdd.add(user); } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { JsonObject errorObj = new JsonObject(); @@ -107,6 +87,8 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S errorObj.addProperty("index", i); errorObj.add("errors", errors); errorsJson.add(errorObj); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new ServletException(e); } } @@ -128,43 +110,4 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S result.addProperty("status", "OK"); super.sendJsonResponse(200, result, resp); } - - private void validateTenantId(AppIdentifier appIdentifier, String tenantId, String recipeId) - throws InvalidBulkImportDataException, ServletException { - if (tenantId == null || tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID)) { - return; - } - - try { - if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) - .noneMatch(t -> t == EE_FEATURES.MULTI_TENANCY)) { - throw new InvalidBulkImportDataException(new ArrayList<>( - Arrays.asList("Multitenancy must be enabled before importing users to a different tenant."))); - } - } catch (TenantOrAppNotFoundException | StorageQueryException e) { - throw new ServletException(e); - } - - TenantConfig[] allTenantConfigs = Multitenancy - .getAllTenantsForApp(appIdentifier, main); - ArrayList validTenantIds = new ArrayList<>(); - Arrays.stream(allTenantConfigs) - .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); - - if (!validTenantIds.contains(tenantId)) { - throw new InvalidBulkImportDataException( - new ArrayList<>(Arrays.asList("Invalid tenantId: " + tenantId + " for " + recipeId + " recipe."))); - } - } - - private void validatePasswordHashingAlgorithm(AppIdentifier appIdentifier, EmailPasswordLoginMethod emailPasswordLoginMethod) throws InvalidBulkImportDataException, ServletException { - try { - CoreConfig.PASSWORD_HASHING_ALG passwordHashingAlgorithm = CoreConfig.PASSWORD_HASHING_ALG.valueOf(emailPasswordLoginMethod.hashingAlgorithm); - PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, emailPasswordLoginMethod.passwordHash, passwordHashingAlgorithm); - } catch (UnsupportedPasswordHashingFormatException | TenantOrAppNotFoundException e) { - throw new InvalidBulkImportDataException(new ArrayList<>(Arrays.asList(e.getMessage()))); - } catch (IllegalArgumentException e) { - throw new InvalidBulkImportDataException(new ArrayList<>(Arrays.asList("Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!"))); - } - } } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index f96d5b485..f20bceb42 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -216,7 +216,7 @@ public void shouldThrow400Error() throws Exception { // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt try { JsonObject request = new JsonParser().parse( - "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"somehash\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", "http://localhost:3567/bulk-import/add-users", @@ -327,7 +327,7 @@ public void shouldThrow400Error() throws Exception { // CASE 1: email, passwordHash and hashingAlgorithm are not present try { JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"somehash\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}").getAsJsonObject(); + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}").getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", "http://localhost:3567/bulk-import/add-users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); From d7ba8d1da71ec1f8dcb54490fc8331af76524b43 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 22 Feb 2024 12:04:16 +0530 Subject: [PATCH 05/15] fix: PR changes --- .../bulkimport/BulkImportUserUtils.java | 230 ++++++++++++------ .../supertokens/utils/JsonValidatorUtils.java | 2 +- 2 files changed, 163 insertions(+), 69 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index dc98dcef1..d9b2c1f7c 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -46,7 +46,7 @@ import io.supertokens.utils.Utils; import io.supertokens.utils.JsonValidatorUtils.ValueType; -import static io.supertokens.utils.JsonValidatorUtils.parseAndValidateField; +import static io.supertokens.utils.JsonValidatorUtils.parseAndValidateFieldType; import static io.supertokens.utils.JsonValidatorUtils.validateJsonFieldType; public class BulkImportUserUtils { @@ -54,9 +54,9 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi throws InvalidBulkImportDataException, StorageQueryException, TenantOrAppNotFoundException { List errors = new ArrayList<>(); - String externalUserId = parseAndValidateField(userData, "externalUserId", ValueType.STRING, false, String.class, + String externalUserId = parseAndValidateFieldType(userData, "externalUserId", ValueType.STRING, false, String.class, errors, "."); - JsonObject userMetadata = parseAndValidateField(userData, "userMetadata", ValueType.OBJECT, false, + JsonObject userMetadata = parseAndValidateFieldType(userData, "userMetadata", ValueType.OBJECT, false, JsonObject.class, errors, "."); List userRoles = getParsedUserRoles(userData, errors); List totpDevices = getParsedTotpDevices(userData, errors); @@ -69,7 +69,7 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi } private static List getParsedUserRoles(JsonObject userData, List errors) { - JsonArray jsonUserRoles = parseAndValidateField(userData, "roles", ValueType.ARRAY_OF_STRING, + JsonArray jsonUserRoles = parseAndValidateFieldType(userData, "roles", ValueType.ARRAY_OF_STRING, false, JsonArray.class, errors, "."); @@ -77,13 +77,14 @@ private static List getParsedUserRoles(JsonObject userData, List return null; } + // We already know that the jsonUserRoles is an array of non-empty strings, we will normalise each role now List userRoles = new ArrayList<>(); - jsonUserRoles.forEach(role -> userRoles.add(role.getAsString().trim())); + jsonUserRoles.forEach(role -> userRoles.add(validateAndNormaliseUserRole(role.getAsString()))); return userRoles; } private static List getParsedTotpDevices(JsonObject userData, List errors) { - JsonArray jsonTotpDevices = parseAndValidateField(userData, "totp", ValueType.ARRAY_OF_OBJECT, false, JsonArray.class, errors, "."); + JsonArray jsonTotpDevices = parseAndValidateFieldType(userData, "totp", ValueType.ARRAY_OF_OBJECT, false, JsonArray.class, errors, "."); if (jsonTotpDevices == null) { return null; } @@ -92,29 +93,26 @@ private static List getParsedTotpDevices(JsonObject userData, List 0 for a totp device."); - } - if (skew != null && skew.intValue() < 0) { - errors.add("skew should be >= 0 for a totp device."); - } - - if (deviceName != null) { - deviceName = deviceName.trim(); - } - totpDevices.add(new TotpDevice(secretKey, period.intValue(), skew.intValue(), deviceName)); + String secretKey = parseAndValidateFieldType(jsonTotpDevice, "secretKey", ValueType.STRING, true, String.class, errors, " for a totp device."); + Number period = parseAndValidateFieldType(jsonTotpDevice, "period", ValueType.NUMBER, true, Number.class, errors, " for a totp device."); + Number skew = parseAndValidateFieldType(jsonTotpDevice, "skew", ValueType.NUMBER, true, Number.class, errors, " for a totp device."); + String deviceName = parseAndValidateFieldType(jsonTotpDevice, "deviceName", ValueType.STRING, false, String.class, errors, " for a totp device."); + + totpDevices.add( + new TotpDevice( + validateAndNormaliseTotpSecretKey(secretKey), + validateAndNormaliseTotpPeriod(period, errors), + validateAndNormaliseTotpSkew(skew, errors), + validateAndNormaliseTotpDeviceName(deviceName) + ) + ); } return totpDevices; } private static List getParsedLoginMethods(Main main, AppIdentifier appIdentifier, JsonObject userData, List errors) throws StorageQueryException, TenantOrAppNotFoundException { - JsonArray jsonLoginMethods = parseAndValidateField(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, true, JsonArray.class, errors, "."); + JsonArray jsonLoginMethods = parseAndValidateFieldType(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, true, JsonArray.class, errors, "."); if (jsonLoginMethods == null) { return new ArrayList<>(); @@ -125,102 +123,198 @@ private static List getParsedLoginMethods(Main main, AppIdentifier return new ArrayList<>(); } - boolean hasPrimaryLoginMethod = false; + validateAndNormaliseIsPrimaryField(jsonLoginMethods, errors); List loginMethods = new ArrayList<>(); + for (JsonElement jsonLoginMethod : jsonLoginMethods) { JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); - if (validateJsonFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN)) { - if (jsonLoginMethodObj.get("isPrimary").getAsBoolean()) { - if (hasPrimaryLoginMethod) { - errors.add("No two loginMethods can have isPrimary as true."); - } - hasPrimaryLoginMethod = true; - } - } + String recipeId = parseAndValidateFieldType(jsonLoginMethodObj, "recipeId", ValueType.STRING, true, String.class, errors, " for a loginMethod."); + String tenantId = parseAndValidateFieldType(jsonLoginMethodObj, "tenantId", ValueType.STRING, false, String.class, errors, " for a loginMethod."); + Boolean isVerified = parseAndValidateFieldType(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); + Boolean isPrimary = parseAndValidateFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); + Number timeJoined = parseAndValidateFieldType(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.NUMBER, false, Number.class, errors, " for a loginMethod"); - String recipeId = parseAndValidateField(jsonLoginMethodObj, "recipeId", ValueType.STRING, true, String.class, errors, " for a loginMethod."); - String tenantId = parseAndValidateField(jsonLoginMethodObj, "tenantId", ValueType.STRING, false, String.class, errors, " for a loginMethod."); - Boolean isVerified = parseAndValidateField(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); - Boolean isPrimary = parseAndValidateField(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); - Number timeJoined = parseAndValidateField(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.NUMBER, false, Number.class, errors, " for a loginMethod"); - long timeJoinedInMSSinceEpoch = timeJoined != null ? timeJoined.longValue() : 0; - - validateTenantId(main, appIdentifier, tenantId, recipeId, errors); + recipeId = validateAndNormaliseRecipeId(recipeId, errors); + tenantId= validateAndNormaliseTenantId(main, appIdentifier, tenantId, recipeId, errors); + isVerified = validateAndNormaliseIsVerified(isVerified); + long timeJoinedInMSSinceEpoch = validateAndNormaliseTimeJoined(timeJoined); if ("emailpassword".equals(recipeId)) { - String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); - String passwordHash = parseAndValidateField(jsonLoginMethodObj, "passwordHash", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); - String hashingAlgorithm = parseAndValidateField(jsonLoginMethodObj, "hashingAlgorithm", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + String passwordHash = parseAndValidateFieldType(jsonLoginMethodObj, "passwordHash", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + String hashingAlgorithm = parseAndValidateFieldType(jsonLoginMethodObj, "hashingAlgorithm", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); - email = email != null ? Utils.normaliseEmail(email) : null; - hashingAlgorithm = hashingAlgorithm != null ? hashingAlgorithm.trim().toUpperCase() : null; + email = validateAndNormaliseEmail(email); + passwordHash = validateAndNormalisePasswordHash(passwordHash); + hashingAlgorithm = validateAndNormaliseHashingAlgorithm(main, appIdentifier, hashingAlgorithm, passwordHash, errors); EmailPasswordLoginMethod emailPasswordLoginMethod = new EmailPasswordLoginMethod(email, passwordHash, hashingAlgorithm); - - validatePasswordHashingAlgorithm(main, appIdentifier, emailPasswordLoginMethod, errors); - loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, emailPasswordLoginMethod, null, null)); } else if ("thirdparty".equals(recipeId)) { - String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); - String thirdPartyId = parseAndValidateField(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); - String thirdPartyUserId = parseAndValidateField(jsonLoginMethodObj, "thirdPartyUserId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + String thirdPartyId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + String thirdPartyUserId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyUserId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); - email = email != null ? Utils.normaliseEmail(email) : null; + email = validateAndNormaliseEmail(email); + thirdPartyId = validateAndNormaliseThirdPartyId(thirdPartyId); + thirdPartyUserId = validateAndNormaliseThirdPartyUserId(thirdPartyUserId); ThirdPartyLoginMethod thirdPartyLoginMethod = new ThirdPartyLoginMethod(email, thirdPartyId, thirdPartyUserId); loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, thirdPartyLoginMethod, null)); } else if ("passwordless".equals(recipeId)) { - String email = parseAndValidateField(jsonLoginMethodObj, "email", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); - String phoneNumber = parseAndValidateField(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); + String phoneNumber = parseAndValidateFieldType(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); - email = email != null ? Utils.normaliseEmail(email) : null; - phoneNumber = Utils.normalizeIfPhoneNumber(phoneNumber); + email = validateAndNormaliseEmail(email); + phoneNumber = validateAndNormalisePhoneNumber(phoneNumber); PasswordlessLoginMethod passwordlessLoginMethod = new PasswordlessLoginMethod(email, phoneNumber); loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, null, passwordlessLoginMethod)); - } else if (recipeId != null) { - errors.add("Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!"); } } return loginMethods; } - private static void validateTenantId(Main main, AppIdentifier appIdentifier, String tenantId, String recipeId, List errors) + private static String validateAndNormaliseUserRole(String role) { + // We just trim the role the CreateRoleAPI.java + return role.trim(); + } + + private static String validateAndNormaliseTotpSecretKey(String secretKey) { + // We don't perform any normalisation on the secretKey in ImportTotpDeviceAPI.java + return secretKey; + } + + private static Integer validateAndNormaliseTotpPeriod(Number period, List errors) { + // We don't perform any normalisation on the period in ImportTotpDeviceAPI.java other than checking if it is > 0 + if (period != null && period.intValue() < 1) { + errors.add("period should be > 0 for a totp device."); + return null; + } + return period != null ? period.intValue() : null; + } + + private static Integer validateAndNormaliseTotpSkew(Number skew, List errors) { + // We don't perform any normalisation on the period in ImportTotpDeviceAPI.java other than checking if it is >= 0 + if (skew != null && skew.intValue() < 0) { + errors.add("skew should be >= 0 for a totp device."); + return null; + } + return skew != null ? skew.intValue() : null; + } + + private static String validateAndNormaliseTotpDeviceName(String deviceName) { + // We normalise the deviceName as per the ImportTotpDeviceAPI.java + return deviceName != null ? deviceName.trim() : null; + } + + private static void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethods, List errors) { + // We are validating that only one loginMethod has isPrimary as true + boolean hasPrimaryLoginMethod = false; + for (JsonElement jsonLoginMethod : jsonLoginMethods) { + JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); + if (validateJsonFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN)) { + if (jsonLoginMethodObj.get("isPrimary").getAsBoolean()) { + if (hasPrimaryLoginMethod) { + errors.add("No two loginMethods can have isPrimary as true."); + } + hasPrimaryLoginMethod = true; + } + } + } + } + + private static String validateAndNormaliseRecipeId(String recipeId, List errors) { + if (recipeId == null) { + return null; + } + + // We don't perform any normalisation on the recipeId after reading it from request header. + // We will validate it as is. + if (!Arrays.asList("emailpassword", "thirdparty", "passwordless").contains(recipeId)) { + errors.add("Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!"); + } + return recipeId; + } + + private static String validateAndNormaliseTenantId(Main main, AppIdentifier appIdentifier, String tenantId, String recipeId, List errors) throws StorageQueryException, TenantOrAppNotFoundException { if (tenantId == null || tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID)) { - return; + return tenantId; } if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) .noneMatch(t -> t == EE_FEATURES.MULTI_TENANCY)) { errors.add("Multitenancy must be enabled before importing users to a different tenant."); - return; + return tenantId; } + // We make the tenantId lowercase while parsing from the request in WebserverAPI.java + String normalisedTenantId = tenantId.trim().toLowerCase(); TenantConfig[] allTenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); ArrayList validTenantIds = new ArrayList<>(); Arrays.stream(allTenantConfigs) .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); - if (!validTenantIds.contains(tenantId)) { + if (!validTenantIds.contains(normalisedTenantId)) { errors.add("Invalid tenantId: " + tenantId + " for " + recipeId + " recipe."); } + return normalisedTenantId; } - private static void validatePasswordHashingAlgorithm(Main main, AppIdentifier appIdentifier, EmailPasswordLoginMethod emailPasswordLoginMethod, List errors) throws TenantOrAppNotFoundException { - if(emailPasswordLoginMethod.hashingAlgorithm == null || emailPasswordLoginMethod.passwordHash == null) { - return; - } + private static Boolean validateAndNormaliseIsVerified(Boolean isPrimary) { + // No normalisation needs to be done for isVerified + return isPrimary; + } + + private static long validateAndNormaliseTimeJoined(Number timeJoined) { + // We default timeJoined to 0 if it is null + return timeJoined != null ? timeJoined.longValue() : 0; + } + + private static String validateAndNormaliseEmail(String email) { + // We normalise the email as per the SignUpAPI.java + return email != null ? Utils.normaliseEmail(email) : null; + } + private static String validateAndNormalisePasswordHash(String passwordHash) { + // We trim the passwordHash as per the ImportUserWithPasswordHashAPI.java + return passwordHash != null ? passwordHash.trim() : null; + } + + private static String validateAndNormaliseHashingAlgorithm(Main main, AppIdentifier appIdentifier, String hashingAlgorithm, String passwordHash, List errors) throws TenantOrAppNotFoundException { + if (hashingAlgorithm == null || passwordHash == null) { + return hashingAlgorithm; + } + try { - CoreConfig.PASSWORD_HASHING_ALG passwordHashingAlgorithm = CoreConfig.PASSWORD_HASHING_ALG.valueOf(emailPasswordLoginMethod.hashingAlgorithm); - PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, emailPasswordLoginMethod.passwordHash, passwordHashingAlgorithm); + // We trim the hashingAlgorithm and make it uppercase as per the ImportUserWithPasswordHashAPI.java + CoreConfig.PASSWORD_HASHING_ALG normalisedHashingAlgorithm = CoreConfig.PASSWORD_HASHING_ALG.valueOf(hashingAlgorithm.trim().toUpperCase()); + PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, passwordHash, normalisedHashingAlgorithm); + return normalisedHashingAlgorithm.toString(); } catch (UnsupportedPasswordHashingFormatException e) { errors.add(e.getMessage()); } catch (IllegalArgumentException e) { errors.add("Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!"); } + return hashingAlgorithm; + } + + private static String validateAndNormaliseThirdPartyId(String thirdPartyId) { + // We don't perform any normalisation on the thirdPartyId in SignInUpAPI.java + return thirdPartyId; + } + + private static String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId) { + // We don't perform any normalisation on the thirdPartyUserId in SignInUpAPI.java + return thirdPartyUserId; + } + + private static String validateAndNormalisePhoneNumber(String phoneNumber) { + // We normalise the phoneNumber as per the CreateCodeAPI.java + return phoneNumber != null ? Utils.normalizeIfPhoneNumber(phoneNumber) : null; } + } diff --git a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java index e8ee6d569..487315a9c 100644 --- a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java +++ b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java @@ -25,7 +25,7 @@ public class JsonValidatorUtils { @SuppressWarnings("unchecked") - public static T parseAndValidateField(JsonObject jsonObject, String key, ValueType expectedType, + public static T parseAndValidateFieldType(JsonObject jsonObject, String key, ValueType expectedType, boolean isRequired, Class targetType, List errors, String errorSuffix) { if (jsonObject.has(key)) { if (validateJsonFieldType(jsonObject, key, expectedType)) { From 4b34b89d437814c879ec6b43d6879fe5dde29dcc Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Fri, 23 Feb 2024 18:42:33 +0530 Subject: [PATCH 06/15] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 1 + .../bulkimport/BulkImportUserUtils.java | 74 +++-- .../supertokens/utils/JsonValidatorUtils.java | 11 +- .../io/supertokens/webserver/Webserver.java | 6 +- .../api/bulkimport/AddBulkImportUsers.java | 113 -------- ...ulkImportUsers.java => BulkImportAPI.java} | 71 ++++- .../test/bulkimport/BulkImportTest.java | 266 ++++++++++++++++++ .../test/bulkimport/BulkImportTestUtils.java | 63 +++++ .../apis/AddBulkImportUsersTest.java | 46 +-- .../apis/DeleteFailedBulkImportUsersTest.java | 164 +++++++++++ .../apis/GetBulkImportUsersTest.java | 2 +- 11 files changed, 640 insertions(+), 177 deletions(-) delete mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java rename src/main/java/io/supertokens/webserver/api/bulkimport/{GetBulkImportUsers.java => BulkImportAPI.java} (58%) create mode 100644 src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/DeleteFailedBulkImportUsersTest.java diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 234c13c3d..49e539bfc 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -32,6 +32,7 @@ public class BulkImport { + public static final int MAX_USERS_TO_ADD = 10000; public static final int GET_USERS_PAGINATION_LIMIT = 500; public static final int GET_USERS_DEFAULT_LIMIT = 100; diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index d9b2c1f7c..e5c107bd8 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -94,18 +94,18 @@ private static List getParsedTotpDevices(JsonObject userData, List getParsedLoginMethods(Main main, AppIdentifier String tenantId = parseAndValidateFieldType(jsonLoginMethodObj, "tenantId", ValueType.STRING, false, String.class, errors, " for a loginMethod."); Boolean isVerified = parseAndValidateFieldType(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); Boolean isPrimary = parseAndValidateFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); - Number timeJoined = parseAndValidateFieldType(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.NUMBER, false, Number.class, errors, " for a loginMethod"); + Integer timeJoined = parseAndValidateFieldType(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.INTEGER, false, Integer.class, errors, " for a loginMethod"); recipeId = validateAndNormaliseRecipeId(recipeId, errors); tenantId= validateAndNormaliseTenantId(main, appIdentifier, tenantId, recipeId, errors); + isPrimary = validateAndNormaliseIsPrimary(isPrimary); isVerified = validateAndNormaliseIsVerified(isVerified); + long timeJoinedInMSSinceEpoch = validateAndNormaliseTimeJoined(timeJoined); if ("emailpassword".equals(recipeId)) { @@ -147,8 +149,9 @@ private static List getParsedLoginMethods(Main main, AppIdentifier String hashingAlgorithm = parseAndValidateFieldType(jsonLoginMethodObj, "hashingAlgorithm", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); email = validateAndNormaliseEmail(email); - passwordHash = validateAndNormalisePasswordHash(passwordHash); - hashingAlgorithm = validateAndNormaliseHashingAlgorithm(main, appIdentifier, hashingAlgorithm, passwordHash, errors); + CoreConfig.PASSWORD_HASHING_ALG normalisedHashingAlgorithm = validateAndNormaliseHashingAlgorithm(hashingAlgorithm, errors); + hashingAlgorithm = normalisedHashingAlgorithm != null ? normalisedHashingAlgorithm.toString() : hashingAlgorithm; + passwordHash = validateAndNormalisePasswordHash(main, appIdentifier, normalisedHashingAlgorithm, passwordHash, errors); EmailPasswordLoginMethod emailPasswordLoginMethod = new EmailPasswordLoginMethod(email, passwordHash, hashingAlgorithm); loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, emailPasswordLoginMethod, null, null)); @@ -264,12 +267,17 @@ private static String validateAndNormaliseTenantId(Main main, AppIdentifier appI return normalisedTenantId; } - private static Boolean validateAndNormaliseIsVerified(Boolean isPrimary) { - // No normalisation needs to be done for isVerified - return isPrimary; + private static Boolean validateAndNormaliseIsPrimary(Boolean isPrimary) { + // We set the default value as false + return isPrimary == null ? false : isPrimary; + } + + private static Boolean validateAndNormaliseIsVerified(Boolean isVerified) { + // We set the default value as false + return isVerified == null ? false : isVerified; } - private static long validateAndNormaliseTimeJoined(Number timeJoined) { + private static long validateAndNormaliseTimeJoined(Integer timeJoined) { // We default timeJoined to 0 if it is null return timeJoined != null ? timeJoined.longValue() : 0; } @@ -279,27 +287,35 @@ private static String validateAndNormaliseEmail(String email) { return email != null ? Utils.normaliseEmail(email) : null; } - private static String validateAndNormalisePasswordHash(String passwordHash) { - // We trim the passwordHash as per the ImportUserWithPasswordHashAPI.java - return passwordHash != null ? passwordHash.trim() : null; + private static CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgorithm(String hashingAlgorithm, List errors) { + if (hashingAlgorithm == null) { + return null; + } + + try { + // We trim the hashingAlgorithm and make it uppercase as per the ImportUserWithPasswordHashAPI.java + return CoreConfig.PASSWORD_HASHING_ALG.valueOf(hashingAlgorithm.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + errors.add("Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!"); + return null; + } } - private static String validateAndNormaliseHashingAlgorithm(Main main, AppIdentifier appIdentifier, String hashingAlgorithm, String passwordHash, List errors) throws TenantOrAppNotFoundException { + private static String validateAndNormalisePasswordHash(Main main, AppIdentifier appIdentifier, CoreConfig.PASSWORD_HASHING_ALG hashingAlgorithm, String passwordHash, List errors) throws TenantOrAppNotFoundException { if (hashingAlgorithm == null || passwordHash == null) { - return hashingAlgorithm; + return passwordHash; } + // We trim the passwordHash and validate it as per ImportUserWithPasswordHashAPI.java + passwordHash = passwordHash.trim(); + try { - // We trim the hashingAlgorithm and make it uppercase as per the ImportUserWithPasswordHashAPI.java - CoreConfig.PASSWORD_HASHING_ALG normalisedHashingAlgorithm = CoreConfig.PASSWORD_HASHING_ALG.valueOf(hashingAlgorithm.trim().toUpperCase()); - PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, passwordHash, normalisedHashingAlgorithm); - return normalisedHashingAlgorithm.toString(); + PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, passwordHash, hashingAlgorithm); } catch (UnsupportedPasswordHashingFormatException e) { errors.add(e.getMessage()); - } catch (IllegalArgumentException e) { - errors.add("Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!"); } - return hashingAlgorithm; + + return passwordHash; } private static String validateAndNormaliseThirdPartyId(String thirdPartyId) { diff --git a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java index 487315a9c..beb65a87b 100644 --- a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java +++ b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java @@ -34,8 +34,9 @@ public static T parseAndValidateFieldType(JsonObject jsonObject, String key, case STRING: value = (T) jsonObject.get(key).getAsString(); break; - case NUMBER: - value = (T) jsonObject.get(key).getAsNumber(); + case INTEGER: + Integer intValue = jsonObject.get(key).getAsNumber().intValue(); + value = (T) intValue; break; case BOOLEAN: Boolean boolValue = jsonObject.get(key).getAsBoolean(); @@ -67,7 +68,7 @@ public static T parseAndValidateFieldType(JsonObject jsonObject, String key, public enum ValueType { STRING, - NUMBER, + INTEGER, BOOLEAN, OBJECT, ARRAY_OF_STRING, @@ -77,7 +78,7 @@ public enum ValueType { private static String getTypeForErrorMessage(ValueType type) { return switch (type) { case STRING -> "string"; - case NUMBER -> "number"; + case INTEGER -> "integer"; case BOOLEAN -> "boolean"; case OBJECT -> "object"; case ARRAY_OF_STRING -> "array of string"; @@ -90,7 +91,7 @@ public static boolean validateJsonFieldType(JsonObject jsonObject, String key, V return switch (expectedType) { case STRING -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isString() && !jsonObject.get(key).getAsString().isEmpty(); - case NUMBER -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isNumber(); + case INTEGER -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isNumber(); case BOOLEAN -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isBoolean(); case OBJECT -> jsonObject.get(key).isJsonObject(); case ARRAY_OF_OBJECT, ARRAY_OF_STRING -> jsonObject.get(key).isJsonArray() diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 6f9c30ef1..78059bf13 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -27,8 +27,7 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifierWithStorage; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; -import io.supertokens.webserver.api.bulkimport.AddBulkImportUsers; -import io.supertokens.webserver.api.bulkimport.GetBulkImportUsers; +import io.supertokens.webserver.api.bulkimport.BulkImportAPI; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -261,8 +260,7 @@ private void setupRoutes() { addAPI(new RequestStatsAPI(main)); - addAPI(new AddBulkImportUsers(main)); - addAPI(new GetBulkImportUsers(main)); + addAPI(new BulkImportAPI(main)); StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java b/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java deleted file mode 100644 index 8239cd0ac..000000000 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/AddBulkImportUsers.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. - * - * This software is licensed under the Apache License, Version 2.0 (the - * "License") as published by the Apache Software Foundation. - * - * You may not use this file except in compliance with the License. You may - * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package io.supertokens.webserver.api.bulkimport; - -import io.supertokens.Main; -import io.supertokens.bulkimport.BulkImport; -import io.supertokens.bulkimport.BulkImportUserUtils; -import io.supertokens.multitenancy.exception.BadPermissionException; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser; -import io.supertokens.pluginInterface.exceptions.StorageQueryException; -import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; -import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; -import io.supertokens.utils.Utils; -import io.supertokens.webserver.InputParser; -import io.supertokens.webserver.WebserverAPI; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; - -import java.io.IOException; -import java.util.ArrayList; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonPrimitive; - -public class AddBulkImportUsers extends WebserverAPI { - private static final int MAX_USERS_TO_ADD = 10000; - - public AddBulkImportUsers(Main main) { - super(main, ""); - } - - @Override - public String getPath() { - return "/bulk-import/add-users"; - } - - @Override - protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - JsonObject input = InputParser.parseJsonObjectOrThrowError(req); - JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); - - if (users.size() <= 0 || users.size() > MAX_USERS_TO_ADD) { - JsonObject errorResponseJson = new JsonObject(); - String errorMsg = users.size() <= 0 ? "You need to add at least one user." - : "You can only add 10000 users at a time."; - errorResponseJson.addProperty("error", errorMsg); - throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); - } - - AppIdentifierWithStorage appIdentifierWithStorage; - try { - appIdentifierWithStorage = getAppIdentifierWithStorageFromRequestAndEnforcePublicTenant(req); - } catch (TenantOrAppNotFoundException | BadPermissionException e) { - throw new ServletException(e); - } - - JsonArray errorsJson = new JsonArray(); - ArrayList usersToAdd = new ArrayList<>(); - - for (int i = 0; i < users.size(); i++) { - try { - BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifierWithStorage, users.get(i).getAsJsonObject(), Utils.getUUID()); - usersToAdd.add(user); - } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { - JsonObject errorObj = new JsonObject(); - - JsonArray errors = e.errors.stream() - .map(JsonPrimitive::new) - .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); - - errorObj.addProperty("index", i); - errorObj.add("errors", errors); - errorsJson.add(errorObj); - } catch (TenantOrAppNotFoundException | StorageQueryException e) { - throw new ServletException(e); - } - } - - if (errorsJson.size() > 0) { - JsonObject errorResponseJson = new JsonObject(); - errorResponseJson.addProperty("error", - "Data has missing or invalid fields. Please check the users field for more details."); - errorResponseJson.add("users", errorsJson); - throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); - } - - try { - BulkImport.addUsers(appIdentifierWithStorage, usersToAdd); - } catch (TenantOrAppNotFoundException | StorageQueryException e) { - throw new ServletException(e); - } - - JsonObject result = new JsonObject(); - result.addProperty("status", "OK"); - super.sendJsonResponse(200, result, resp); - } -} diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java similarity index 58% rename from src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java rename to src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index f89fe39c3..9c6d64310 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/GetBulkImportUsers.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -17,17 +17,21 @@ package io.supertokens.webserver.api.bulkimport; import java.io.IOException; +import java.util.ArrayList; import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; import io.supertokens.Main; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserPaginationContainer; import io.supertokens.bulkimport.BulkImportUserPaginationToken; +import io.supertokens.bulkimport.BulkImportUserUtils; import io.supertokens.multitenancy.exception.BadPermissionException; import io.supertokens.output.Logging; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportUserInfo; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; @@ -39,8 +43,8 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; -public class GetBulkImportUsers extends WebserverAPI { - public GetBulkImportUsers(Main main) { +public class BulkImportAPI extends WebserverAPI { + public BulkImportAPI(Main main) { super(main, ""); } @@ -105,4 +109,67 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se throw new ServletException(e); } } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); + + if (users.size() <= 0 || users.size() > BulkImport.MAX_USERS_TO_ADD) { + JsonObject errorResponseJson = new JsonObject(); + String errorMsg = users.size() <= 0 ? "You need to add at least one user." + : "You can only add " + BulkImport.MAX_USERS_TO_ADD + " users at a time."; + errorResponseJson.addProperty("error", errorMsg); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } + + AppIdentifierWithStorage appIdentifierWithStorage; + try { + appIdentifierWithStorage = getAppIdentifierWithStorageFromRequestAndEnforcePublicTenant(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + JsonArray errorsJson = new JsonArray(); + ArrayList usersToAdd = new ArrayList<>(); + + for (int i = 0; i < users.size(); i++) { + try { + BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifierWithStorage, users.get(i).getAsJsonObject(), Utils.getUUID()); + usersToAdd.add(user); + } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { + JsonObject errorObj = new JsonObject(); + + JsonArray errors = e.errors.stream() + .map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + + errorObj.addProperty("index", i); + errorObj.add("errors", errors); + errorsJson.add(errorObj); + } catch (Exception e) { + System.out.println("error: " + e.getMessage()); + e.printStackTrace(); + throw new ServletException(e); + } + } + + if (errorsJson.size() > 0) { + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.addProperty("error", + "Data has missing or invalid fields. Please check the users field for more details."); + errorResponseJson.add("users", errorsJson); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } + + try { + BulkImport.addUsers(appIdentifierWithStorage, usersToAdd); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new ServletException(e); + } + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + super.sendJsonResponse(200, result, resp); + } } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java new file mode 100644 index 000000000..6d9b327c5 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -0,0 +1,266 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUserInfo; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; + +public class BulkImportTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldAddUsersInBulkImportUsersTable() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + List users = generateBulkImportUser(10); + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + BulkImport.addUsers(new AppIdentifierWithStorage(null, null, storage), users); + + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, BulkImportUserStatus.NEW, null, null); + + // Verify that all users are present in addedUsers + for (BulkImportUser user : users) { + BulkImportUserInfo matchingUser = addedUsers.stream() + .filter(addedUser -> user.id.equals(addedUser.id)) + .findFirst() + .orElse(null); + + assertNotNull(matchingUser); + assertEquals(BulkImportUserStatus.NEW, matchingUser.status); + assertEquals(user.toString(), matchingUser.rawData); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + List users = generateBulkImportUser(10); + + // We are setting the id of the second user to be the same as the first user to ensure a duplicate id is present + users.get(1).id = users.get(0).id; + + List initialIds = users.stream().map(user -> user.id).collect(Collectors.toList()); + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifierWithStorage appIdentifierWithStorage = new AppIdentifierWithStorage(null, null, storage); + BulkImport.addUsers(appIdentifierWithStorage, users); + + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.NEW, null, null); + + // Verify that the other properties are same but ids changed + for (BulkImportUser user : users) { + BulkImportUserInfo matchingUser = addedUsers.stream() + .filter(addedUser -> user.toString().equals(addedUser.rawData)) + .findFirst() + .orElse(null); + + assertNotNull(matchingUser); + assertEquals(BulkImportUserStatus.NEW, matchingUser.status); + assertFalse(initialIds.contains(matchingUser.id)); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void testGetUsersStatusFilter() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(process.main); + AppIdentifierWithStorage appIdentifierWithStorage = new AppIdentifierWithStorage(null, null, storage); + + // Test with status = 'NEW' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifierWithStorage, users); + + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.NEW, null, null); + assertEquals(10, addedUsers.size()); + } + + // Test with status = 'PROCESSING' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifierWithStorage, users); + + // Update the users status to PROCESSING + String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); + + storage.startTransaction(con -> { + storage.updateBulkImportUserStatus_Transaction(appIdentifierWithStorage, con, userIds, BulkImportUserStatus.PROCESSING); + storage.commitTransaction(con); + return null; + }); + + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.PROCESSING, null, null); + assertEquals(10, addedUsers.size()); + } + + // Test with status = 'FAILED' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifierWithStorage, users); + + // Update the users status to FAILED + String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); + + storage.startTransaction(con -> { + storage.updateBulkImportUserStatus_Transaction(appIdentifierWithStorage, con, userIds, BulkImportUserStatus.FAILED); + storage.commitTransaction(con); + return null; + }); + + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.FAILED, null, null); + assertEquals(10, addedUsers.size()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void randomPaginationTest() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + + int numberOfUsers = 500; + // Insert users in batches + { + int batchSize = 100; + for (int i = 0; i < numberOfUsers; i += batchSize) { + List users = generateBulkImportUser(batchSize); + BulkImport.addUsers(new AppIdentifierWithStorage(null, null, storage), users); + // Adding a delay between each batch to ensure the createdAt different + Thread.sleep(1000); + } + } + + // Get all inserted users + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, null, null, null); + assertEquals(numberOfUsers, addedUsers.size()); + + // We are sorting the users based on createdAt and id like we do in the storage layer + List sortedUsers = addedUsers.stream() + .sorted((user1, user2) -> { + int compareResult = user2.createdAt.compareTo(user1.createdAt); + if (compareResult == 0) { + return user2.id.compareTo(user1.id); + } + return compareResult; + }) + .collect(Collectors.toList()); + + int[] limits = new int[]{10, 14, 20, 23, 50, 100, 110, 150, 200, 510}; + + for (int limit : limits) { + int indexIntoUsers = 0; + String paginationToken = null; + do { + BulkImportUserPaginationContainer users = BulkImport.getUsers(new AppIdentifierWithStorage(null, null, storage), limit, null, paginationToken); + + for (BulkImportUserInfo actualUser : users.users) { + BulkImportUserInfo expectedUser = sortedUsers.get(indexIntoUsers); + + assertEquals(expectedUser.id, actualUser.id); + assertEquals(expectedUser.rawData, actualUser.rawData); + indexIntoUsers++; + } + + paginationToken = users.nextPaginationToken; + } while (paginationToken != null); + + assert (indexIntoUsers == sortedUsers.size()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java new file mode 100644 index 000000000..86952977f --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + + +package io.supertokens.test.bulkimport; + +import java.util.ArrayList; +import java.util.List; + +import com.google.gson.JsonObject; + +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.EmailPasswordLoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.PasswordlessLoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.ThirdPartyLoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; + +public class BulkImportTestUtils { + public static List generateBulkImportUser(int numberOfUsers) { + List users = new ArrayList<>(); + + for (int i = 0; i < numberOfUsers; i++) { + String email = "user" + i + "@example.com"; + String id = io.supertokens.utils.Utils.getUUID(); + String externalId = io.supertokens.utils.Utils.getUUID(); + + JsonObject userMetadata = new JsonObject(); + userMetadata.addProperty("key", "value"); + + List userRoles = new ArrayList<>(); + userRoles.add("role1"); + userRoles.add("role2"); + + List totpDevices = new ArrayList<>(); + totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); + + EmailPasswordLoginMethod emailPasswordLoginMethod = new EmailPasswordLoginMethod(email, "$2a", "BCRYPT"); + ThirdPartyLoginMethod thirdPartyLoginMethod = new ThirdPartyLoginMethod(email, "thirdPartyId", "thirdPartyUserId"); + PasswordlessLoginMethod passwordlessLoginMethod = new PasswordlessLoginMethod(email, "+911234567890"); + + List loginMethods = new ArrayList<>(); + loginMethods.add(new LoginMethod("public", "emailpassword", true, true, 0, emailPasswordLoginMethod, null, null)); + loginMethods.add(new LoginMethod("public", "thirdparty", true, false, 0, null, thirdPartyLoginMethod, null)); + loginMethods.add(new LoginMethod("public", "passwordless", true, false, 0, null, null, passwordlessLoginMethod)); + users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); + } + return users; + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index f20bceb42..376f4b3ff 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -79,7 +79,7 @@ public void shouldThrow400Error() throws Exception { try { JsonObject request = new JsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -90,7 +90,7 @@ public void shouldThrow400Error() throws Exception { try { JsonObject request = new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -104,7 +104,7 @@ public void shouldThrow400Error() throws Exception { try { JsonObject request = new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -116,7 +116,7 @@ public void shouldThrow400Error() throws Exception { JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -128,7 +128,7 @@ public void shouldThrow400Error() throws Exception { try { JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -143,7 +143,7 @@ public void shouldThrow400Error() throws Exception { .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"roles\":{},\"totp\":{}}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -159,13 +159,13 @@ public void shouldThrow400Error() throws Exception { "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantId\":[],\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); assertEquals(400, e.statusCode); assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantId should be of type string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type number for a loginMethod\"]}]}"); + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantId should be of type string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); } } // Invalid recipeId @@ -175,7 +175,7 @@ public void shouldThrow400Error() throws Exception { .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -191,7 +191,7 @@ public void shouldThrow400Error() throws Exception { JsonObject request = new JsonParser() .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}").getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -205,7 +205,7 @@ public void shouldThrow400Error() throws Exception { "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":[],\"passwordHash\":[],\"hashingAlgorithm\":[]}]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -219,7 +219,7 @@ public void shouldThrow400Error() throws Exception { "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -235,7 +235,7 @@ public void shouldThrow400Error() throws Exception { JsonObject request = new JsonParser() .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}").getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -249,7 +249,7 @@ public void shouldThrow400Error() throws Exception { "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\",\"email\":[],\"thirdPartyId\":[],\"thirdPartyUserId\":[]}]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -265,7 +265,7 @@ public void shouldThrow400Error() throws Exception { JsonObject request = new JsonParser() .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}").getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -279,7 +279,7 @@ public void shouldThrow400Error() throws Exception { "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":[],\"phoneNumber\":[]}]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -296,7 +296,7 @@ public void shouldThrow400Error() throws Exception { "{\"users\":[{\"loginMethods\":[{\"tenantId\":\"invalid\",\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -313,7 +313,7 @@ public void shouldThrow400Error() throws Exception { "{\"users\":[{\"loginMethods\":[{\"tenantId\":\"invalid\",\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -329,7 +329,7 @@ public void shouldThrow400Error() throws Exception { JsonObject request = new JsonParser() .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}").getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -343,7 +343,7 @@ public void shouldThrow400Error() throws Exception { try { JsonObject request = generateUsersJson(0); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -356,7 +356,7 @@ public void shouldThrow400Error() throws Exception { try { JsonObject request = generateUsersJson(10001); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { String responseString = getResponseMessageFromError(e.getMessage()); @@ -382,7 +382,7 @@ public void shouldReturn200Response() throws Exception { JsonObject request = generateUsersJson(10000); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); @@ -403,7 +403,7 @@ public void shouldNormaliseFields() throws Exception { JsonObject request = generateUsersJson(1); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteFailedBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteFailedBulkImportUsersTest.java new file mode 100644 index 000000000..0e8ff9fdf --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteFailedBulkImportUsersTest.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.List; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; + +public class DeleteFailedBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + { + try { + JsonObject request = new JsonObject(); + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' is invalid in JSON input", e.getMessage()); + } + } + { + try { + JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot be an empty array", e.getMessage()); + } + } + { + try { + JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain an empty string", e.getMessage()); + } + } + { + try { + // Create a string array of 500 uuids + JsonObject request = new JsonObject(); + JsonArray ids = new JsonArray(); + for (int i = 0; i < 501; i++) { + ids.add(new JsonPrimitive(io.supertokens.utils.Utils.getUUID())); + } + request.add("ids", ids); + + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain more than 500 elements", e.getMessage()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifierWithStorage appIdentifierWithStorage = new AppIdentifierWithStorage(null, null, storage); + + // Insert users + List users = generateBulkImportUser(5); + BulkImport.addUsers(appIdentifierWithStorage, users); + + JsonObject request = new JsonObject(); + JsonArray ids = new JsonArray(); + for (BulkImportUser user : users) { + ids.add(new JsonPrimitive(user.id)); + } + request.add("ids", ids); + + JsonObject response = HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java index 64539bee8..77fad9b02 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -136,7 +136,7 @@ public void shouldReturn200Response() throws Exception { { JsonObject request = new JsonParser().parse(rawData).getAsJsonObject(); JsonObject res = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/add-users", + "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assert res.get("status").getAsString().equals("OK"); } From ee82e2d2d083396df1882f085316b16b779c91ab Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 26 Feb 2024 11:30:20 +0530 Subject: [PATCH 07/15] fix: PR changes --- .../bulkimport/BulkImportUserUtils.java | 120 ++++++++++++++---- 1 file changed, 95 insertions(+), 25 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index e5c107bd8..21349d18c 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -18,7 +18,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -79,7 +81,7 @@ private static List getParsedUserRoles(JsonObject userData, List // We already know that the jsonUserRoles is an array of non-empty strings, we will normalise each role now List userRoles = new ArrayList<>(); - jsonUserRoles.forEach(role -> userRoles.add(validateAndNormaliseUserRole(role.getAsString()))); + jsonUserRoles.forEach(role -> userRoles.add(validateAndNormaliseUserRole(role.getAsString(), errors))); return userRoles; } @@ -98,10 +100,10 @@ private static List getParsedTotpDevices(JsonObject userData, List getParsedLoginMethods(Main main, AppIdentifier isPrimary = validateAndNormaliseIsPrimary(isPrimary); isVerified = validateAndNormaliseIsVerified(isVerified); - long timeJoinedInMSSinceEpoch = validateAndNormaliseTimeJoined(timeJoined); + long timeJoinedInMSSinceEpoch = validateAndNormaliseTimeJoined(timeJoined, errors); if ("emailpassword".equals(recipeId)) { String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); String passwordHash = parseAndValidateFieldType(jsonLoginMethodObj, "passwordHash", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); String hashingAlgorithm = parseAndValidateFieldType(jsonLoginMethodObj, "hashingAlgorithm", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); - email = validateAndNormaliseEmail(email); + email = validateAndNormaliseEmail(email, errors); CoreConfig.PASSWORD_HASHING_ALG normalisedHashingAlgorithm = validateAndNormaliseHashingAlgorithm(hashingAlgorithm, errors); hashingAlgorithm = normalisedHashingAlgorithm != null ? normalisedHashingAlgorithm.toString() : hashingAlgorithm; passwordHash = validateAndNormalisePasswordHash(main, appIdentifier, normalisedHashingAlgorithm, passwordHash, errors); @@ -160,9 +162,9 @@ private static List getParsedLoginMethods(Main main, AppIdentifier String thirdPartyId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); String thirdPartyUserId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyUserId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); - email = validateAndNormaliseEmail(email); - thirdPartyId = validateAndNormaliseThirdPartyId(thirdPartyId); - thirdPartyUserId = validateAndNormaliseThirdPartyUserId(thirdPartyUserId); + email = validateAndNormaliseEmail(email, errors); + thirdPartyId = validateAndNormaliseThirdPartyId(thirdPartyId, errors); + thirdPartyUserId = validateAndNormaliseThirdPartyUserId(thirdPartyUserId, errors); ThirdPartyLoginMethod thirdPartyLoginMethod = new ThirdPartyLoginMethod(email, thirdPartyId, thirdPartyUserId); loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, thirdPartyLoginMethod, null)); @@ -170,8 +172,8 @@ private static List getParsedLoginMethods(Main main, AppIdentifier String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); String phoneNumber = parseAndValidateFieldType(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); - email = validateAndNormaliseEmail(email); - phoneNumber = validateAndNormalisePhoneNumber(phoneNumber); + email = validateAndNormaliseEmail(email, errors); + phoneNumber = validateAndNormalisePhoneNumber(phoneNumber, errors); PasswordlessLoginMethod passwordlessLoginMethod = new PasswordlessLoginMethod(email, phoneNumber); loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, null, passwordlessLoginMethod)); @@ -180,17 +182,29 @@ private static List getParsedLoginMethods(Main main, AppIdentifier return loginMethods; } - private static String validateAndNormaliseUserRole(String role) { + private static String validateAndNormaliseUserRole(String role, List errors) { + if (role.length() > 255) { + errors.add("role " + role + " is too long. Max length is 255."); + } + // We just trim the role the CreateRoleAPI.java return role.trim(); } - private static String validateAndNormaliseTotpSecretKey(String secretKey) { + private static String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { + if (secretKey == null ) { + return null; + } + + if (secretKey.length() > 256) { + errors.add("TOTP secretKey " + secretKey + " is too long. Max length is 256."); + } + // We don't perform any normalisation on the secretKey in ImportTotpDeviceAPI.java return secretKey; } - private static Integer validateAndNormaliseTotpPeriod(Number period, List errors) { + private static Integer validateAndNormaliseTotpPeriod(Integer period, List errors) { // We don't perform any normalisation on the period in ImportTotpDeviceAPI.java other than checking if it is > 0 if (period != null && period.intValue() < 1) { errors.add("period should be > 0 for a totp device."); @@ -199,7 +213,7 @@ private static Integer validateAndNormaliseTotpPeriod(Number period, List errors) { + private static Integer validateAndNormaliseTotpSkew(Integer skew, List errors) { // We don't perform any normalisation on the period in ImportTotpDeviceAPI.java other than checking if it is >= 0 if (skew != null && skew.intValue() < 0) { errors.add("skew should be >= 0 for a totp device."); @@ -208,9 +222,17 @@ private static Integer validateAndNormaliseTotpSkew(Number skew, List er return skew != null ? skew.intValue() : null; } - private static String validateAndNormaliseTotpDeviceName(String deviceName) { + private static String validateAndNormaliseTotpDeviceName(String deviceName, List errors) { + if (deviceName == null ) { + return null; + } + + if (deviceName.length() > 256) { + errors.add("TOTP deviceName " + deviceName + " is too long. Max length is 256."); + } + // We normalise the deviceName as per the ImportTotpDeviceAPI.java - return deviceName != null ? deviceName.trim() : null; + return deviceName.trim(); } private static void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethods, List errors) { @@ -257,7 +279,7 @@ private static String validateAndNormaliseTenantId(Main main, AppIdentifier appI // We make the tenantId lowercase while parsing from the request in WebserverAPI.java String normalisedTenantId = tenantId.trim().toLowerCase(); TenantConfig[] allTenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); - ArrayList validTenantIds = new ArrayList<>(); + Set validTenantIds = new HashSet<>(); Arrays.stream(allTenantConfigs) .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); @@ -277,14 +299,34 @@ private static Boolean validateAndNormaliseIsVerified(Boolean isVerified) { return isVerified == null ? false : isVerified; } - private static long validateAndNormaliseTimeJoined(Integer timeJoined) { + private static long validateAndNormaliseTimeJoined(Integer timeJoined, List errors) { // We default timeJoined to 0 if it is null - return timeJoined != null ? timeJoined.longValue() : 0; + if (timeJoined == null) { + return 0; + } + + if (timeJoined > System.currentTimeMillis()) { + errors.add("timeJoined cannot be in future for a loginMethod."); + } + + if (timeJoined < 0) { + errors.add("timeJoined cannot be < 0 for a loginMethod."); + } + + return timeJoined.longValue(); } - private static String validateAndNormaliseEmail(String email) { + private static String validateAndNormaliseEmail(String email, List errors) { + if (email == null) { + return null; + } + + if (email.length() > 255) { + errors.add("email " + email + " is too long. Max length is 256."); + } + // We normalise the email as per the SignUpAPI.java - return email != null ? Utils.normaliseEmail(email) : null; + return Utils.normaliseEmail(email); } private static CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgorithm(String hashingAlgorithm, List errors) { @@ -305,6 +347,10 @@ private static String validateAndNormalisePasswordHash(Main main, AppIdentifier if (hashingAlgorithm == null || passwordHash == null) { return passwordHash; } + + if (passwordHash.length() > 256) { + errors.add("passwordHash is too long. Max length is 256."); + } // We trim the passwordHash and validate it as per ImportUserWithPasswordHashAPI.java passwordHash = passwordHash.trim(); @@ -318,19 +364,43 @@ private static String validateAndNormalisePasswordHash(Main main, AppIdentifier return passwordHash; } - private static String validateAndNormaliseThirdPartyId(String thirdPartyId) { + private static String validateAndNormaliseThirdPartyId(String thirdPartyId, List errors) { + if (thirdPartyId == null) { + return null; + } + + if (thirdPartyId.length() > 28) { + errors.add("thirdPartyId " + thirdPartyId + " is too long. Max length is 28."); + } + // We don't perform any normalisation on the thirdPartyId in SignInUpAPI.java return thirdPartyId; } - private static String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId) { + private static String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId, List errors) { + if (thirdPartyUserId == null) { + return null; + } + + if (thirdPartyUserId.length() > 256) { + errors.add("thirdPartyUserId " + thirdPartyUserId + " is too long. Max length is 256."); + } + // We don't perform any normalisation on the thirdPartyUserId in SignInUpAPI.java return thirdPartyUserId; } - private static String validateAndNormalisePhoneNumber(String phoneNumber) { + private static String validateAndNormalisePhoneNumber(String phoneNumber, List errors) { + if (phoneNumber == null) { + return null; + } + + if (phoneNumber.length() > 256) { + errors.add("phoneNumber " + phoneNumber + " is too long. Max length is 256."); + } + // We normalise the phoneNumber as per the CreateCodeAPI.java - return phoneNumber != null ? Utils.normalizeIfPhoneNumber(phoneNumber) : null; + return Utils.normalizeIfPhoneNumber(phoneNumber); } } From 39821968017d23fcac17c0507acccb0c6741378e Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 27 Feb 2024 14:59:52 +0530 Subject: [PATCH 08/15] fix: PR changes --- .../java/io/supertokens/inmemorydb/config/SQLiteConfig.java | 4 ---- .../io/supertokens/inmemorydb/queries/GeneralQueries.java | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/io/supertokens/inmemorydb/config/SQLiteConfig.java b/src/main/java/io/supertokens/inmemorydb/config/SQLiteConfig.java index e27c3ea16..ee8c43241 100644 --- a/src/main/java/io/supertokens/inmemorydb/config/SQLiteConfig.java +++ b/src/main/java/io/supertokens/inmemorydb/config/SQLiteConfig.java @@ -164,8 +164,4 @@ public String getDashboardUsersTable() { public String getDashboardSessionsTable() { return "dashboard_user_sessions"; } - - public String getBulkImportUsersTable() { - return "bulk_import_users"; - } } diff --git a/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java b/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java index 0f519dabf..46fc162af 100644 --- a/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java +++ b/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java @@ -416,6 +416,7 @@ public static void createTablesIfNotExists(Start start, Main main) throws SQLExc // index: update(start, TOTPQueries.getQueryToCreateUsedCodesExpiryTimeIndex(start), NO_OP_SETTER); } + } From 7d25645c1ab25454a70414263311b6704c0271fe Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 27 Feb 2024 16:52:35 +0530 Subject: [PATCH 09/15] fix: PR changes --- .../bulkimport/BulkImportUserUtils.java | 17 ++++++++++++++++- .../supertokens/utils/JsonValidatorUtils.java | 4 ++-- .../test/bulkimport/BulkImportTestUtils.java | 5 +++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 21349d18c..083d280f1 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -64,6 +64,8 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi List totpDevices = getParsedTotpDevices(userData, errors); List loginMethods = getParsedLoginMethods(main, appIdentifier, userData, errors); + externalUserId = validateAndNormaliseExternalUserId(externalUserId, errors); + if (!errors.isEmpty()) { throw new InvalidBulkImportDataException(errors); } @@ -182,12 +184,25 @@ private static List getParsedLoginMethods(Main main, AppIdentifier return loginMethods; } + private static String validateAndNormaliseExternalUserId(String externalUserId, List errors) { + if (externalUserId == null ) { + return null; + } + + if (externalUserId.length() > 255) { + errors.add("externalUserId " + externalUserId + " is too long. Max length is 128."); + } + + // We just trim the externalUserId as per the UpdateExternalUserIdInfoAPI.java + return externalUserId.trim(); + } + private static String validateAndNormaliseUserRole(String role, List errors) { if (role.length() > 255) { errors.add("role " + role + " is too long. Max length is 255."); } - // We just trim the role the CreateRoleAPI.java + // We just trim the role as per the CreateRoleAPI.java return role.trim(); } diff --git a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java index beb65a87b..7fdc564fa 100644 --- a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java +++ b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java @@ -90,7 +90,7 @@ public static boolean validateJsonFieldType(JsonObject jsonObject, String key, V if (jsonObject.has(key)) { return switch (expectedType) { case STRING -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isString() - && !jsonObject.get(key).getAsString().isEmpty(); + && !jsonObject.get(key).getAsString().isBlank(); case INTEGER -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isNumber(); case BOOLEAN -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isBoolean(); case OBJECT -> jsonObject.get(key).isJsonObject(); @@ -110,7 +110,7 @@ public static boolean validateArrayElements(JsonArray array, ValueType expectedT case ARRAY_OF_OBJECT -> elements.stream().allMatch(JsonElement::isJsonObject); case ARRAY_OF_STRING -> elements.stream().allMatch(el -> el.isJsonPrimitive() && el.getAsJsonPrimitive().isString() - && !el.getAsString().isEmpty()); + && !el.getAsString().isBlank()); default -> false; }; } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index 86952977f..61e9c6241 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -21,6 +21,7 @@ import java.util.List; import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; @@ -32,14 +33,14 @@ public class BulkImportTestUtils { public static List generateBulkImportUser(int numberOfUsers) { List users = new ArrayList<>(); + JsonParser parser = new JsonParser(); for (int i = 0; i < numberOfUsers; i++) { String email = "user" + i + "@example.com"; String id = io.supertokens.utils.Utils.getUUID(); String externalId = io.supertokens.utils.Utils.getUUID(); - JsonObject userMetadata = new JsonObject(); - userMetadata.addProperty("key", "value"); + JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}").getAsJsonObject(); List userRoles = new ArrayList<>(); userRoles.add("role1"); From 3bb6adfdc651bacf5d87bcc2b416aa6ee482e916 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 28 Feb 2024 10:16:27 +0530 Subject: [PATCH 10/15] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 7 ++-- .../BulkImportUserPaginationContainer.java | 6 ++-- .../bulkimport/BulkImportUserUtils.java | 35 +++++++++--------- .../supertokens/utils/JsonValidatorUtils.java | 8 ++++- .../api/bulkimport/BulkImportAPI.java | 17 +++++---- .../test/bulkimport/BulkImportTest.java | 31 ++++++++-------- .../test/bulkimport/BulkImportTestUtils.java | 15 ++------ .../apis/AddBulkImportUsersTest.java | 36 ++++++++++++++++--- 8 files changed, 92 insertions(+), 63 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 49e539bfc..f2136064d 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -18,7 +18,6 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; -import io.supertokens.pluginInterface.bulkimport.BulkImportUserInfo; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; @@ -54,7 +53,7 @@ public static void addUsers(AppIdentifierWithStorage appIdentifierWithStorage, L public static BulkImportUserPaginationContainer getUsers(AppIdentifierWithStorage appIdentifierWithStorage, @Nonnull Integer limit, @Nullable BulkImportUserStatus status, @Nullable String paginationToken) throws StorageQueryException, BulkImportUserPaginationToken.InvalidTokenException { - List users; + List users; if (paginationToken == null) { users = appIdentifierWithStorage.getBulkImportStorage() @@ -69,11 +68,11 @@ public static BulkImportUserPaginationContainer getUsers(AppIdentifierWithStorag int maxLoop = users.size(); if (users.size() == limit + 1) { maxLoop = limit; - BulkImportUserInfo user = users.get(limit); + BulkImportUser user = users.get(limit); nextPaginationToken = new BulkImportUserPaginationToken(user.id, user.createdAt).generateToken(); } - List resultUsers = users.subList(0, maxLoop); + List resultUsers = users.subList(0, maxLoop); return new BulkImportUserPaginationContainer(resultUsers, nextPaginationToken); } } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java index 2993a83a7..f691c68c3 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java @@ -21,13 +21,13 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import io.supertokens.pluginInterface.bulkimport.BulkImportUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; public class BulkImportUserPaginationContainer { - public final List users; + public final List users; public final String nextPaginationToken; - public BulkImportUserPaginationContainer(@Nonnull List users, @Nullable String nextPaginationToken) { + public BulkImportUserPaginationContainer(@Nonnull List users, @Nullable String nextPaginationToken) { this.users = users; this.nextPaginationToken = nextPaginationToken; } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 083d280f1..7b1f20a3c 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -36,9 +36,6 @@ import io.supertokens.multitenancy.Multitenancy; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.EmailPasswordLoginMethod; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.ThirdPartyLoginMethod; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.PasswordlessLoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; @@ -52,7 +49,7 @@ import static io.supertokens.utils.JsonValidatorUtils.validateJsonFieldType; public class BulkImportUserUtils { - public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, JsonObject userData, String id) + public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, JsonObject userData, String id, String[] allUserRoles) throws InvalidBulkImportDataException, StorageQueryException, TenantOrAppNotFoundException { List errors = new ArrayList<>(); @@ -60,7 +57,7 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi errors, "."); JsonObject userMetadata = parseAndValidateFieldType(userData, "userMetadata", ValueType.OBJECT, false, JsonObject.class, errors, "."); - List userRoles = getParsedUserRoles(userData, errors); + List userRoles = getParsedUserRoles(userData, allUserRoles, errors); List totpDevices = getParsedTotpDevices(userData, errors); List loginMethods = getParsedLoginMethods(main, appIdentifier, userData, errors); @@ -72,7 +69,7 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi return new BulkImportUser(id, externalUserId, userMetadata, userRoles, totpDevices, loginMethods); } - private static List getParsedUserRoles(JsonObject userData, List errors) { + private static List getParsedUserRoles(JsonObject userData, String[] allUserRoles, List errors) { JsonArray jsonUserRoles = parseAndValidateFieldType(userData, "roles", ValueType.ARRAY_OF_STRING, false, JsonArray.class, errors, "."); @@ -83,7 +80,14 @@ private static List getParsedUserRoles(JsonObject userData, List // We already know that the jsonUserRoles is an array of non-empty strings, we will normalise each role now List userRoles = new ArrayList<>(); - jsonUserRoles.forEach(role -> userRoles.add(validateAndNormaliseUserRole(role.getAsString(), errors))); + jsonUserRoles.forEach(role -> { + String normalisedRole = validateAndNormaliseUserRole(role.getAsString(), errors); + if (Arrays.asList(allUserRoles).contains(normalisedRole)) { + userRoles.add(normalisedRole); + } else { + errors.add("Role " + normalisedRole + " does not exist."); + } + }); return userRoles; } @@ -138,7 +142,7 @@ private static List getParsedLoginMethods(Main main, AppIdentifier String tenantId = parseAndValidateFieldType(jsonLoginMethodObj, "tenantId", ValueType.STRING, false, String.class, errors, " for a loginMethod."); Boolean isVerified = parseAndValidateFieldType(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); Boolean isPrimary = parseAndValidateFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, Boolean.class, errors, " for a loginMethod."); - Integer timeJoined = parseAndValidateFieldType(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.INTEGER, false, Integer.class, errors, " for a loginMethod"); + Long timeJoined = parseAndValidateFieldType(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.LONG, false, Long.class, errors, " for a loginMethod"); recipeId = validateAndNormaliseRecipeId(recipeId, errors); tenantId= validateAndNormaliseTenantId(main, appIdentifier, tenantId, recipeId, errors); @@ -157,8 +161,7 @@ private static List getParsedLoginMethods(Main main, AppIdentifier hashingAlgorithm = normalisedHashingAlgorithm != null ? normalisedHashingAlgorithm.toString() : hashingAlgorithm; passwordHash = validateAndNormalisePasswordHash(main, appIdentifier, normalisedHashingAlgorithm, passwordHash, errors); - EmailPasswordLoginMethod emailPasswordLoginMethod = new EmailPasswordLoginMethod(email, passwordHash, hashingAlgorithm); - loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, emailPasswordLoginMethod, null, null)); + loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, email, passwordHash, hashingAlgorithm, null, null, null)); } else if ("thirdparty".equals(recipeId)) { String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); String thirdPartyId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); @@ -168,8 +171,7 @@ private static List getParsedLoginMethods(Main main, AppIdentifier thirdPartyId = validateAndNormaliseThirdPartyId(thirdPartyId, errors); thirdPartyUserId = validateAndNormaliseThirdPartyUserId(thirdPartyUserId, errors); - ThirdPartyLoginMethod thirdPartyLoginMethod = new ThirdPartyLoginMethod(email, thirdPartyId, thirdPartyUserId); - loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, thirdPartyLoginMethod, null)); + loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, email, null, null, thirdPartyId, thirdPartyUserId, null)); } else if ("passwordless".equals(recipeId)) { String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); String phoneNumber = parseAndValidateFieldType(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); @@ -177,8 +179,7 @@ private static List getParsedLoginMethods(Main main, AppIdentifier email = validateAndNormaliseEmail(email, errors); phoneNumber = validateAndNormalisePhoneNumber(phoneNumber, errors); - PasswordlessLoginMethod passwordlessLoginMethod = new PasswordlessLoginMethod(email, phoneNumber); - loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, null, null, passwordlessLoginMethod)); + loginMethods.add(new LoginMethod(tenantId, recipeId, isVerified, isPrimary, timeJoinedInMSSinceEpoch, email, null, null, null, null, phoneNumber)); } } return loginMethods; @@ -314,10 +315,10 @@ private static Boolean validateAndNormaliseIsVerified(Boolean isVerified) { return isVerified == null ? false : isVerified; } - private static long validateAndNormaliseTimeJoined(Integer timeJoined, List errors) { - // We default timeJoined to 0 if it is null + private static long validateAndNormaliseTimeJoined(Long timeJoined, List errors) { + // We default timeJoined to currentTime if it is null if (timeJoined == null) { - return 0; + return System.currentTimeMillis(); } if (timeJoined > System.currentTimeMillis()) { diff --git a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java index 7fdc564fa..6c2f81cf3 100644 --- a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java +++ b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java @@ -38,6 +38,10 @@ public static T parseAndValidateFieldType(JsonObject jsonObject, String key, Integer intValue = jsonObject.get(key).getAsNumber().intValue(); value = (T) intValue; break; + case LONG: + Long longValue = jsonObject.get(key).getAsNumber().longValue(); + value = (T) longValue; + break; case BOOLEAN: Boolean boolValue = jsonObject.get(key).getAsBoolean(); value = (T) boolValue; @@ -69,6 +73,7 @@ public static T parseAndValidateFieldType(JsonObject jsonObject, String key, public enum ValueType { STRING, INTEGER, + LONG, BOOLEAN, OBJECT, ARRAY_OF_STRING, @@ -79,6 +84,7 @@ private static String getTypeForErrorMessage(ValueType type) { return switch (type) { case STRING -> "string"; case INTEGER -> "integer"; + case LONG -> "integer"; // choosing integer over long because it is user facing case BOOLEAN -> "boolean"; case OBJECT -> "object"; case ARRAY_OF_STRING -> "array of string"; @@ -91,7 +97,7 @@ public static boolean validateJsonFieldType(JsonObject jsonObject, String key, V return switch (expectedType) { case STRING -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isString() && !jsonObject.get(key).getAsString().isBlank(); - case INTEGER -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isNumber(); + case INTEGER, LONG -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isNumber(); case BOOLEAN -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isBoolean(); case OBJECT -> jsonObject.get(key).isJsonObject(); case ARRAY_OF_OBJECT, ARRAY_OF_STRING -> jsonObject.get(key).isJsonArray() diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index 9c6d64310..2f1f5f94a 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -32,7 +32,6 @@ import io.supertokens.output.Logging; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; -import io.supertokens.pluginInterface.bulkimport.BulkImportUserInfo; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; @@ -93,7 +92,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se result.addProperty("status", "OK"); JsonArray usersJson = new JsonArray(); - for (BulkImportUserInfo user : users.users) { + for (BulkImportUser user : users.users) { usersJson.add(user.toJsonObject()); } result.add("users", usersJson); @@ -130,12 +129,20 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S throw new ServletException(e); } + String[] allUserRoles = null; + + try { + allUserRoles = appIdentifierWithStorage.getUserRolesStorage().getRoles(appIdentifierWithStorage); + } catch (StorageQueryException e) { + throw new ServletException(e); + } + JsonArray errorsJson = new JsonArray(); ArrayList usersToAdd = new ArrayList<>(); for (int i = 0; i < users.size(); i++) { try { - BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifierWithStorage, users.get(i).getAsJsonObject(), Utils.getUUID()); + BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifierWithStorage, users.get(i).getAsJsonObject(), Utils.getUUID(), allUserRoles); usersToAdd.add(user); } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { JsonObject errorObj = new JsonObject(); @@ -147,9 +154,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S errorObj.addProperty("index", i); errorObj.add("errors", errors); errorsJson.add(errorObj); - } catch (Exception e) { - System.out.println("error: " + e.getMessage()); - e.printStackTrace(); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new ServletException(e); } } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index 6d9b327c5..4ff949e09 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -37,7 +37,6 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; -import io.supertokens.pluginInterface.bulkimport.BulkImportUserInfo; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; import io.supertokens.storageLayer.StorageLayer; @@ -76,18 +75,18 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); BulkImport.addUsers(new AppIdentifierWithStorage(null, null, storage), users); - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, BulkImportUserStatus.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, BulkImportUserStatus.NEW, null, null); // Verify that all users are present in addedUsers for (BulkImportUser user : users) { - BulkImportUserInfo matchingUser = addedUsers.stream() + BulkImportUser matchingUser = addedUsers.stream() .filter(addedUser -> user.id.equals(addedUser.id)) .findFirst() .orElse(null); assertNotNull(matchingUser); assertEquals(BulkImportUserStatus.NEW, matchingUser.status); - assertEquals(user.toString(), matchingUser.rawData); + assertEquals(user.toString(), matchingUser.toRawData()); } process.kill(); @@ -116,12 +115,12 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { AppIdentifierWithStorage appIdentifierWithStorage = new AppIdentifierWithStorage(null, null, storage); BulkImport.addUsers(appIdentifierWithStorage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.NEW, null, null); // Verify that the other properties are same but ids changed for (BulkImportUser user : users) { - BulkImportUserInfo matchingUser = addedUsers.stream() - .filter(addedUser -> user.toString().equals(addedUser.rawData)) + BulkImportUser matchingUser = addedUsers.stream() + .filter(addedUser -> user.toString().equals(addedUser.toRawData())) .findFirst() .orElse(null); @@ -153,7 +152,7 @@ public void testGetUsersStatusFilter() throws Exception { List users = generateBulkImportUser(10); BulkImport.addUsers(appIdentifierWithStorage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.NEW, null, null); assertEquals(10, addedUsers.size()); } @@ -171,7 +170,7 @@ public void testGetUsersStatusFilter() throws Exception { return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.PROCESSING, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.PROCESSING, null, null); assertEquals(10, addedUsers.size()); } @@ -189,7 +188,7 @@ public void testGetUsersStatusFilter() throws Exception { return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.FAILED, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.FAILED, null, null); assertEquals(10, addedUsers.size()); } @@ -223,13 +222,13 @@ public void randomPaginationTest() throws Exception { } // Get all inserted users - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, null, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, null, null, null); assertEquals(numberOfUsers, addedUsers.size()); // We are sorting the users based on createdAt and id like we do in the storage layer - List sortedUsers = addedUsers.stream() + List sortedUsers = addedUsers.stream() .sorted((user1, user2) -> { - int compareResult = user2.createdAt.compareTo(user1.createdAt); + int compareResult = Long.compare(user2.createdAt, user1.createdAt); if (compareResult == 0) { return user2.id.compareTo(user1.id); } @@ -245,11 +244,11 @@ public void randomPaginationTest() throws Exception { do { BulkImportUserPaginationContainer users = BulkImport.getUsers(new AppIdentifierWithStorage(null, null, storage), limit, null, paginationToken); - for (BulkImportUserInfo actualUser : users.users) { - BulkImportUserInfo expectedUser = sortedUsers.get(indexIntoUsers); + for (BulkImportUser actualUser : users.users) { + BulkImportUser expectedUser = sortedUsers.get(indexIntoUsers); assertEquals(expectedUser.id, actualUser.id); - assertEquals(expectedUser.rawData, actualUser.rawData); + assertEquals(expectedUser.toString(), actualUser.toString()); indexIntoUsers++; } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index 61e9c6241..1aecc66c6 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -25,9 +25,6 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.EmailPasswordLoginMethod; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.PasswordlessLoginMethod; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod.ThirdPartyLoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; public class BulkImportTestUtils { @@ -43,20 +40,14 @@ public static List generateBulkImportUser(int numberOfUsers) { JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}").getAsJsonObject(); List userRoles = new ArrayList<>(); - userRoles.add("role1"); - userRoles.add("role2"); List totpDevices = new ArrayList<>(); totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); - EmailPasswordLoginMethod emailPasswordLoginMethod = new EmailPasswordLoginMethod(email, "$2a", "BCRYPT"); - ThirdPartyLoginMethod thirdPartyLoginMethod = new ThirdPartyLoginMethod(email, "thirdPartyId", "thirdPartyUserId"); - PasswordlessLoginMethod passwordlessLoginMethod = new PasswordlessLoginMethod(email, "+911234567890"); - List loginMethods = new ArrayList<>(); - loginMethods.add(new LoginMethod("public", "emailpassword", true, true, 0, emailPasswordLoginMethod, null, null)); - loginMethods.add(new LoginMethod("public", "thirdparty", true, false, 0, null, thirdPartyLoginMethod, null)); - loginMethods.add(new LoginMethod("public", "passwordless", true, false, 0, null, null, passwordlessLoginMethod)); + loginMethods.add(new LoginMethod("public", "emailpassword", true, true, 0, email, "$2a", "BCRYPT", null, null, null)); + loginMethods.add(new LoginMethod("public", "thirdparty", true, false, 0, email, null, null, "thirdPartyId", "thirdPartyUserId", null)); + loginMethods.add(new LoginMethod("public", "passwordless", true, false, 0, email, null, null, null, null, "+911234567890")); users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); } return users; diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 376f4b3ff..4684b0f92 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -41,6 +41,7 @@ import io.supertokens.test.TestingProcessManager; import io.supertokens.test.Utils; import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.userroles.UserRoles; public class AddBulkImportUsersTest { @Rule @@ -151,6 +152,21 @@ public void shouldThrow400Error() throws Exception { assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"roles should be of type array of string.\",\"totp should be of type array of object.\",\"loginMethods is required.\"]}]}"); } + // Invalid role (does not exist) + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"roles\":[\"role5\"]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); + } } // Invalid field type of non required fields inside loginMethod { @@ -380,6 +396,12 @@ public void shouldReturn200Response() throws Exception { return; } + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + } + JsonObject request = generateUsersJson(10000); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", "http://localhost:3567/bulk-import/users", @@ -401,6 +423,12 @@ public void shouldNormaliseFields() throws Exception { return; } + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + } + JsonObject request = generateUsersJson(1); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", "http://localhost:3567/bulk-import/users", @@ -422,10 +450,10 @@ public void shouldNormaliseFields() throws Exception { for (int i = 0; i < loginMethods.size(); i++) { JsonObject loginMethod = loginMethods.get(i).getAsJsonObject(); if (loginMethod.has("email")) { - assertEquals("johndoe+1@gmail.com", loginMethod.get("hashingAlgorithm").getAsString()); + assertEquals("johndoe+0@gmail.com", loginMethod.get("email").getAsString()); } if (loginMethod.has("phoneNumber")) { - assertEquals("919999999999", loginMethod.get("phoneNumber").getAsString()); + assertEquals("+919999999999", loginMethod.get("phoneNumber").getAsString()); } if (loginMethod.has("hashingAlgorithm")) { assertEquals("ARGON2", loginMethod.get("hashingAlgorithm").getAsString()); @@ -446,8 +474,8 @@ public static JsonObject generateUsersJson(int numberOfUsers) { user.addProperty("externalUserId", UUID.randomUUID().toString()); user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); - user.add("roles", parser.parse("[\"role1\", \"role2\"]")); - user.add("totp", parser.parse("[{\"secretKey\":\"secretKey\",\"period\": 30,\"skew\":1,\"deviceName\":\"deviceName\"}]")); + user.add("userRoles", parser.parse("[\"role1\", \"role2\"]")); + user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"period\": 30,\"skew\":1,\"deviceName\":\"deviceName\"}]")); String email = " johndoe+" + i + "@gmail.com "; From 928563235f1af008b0cbce870cb9169565e0f356 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 28 Feb 2024 10:46:41 +0530 Subject: [PATCH 11/15] fix: PR changes --- .../test/bulkimport/apis/AddBulkImportUsersTest.java | 3 +-- .../test/bulkimport/apis/GetBulkImportUsersTest.java | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 4684b0f92..94ce885b0 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -443,9 +443,8 @@ public void shouldNormaliseFields() throws Exception { JsonArray bulkImportUsers = getResponse.get("users").getAsJsonArray(); assertEquals(1, bulkImportUsers.size()); - JsonParser parser = new JsonParser(); JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); - JsonArray loginMethods = parser.parse(bulkImportUserJson.get("rawData").getAsString()).getAsJsonObject().getAsJsonArray("loginMethods"); + JsonArray loginMethods = bulkImportUserJson.getAsJsonArray("loginMethods"); for (int i = 0; i < loginMethods.size(); i++) { JsonObject loginMethod = loginMethods.get(i).getAsJsonObject(); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java index 77fad9b02..5efb99a58 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -35,6 +35,7 @@ import io.supertokens.ProcessState; import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.Utils; @@ -150,7 +151,7 @@ public void shouldReturn200Response() throws Exception { assertEquals(1, bulkImportUsers.size()); JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); bulkImportUserJson.get("status").getAsString().equals("NEW"); - bulkImportUserJson.get("rawData").getAsString().equals(rawData); + BulkImportUser.fromJson(bulkImportUserJson).toRawData().equals(rawData); process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); From f09cf721d5aa2b8f3c3cd08bd5d78e7b2826e881 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 28 Feb 2024 12:33:56 +0530 Subject: [PATCH 12/15] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 4 +-- .../bulkimport/BulkImportUserUtils.java | 19 +++++++------ .../api/bulkimport/BulkImportAPI.java | 6 ++--- .../test/bulkimport/BulkImportTest.java | 27 ++++++++++--------- .../apis/GetBulkImportUsersTest.java | 2 +- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index f2136064d..1f9e38d06 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -16,7 +16,7 @@ package io.supertokens.bulkimport; -import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; @@ -51,7 +51,7 @@ public static void addUsers(AppIdentifierWithStorage appIdentifierWithStorage, L } public static BulkImportUserPaginationContainer getUsers(AppIdentifierWithStorage appIdentifierWithStorage, - @Nonnull Integer limit, @Nullable BulkImportUserStatus status, @Nullable String paginationToken) + @Nonnull Integer limit, @Nullable BULK_IMPORT_USER_STATUS status, @Nullable String paginationToken) throws StorageQueryException, BulkImportUserPaginationToken.InvalidTokenException { List users; diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 7b1f20a3c..2d3f0003d 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -80,14 +80,7 @@ private static List getParsedUserRoles(JsonObject userData, String[] all // We already know that the jsonUserRoles is an array of non-empty strings, we will normalise each role now List userRoles = new ArrayList<>(); - jsonUserRoles.forEach(role -> { - String normalisedRole = validateAndNormaliseUserRole(role.getAsString(), errors); - if (Arrays.asList(allUserRoles).contains(normalisedRole)) { - userRoles.add(normalisedRole); - } else { - errors.add("Role " + normalisedRole + " does not exist."); - } - }); + jsonUserRoles.forEach(role -> validateAndNormaliseUserRole(role.getAsString(), allUserRoles, errors)); return userRoles; } @@ -198,13 +191,19 @@ private static String validateAndNormaliseExternalUserId(String externalUserId, return externalUserId.trim(); } - private static String validateAndNormaliseUserRole(String role, List errors) { + private static String validateAndNormaliseUserRole(String role, String[] allUserRoles, List errors) { if (role.length() > 255) { errors.add("role " + role + " is too long. Max length is 255."); } // We just trim the role as per the CreateRoleAPI.java - return role.trim(); + String normalisedRole = role.trim(); + + if (!Arrays.asList(allUserRoles).contains(normalisedRole)) { + errors.add("Role " + normalisedRole + " does not exist."); + } + + return normalisedRole; } private static String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index 2f1f5f94a..d6e903bb9 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -30,7 +30,7 @@ import io.supertokens.bulkimport.BulkImportUserUtils; import io.supertokens.multitenancy.exception.BadPermissionException; import io.supertokens.output.Logging; -import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; @@ -69,10 +69,10 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se limit = BulkImport.GET_USERS_DEFAULT_LIMIT; } - BulkImportUserStatus status = null; + BULK_IMPORT_USER_STATUS status = null; if (statusString != null) { try { - status = BulkImportUserStatus.valueOf(statusString); + status = BULK_IMPORT_USER_STATUS.valueOf(statusString); } catch (IllegalArgumentException e) { throw new ServletException(new BadRequestException("Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!")); } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index 4ff949e09..dba270c3b 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -35,7 +35,7 @@ import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; -import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BulkImportUserStatus; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.AppIdentifierWithStorage; @@ -75,7 +75,7 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); BulkImport.addUsers(new AppIdentifierWithStorage(null, null, storage), users); - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, BulkImportUserStatus.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, BULK_IMPORT_USER_STATUS.NEW, null, null); // Verify that all users are present in addedUsers for (BulkImportUser user : users) { @@ -85,8 +85,8 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { .orElse(null); assertNotNull(matchingUser); - assertEquals(BulkImportUserStatus.NEW, matchingUser.status); - assertEquals(user.toString(), matchingUser.toRawData()); + assertEquals(BULK_IMPORT_USER_STATUS.NEW, matchingUser.status); + assertEquals(user.toRawDataForDbStorage(), matchingUser.toRawDataForDbStorage()); } process.kill(); @@ -115,17 +115,17 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { AppIdentifierWithStorage appIdentifierWithStorage = new AppIdentifierWithStorage(null, null, storage); BulkImport.addUsers(appIdentifierWithStorage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BULK_IMPORT_USER_STATUS.NEW, null, null); // Verify that the other properties are same but ids changed for (BulkImportUser user : users) { BulkImportUser matchingUser = addedUsers.stream() - .filter(addedUser -> user.toString().equals(addedUser.toRawData())) + .filter(addedUser -> user.toRawDataForDbStorage().equals(addedUser.toRawDataForDbStorage())) .findFirst() .orElse(null); assertNotNull(matchingUser); - assertEquals(BulkImportUserStatus.NEW, matchingUser.status); + assertEquals(BULK_IMPORT_USER_STATUS.NEW, matchingUser.status); assertFalse(initialIds.contains(matchingUser.id)); } @@ -152,7 +152,7 @@ public void testGetUsersStatusFilter() throws Exception { List users = generateBulkImportUser(10); BulkImport.addUsers(appIdentifierWithStorage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BULK_IMPORT_USER_STATUS.NEW, null, null); assertEquals(10, addedUsers.size()); } @@ -165,12 +165,12 @@ public void testGetUsersStatusFilter() throws Exception { String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); storage.startTransaction(con -> { - storage.updateBulkImportUserStatus_Transaction(appIdentifierWithStorage, con, userIds, BulkImportUserStatus.PROCESSING); + storage.updateBulkImportUserStatus_Transaction(appIdentifierWithStorage, con, userIds, BULK_IMPORT_USER_STATUS.PROCESSING); storage.commitTransaction(con); return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.PROCESSING, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BULK_IMPORT_USER_STATUS.PROCESSING, null, null); assertEquals(10, addedUsers.size()); } @@ -183,12 +183,12 @@ public void testGetUsersStatusFilter() throws Exception { String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); storage.startTransaction(con -> { - storage.updateBulkImportUserStatus_Transaction(appIdentifierWithStorage, con, userIds, BulkImportUserStatus.FAILED); + storage.updateBulkImportUserStatus_Transaction(appIdentifierWithStorage, con, userIds, BULK_IMPORT_USER_STATUS.FAILED); storage.commitTransaction(con); return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BulkImportUserStatus.FAILED, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifierWithStorage, null, BULK_IMPORT_USER_STATUS.FAILED, null, null); assertEquals(10, addedUsers.size()); } @@ -248,7 +248,8 @@ public void randomPaginationTest() throws Exception { BulkImportUser expectedUser = sortedUsers.get(indexIntoUsers); assertEquals(expectedUser.id, actualUser.id); - assertEquals(expectedUser.toString(), actualUser.toString()); + assertEquals(expectedUser.status, actualUser.status); + assertEquals(expectedUser.toRawDataForDbStorage(), actualUser.toRawDataForDbStorage()); indexIntoUsers++; } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java index 5efb99a58..c5c3aeb73 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -151,7 +151,7 @@ public void shouldReturn200Response() throws Exception { assertEquals(1, bulkImportUsers.size()); JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); bulkImportUserJson.get("status").getAsString().equals("NEW"); - BulkImportUser.fromJson(bulkImportUserJson).toRawData().equals(rawData); + BulkImportUser.fromJson(bulkImportUserJson).toRawDataForDbStorage().equals(rawData); process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); From 9967d6d923bebe34c8537def50750a9a1f1d723f Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 28 Feb 2024 13:07:16 +0530 Subject: [PATCH 13/15] fix: PR changes --- .../test/bulkimport/apis/GetBulkImportUsersTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java index c5c3aeb73..8b40f96bf 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -151,7 +151,7 @@ public void shouldReturn200Response() throws Exception { assertEquals(1, bulkImportUsers.size()); JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); bulkImportUserJson.get("status").getAsString().equals("NEW"); - BulkImportUser.fromJson(bulkImportUserJson).toRawDataForDbStorage().equals(rawData); + BulkImportUser.fromTesting_fromJson(bulkImportUserJson).toRawDataForDbStorage().equals(rawData); process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); From 0574bfe55c39055b33158415c9af5f575e646125 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 28 Feb 2024 14:53:51 +0530 Subject: [PATCH 14/15] fix: PR changes --- .../java/io/supertokens/bulkimport/BulkImportUserUtils.java | 6 +++--- .../test/bulkimport/apis/AddBulkImportUsersTest.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 2d3f0003d..e94ce5883 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -70,7 +70,7 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi } private static List getParsedUserRoles(JsonObject userData, String[] allUserRoles, List errors) { - JsonArray jsonUserRoles = parseAndValidateFieldType(userData, "roles", ValueType.ARRAY_OF_STRING, + JsonArray jsonUserRoles = parseAndValidateFieldType(userData, "userRoles", ValueType.ARRAY_OF_STRING, false, JsonArray.class, errors, "."); @@ -80,12 +80,12 @@ private static List getParsedUserRoles(JsonObject userData, String[] all // We already know that the jsonUserRoles is an array of non-empty strings, we will normalise each role now List userRoles = new ArrayList<>(); - jsonUserRoles.forEach(role -> validateAndNormaliseUserRole(role.getAsString(), allUserRoles, errors)); + jsonUserRoles.forEach(role -> userRoles.add(validateAndNormaliseUserRole(role.getAsString(), allUserRoles, errors))); return userRoles; } private static List getParsedTotpDevices(JsonObject userData, List errors) { - JsonArray jsonTotpDevices = parseAndValidateFieldType(userData, "totp", ValueType.ARRAY_OF_OBJECT, false, JsonArray.class, errors, "."); + JsonArray jsonTotpDevices = parseAndValidateFieldType(userData, "totpDevices", ValueType.ARRAY_OF_OBJECT, false, JsonArray.class, errors, "."); if (jsonTotpDevices == null) { return null; } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 94ce885b0..709455605 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -141,7 +141,7 @@ public void shouldThrow400Error() throws Exception { { try { JsonObject request = new JsonParser() - .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"roles\":{},\"totp\":{}}]}") + .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", "http://localhost:3567/bulk-import/users", @@ -150,12 +150,12 @@ public void shouldThrow400Error() throws Exception { String responseString = getResponseMessageFromError(e.getMessage()); assertEquals(400, e.statusCode); assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"roles should be of type array of string.\",\"totp should be of type array of object.\",\"loginMethods is required.\"]}]}"); + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of string.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); } // Invalid role (does not exist) try { JsonObject request = new JsonParser() - .parse("{\"users\":[{\"roles\":[\"role5\"]}]}") + .parse("{\"users\":[{\"userRoles\":[\"role5\"]}]}") .getAsJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", "http://localhost:3567/bulk-import/users", From ab7d2bf62b2bcb98fc2f551cdf5d8472143fc54b Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 28 Feb 2024 15:14:40 +0530 Subject: [PATCH 15/15] fix: Make period and skew optional --- .../bulkimport/BulkImportUserUtils.java | 24 +++++++++++------ .../apis/AddBulkImportUsersTest.java | 26 +++++++++++++++++-- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index e94ce5883..c006c3640 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -95,8 +95,8 @@ private static List getParsedTotpDevices(JsonObject userData, List errors) { - // We don't perform any normalisation on the period in ImportTotpDeviceAPI.java other than checking if it is > 0 - if (period != null && period.intValue() < 1) { + // We default to 30 if period is null + if (period == null) { + return 30; + } + + if (period.intValue() < 1) { errors.add("period should be > 0 for a totp device."); return null; } - return period != null ? period.intValue() : null; + return period; } private static Integer validateAndNormaliseTotpSkew(Integer skew, List errors) { - // We don't perform any normalisation on the period in ImportTotpDeviceAPI.java other than checking if it is >= 0 - if (skew != null && skew.intValue() < 0) { + // We default to 1 if skew is null + if (skew == null) { + return 1; + } + + if (skew.intValue() < 0) { errors.add("skew should be >= 0 for a totp device."); return null; } - return skew != null ? skew.intValue() : null; + return skew; } private static String validateAndNormaliseTotpDeviceName(String deviceName, List errors) { diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 709455605..b3e37cbf3 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -152,6 +152,20 @@ public void shouldThrow400Error() throws Exception { assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of string.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); } + // secretKey is required in totpDevices + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); + } // Invalid role (does not exist) try { JsonObject request = new JsonParser() @@ -444,8 +458,16 @@ public void shouldNormaliseFields() throws Exception { assertEquals(1, bulkImportUsers.size()); JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); - JsonArray loginMethods = bulkImportUserJson.getAsJsonArray("loginMethods"); + // Test if default values were set in totpDevices + JsonArray totpDevices = bulkImportUserJson.getAsJsonArray("totpDevices"); + for (int i = 0; i < totpDevices.size(); i++) { + JsonObject totpDevice = totpDevices.get(i).getAsJsonObject(); + assertEquals(30, totpDevice.get("period").getAsInt()); + assertEquals(1, totpDevice.get("skew").getAsInt()); + } + + JsonArray loginMethods = bulkImportUserJson.getAsJsonArray("loginMethods"); for (int i = 0; i < loginMethods.size(); i++) { JsonObject loginMethod = loginMethods.get(i).getAsJsonObject(); if (loginMethod.has("email")) { @@ -474,7 +496,7 @@ public static JsonObject generateUsersJson(int numberOfUsers) { user.addProperty("externalUserId", UUID.randomUUID().toString()); user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); user.add("userRoles", parser.parse("[\"role1\", \"role2\"]")); - user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"period\": 30,\"skew\":1,\"deviceName\":\"deviceName\"}]")); + user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"deviceName\":\"deviceName\"}]")); String email = " johndoe+" + i + "@gmail.com ";