fix(users): handle external change of user email address by storing p…

…reviously known email addresses and falling back to search by former addresses when necessary

introduce generic ids for users instead of using emails as ids

closes #242

backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/UserRepository.java

@@ -11,13 +11,16 @@
 package org.eclipse.sw360.datahandler.db;
 
 import org.eclipse.sw360.components.summary.UserSummary;
+import org.eclipse.sw360.datahandler.common.CommonUtils;
 import org.eclipse.sw360.datahandler.couchdb.DatabaseConnector;
 import org.eclipse.sw360.datahandler.couchdb.SummaryAwareRepository;
 import org.eclipse.sw360.datahandler.thrift.users.User;
 import org.ektorp.support.View;
+import org.ektorp.support.Views;
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
 /**
  * CRUD access for the User class
@@ -26,7 +29,24 @@
  * @author [email protected]
  */
 
-@View(name = "all", map = "function(doc) { if (doc.type == 'user') emit(null, doc._id) }")
+@Views({
+        @View(name = "all",
+                map = "function(doc) { if (doc.type == 'user') emit(null, doc._id) }"),
+        @View(name = "byExternalId",
+                map = "function(doc) { if (doc.type == 'user') emit(doc.externalid, doc._id) }"),
+        @View(name = "byEmail",
+                map = "function(doc) { " +
+                        "  if (doc.type == 'user') {" +
+                        "    emit(doc.email, doc._id); " +
+                        "    if (doc.formerEmailAddresses && Array.isArray(doc.formerEmailAddresses)) {" +
+                        "      var arr = doc.formerEmailAddresses;" +
+                        "      for (var i = 0; i < arr.length; i++){" +
+                        "        emit(arr[i], doc._id);" +
+                        "      }" +
+                        "    }" +
+                        "  }" +
+                        "}"),
+})
 public class UserRepository extends SummaryAwareRepository<User> {
     public UserRepository(DatabaseConnector databaseConnector) {
         super(User.class, databaseConnector, new UserSummary());
@@ -37,4 +57,18 @@ public UserRepository(DatabaseConnector databaseConnector) {
     public List<User> get(Collection<String> ids) {
         return getConnector().get(User.class, ids, true);
     }
+
+    public User getByExternalId(String externalId) {
+        final Set<String> userIds = queryForIdsAsValue("byExternalId", externalId);
+        if (userIds != null && !userIds.isEmpty())
+            return get(CommonUtils.getFirst(userIds));
+        return null;
+    }
+
+    public User getByEmail(String email) {
+        final Set<String> userIds = queryForIdsAsValue("byEmail", email);
+        if (userIds != null && !userIds.isEmpty())
+            return get(CommonUtils.getFirst(userIds));
+        return null;
+    }
 }

backend/src/src-components/src/test/java/org/eclipse/sw360/components/db/ComponentDatabaseHandlerTest.java

@@ -58,8 +58,8 @@ public class ComponentDatabaseHandlerTest {
     private static final String email1 = "[email protected]";
     private static final String email2 = "[email protected]";
 
-    private static final User user1 = new User().setEmail(email1).setDepartment("AB CD EF").setId(email1);
-    private static final User user2 = new User().setEmail(email2).setDepartment("AB CD EF").setId(email2);
+    private static final User user1 = new User().setEmail(email1).setDepartment("AB CD EF").setId("481489458");
+    private static final User user2 = new User().setEmail(email2).setDepartment("AB CD EF").setId("4786487647680");
 
     @Rule
     public final ExpectedException exception = ExpectedException.none();

backend/src/src-moderation/src/test/java/org/eclipse/sw360/moderation/TestModerationClient.java

@@ -33,7 +33,7 @@ public static void main(String[] args) throws TException, IOException {
         TProtocol protocol = new TCompactProtocol(thriftClient);
         ModerationService.Iface client = new ModerationService.Client(protocol);
 
-        List<ModerationRequest> requestsByModerator = client.getRequestsByModerator(new User().setId("").setEmail("[email protected]").setDepartment("BB"));
+        List<ModerationRequest> requestsByModerator = client.getRequestsByModerator(new User().setId("58245y9845").setEmail("[email protected]").setDepartment("BB"));
 
 
         System.out.println("Fetched " + requestsByModerator.size() + " moderation requests from moderation service");

backend/src/src-users/src/main/java/org/eclipse/sw360/users/UserHandler.java

@@ -10,16 +10,15 @@
  */
 package org.eclipse.sw360.users;
 
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
 import org.eclipse.sw360.datahandler.common.DatabaseSettings;
 import org.eclipse.sw360.datahandler.thrift.RequestStatus;
 import org.eclipse.sw360.datahandler.thrift.users.User;
 import org.eclipse.sw360.datahandler.thrift.users.UserService;
 import org.eclipse.sw360.users.db.UserDatabaseHandler;
-import org.apache.log4j.Logger;
-import org.apache.thrift.TException;
 
 import java.io.IOException;
-import java.net.MalformedURLException;
 import java.util.List;
 
 import static org.eclipse.sw360.datahandler.common.SW360Assert.assertNotEmpty;
@@ -40,17 +39,26 @@ public UserHandler() throws IOException {
         db = new UserDatabaseHandler(DatabaseSettings.getConfiguredHttpClient(), DatabaseSettings.COUCH_DB_USERS);
     }
 
+    @Override
+    public User getUser(String id) throws TException {
+        return db.getUser(id);
+    }
+
     @Override
     public User getByEmail(String email) throws TException {
         StackTraceElement stackTraceElement = Thread.currentThread().getStackTrace()[2];
         assertNotEmpty(email, "Invalid empty email " + stackTraceElement.getFileName() + ": "  + stackTraceElement.getLineNumber());
 
         if (log.isTraceEnabled()) log.trace("getByEmail: " + email);
 
-        // Get user from database
-        User user = db.getByEmail(email);
+        return db.getByEmail(email);
+    }
+
+    @Override
+    public User getByEmailOrExternalId(String email, String externalId) throws TException {
+        User user = getByEmail(email);
         if (user == null) {
-            log.info("User does not exist in DB");
+            user = db.getByExternalId(externalId);
         }
         return user;
     }

backend/src/src-users/src/main/java/org/eclipse/sw360/users/db/UserDatabaseHandler.java

@@ -21,7 +21,6 @@
 import org.ektorp.http.HttpClient;
 
 import java.io.IOException;
-import java.net.MalformedURLException;
 import java.util.List;
 import java.util.function.Supplier;
 
@@ -49,7 +48,11 @@ public UserDatabaseHandler(Supplier<HttpClient> httpClient, String dbName) throw
     }
 
     public User getByEmail(String email) {
-        return db.get(User.class, email);
+        return repository.getByEmail(email);
+    }
+
+    public User getUser(String id) {
+        return db.get(User.class, id);
     }
 
     private void prepareUser(User user) throws SW360Exception {
@@ -87,4 +90,8 @@ public List<User> getAll() {
     public List<User> searchUsers(String searchText) {
         return userSearch.searchByNameAndEmail(searchText);
     }
+
+    public User getByExternalId(String externalId) {
+        return repository.getByExternalId(externalId);
+    }
 }

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/CustomFieldHelper.java

@@ -13,22 +13,20 @@
 
 import com.liferay.portal.kernel.exception.PortalException;
 import com.liferay.portal.kernel.exception.SystemException;
-import com.liferay.portal.kernel.util.WebKeys;
 import com.liferay.portal.model.ResourceConstants;
 import com.liferay.portal.model.Role;
 import com.liferay.portal.model.RoleConstants;
 import com.liferay.portal.security.permission.ActionKeys;
 import com.liferay.portal.service.ResourcePermissionLocalServiceUtil;
 import com.liferay.portal.service.RoleLocalServiceUtil;
-import com.liferay.portal.service.UserLocalServiceUtil;
-import com.liferay.portal.theme.ThemeDisplay;
 import com.liferay.portlet.expando.model.ExpandoBridge;
 import com.liferay.portlet.expando.model.ExpandoColumn;
 import com.liferay.portlet.expando.model.ExpandoColumnConstants;
 import com.liferay.portlet.expando.model.ExpandoTableConstants;
 import com.liferay.portlet.expando.service.ExpandoColumnLocalServiceUtil;
 import org.apache.log4j.Logger;
 import org.eclipse.sw360.datahandler.thrift.users.User;
+import org.eclipse.sw360.portal.users.UserUtils;
 
 import javax.portlet.PortletRequest;
 
@@ -83,19 +81,13 @@ public static <T extends Serializable> Optional<T> loadField(Class<T> type, Port
     }
 
     private static ExpandoBridge getUserExpandoBridge(PortletRequest request, User user) throws PortalException, SystemException {
-        com.liferay.portal.model.User liferayUser = getLiferayUser(request, user);
+        com.liferay.portal.model.User liferayUser = UserUtils.findLiferayUser(request, user);
         ensureUserCustomFieldExists(liferayUser, CUSTOM_FIELD_PROJECT_GROUP_FILTER, ExpandoColumnConstants.STRING);
         ensureUserCustomFieldExists(liferayUser, CUSTOM_FIELD_COMPONENTS_VIEW_SIZE, ExpandoColumnConstants.INTEGER);
         ensureUserCustomFieldExists(liferayUser, CUSTOM_FIELD_VULNERABILITIES_VIEW_SIZE, ExpandoColumnConstants.INTEGER);
         return liferayUser.getExpandoBridge();
     }
 
-    private static com.liferay.portal.model.User getLiferayUser(PortletRequest request, User user) throws PortalException, SystemException {
-        ThemeDisplay themeDisplay = (ThemeDisplay) request.getAttribute(WebKeys.THEME_DISPLAY);
-        long companyId = themeDisplay.getCompanyId();
-        return UserLocalServiceUtil.getUserByEmailAddress(companyId, user.email);
-    }
-
     private static void ensureUserCustomFieldExists(com.liferay.portal.model.User liferayUser, String customFieldName, int customFieldType) throws PortalException, SystemException {
         ExpandoBridge exp = liferayUser.getExpandoBridge();
         if (!exp.hasAttribute(customFieldName)) {

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/admin/UserPortlet.java

@@ -19,14 +19,16 @@
 import com.liferay.portal.kernel.exception.SystemException;
 import com.liferay.portal.kernel.portlet.PortletResponseUtil;
 import com.liferay.portal.kernel.upload.UploadPortletRequest;
-import com.liferay.portal.kernel.util.WebKeys;
 import com.liferay.portal.model.*;
-import com.liferay.portal.model.User;
 import com.liferay.portal.service.*;
-import com.liferay.portal.theme.ThemeDisplay;
 import com.liferay.portal.util.PortalUtil;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
 import org.eclipse.sw360.datahandler.common.CommonUtils;
-import org.eclipse.sw360.datahandler.thrift.SW360Exception;
 import org.eclipse.sw360.datahandler.thrift.users.UserGroup;
 import org.eclipse.sw360.datahandler.thrift.users.UserService;
 import org.eclipse.sw360.portal.common.PortalConstants;
@@ -35,12 +37,6 @@
 import org.eclipse.sw360.portal.users.UserCSV;
 import org.eclipse.sw360.portal.users.UserCacheHolder;
 import org.eclipse.sw360.portal.users.UserUtils;
-import org.apache.commons.csv.CSVFormat;
-import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVPrinter;
-import org.apache.commons.csv.CSVRecord;
-import org.apache.log4j.Logger;
-import org.apache.thrift.TException;
 
 import javax.portlet.*;
 import java.io.*;
@@ -222,19 +218,13 @@ public void backUpUsers(ResourceRequest request, ResourceResponse response) thro
     }
 
     @UsedAsLiferayAction
-    public void updateUsers(ActionRequest request, ActionResponse response) throws PortletException, IOException {
+    public void updateUsers(ActionRequest request, ActionResponse response) throws IOException {
 
-        List<UserCSV> users;
-        try {
-            users = getUsersFromRequest(request, "file");
-        } catch (TException e) {
-            log.error("Error processing csv file", e);
-            users = Collections.emptyList();
-        }
+        List<UserCSV> users = getUsersFromRequest(request, "file");
 
         try {
             createOrganizations(request, users);
-        } catch (SW360Exception | SystemException | PortalException e) {
+        } catch (SystemException | PortalException e) {
             log.error("Error creating organizations", e);
         }
 
@@ -251,7 +241,7 @@ private String extractHeadDept(String input) {
 
     }
 
-    private void createOrganizations(PortletRequest request, List<UserCSV> users) throws SW360Exception, SystemException, PortalException {
+    private void createOrganizations(PortletRequest request, List<UserCSV> users) throws SystemException, PortalException {
 
         /* Find the departments of the users, create the head departments and then create the organizations */
 
@@ -260,13 +250,12 @@ private void createOrganizations(PortletRequest request, List<UserCSV> users) th
         createOrganizations(request, departments);
     }
 
-    public void createOrganizations(PortletRequest request, Iterable<String> departments) throws PortalException, SystemException {
+    private void createOrganizations(PortletRequest request, Iterable<String> departments) throws PortalException, SystemException {
         ImmutableSet<String> headDepartments = FluentIterable.from(departments).transform(department -> extractHeadDept(department)).toSet();
 
         Map<String, Long> organizationIds = new HashMap<>();
         ServiceContext serviceContext = ServiceContextFactory.getInstance(request);
-        ThemeDisplay themeDisplay = (ThemeDisplay) request.getAttribute(WebKeys.THEME_DISPLAY);
-        long companyId = themeDisplay.getCompanyId();
+        long companyId = UserUtils.getCompanyId(request);
         for (String headDepartment : headDepartments) {
 
             long organizationId;
@@ -311,20 +300,7 @@ private Organization createOrganization(ServiceContext serviceContext, String he
         );
     }
 
-    private User getCurrentUser(PortletRequest request) throws SW360Exception {
-        User user;
-        ThemeDisplay themeDisplay = (ThemeDisplay) request.getAttribute(WebKeys.THEME_DISPLAY);
-
-        if (themeDisplay.isSignedIn())
-            user = themeDisplay.getUser();
-        else {
-            throw new SW360Exception("Broken portlet!");
-        }
-        return user;
-    }
-
-
-    private List<UserCSV> getUsersFromRequest(PortletRequest request, String fileUploadFormId) throws IOException, TException {
+    private List<UserCSV> getUsersFromRequest(PortletRequest request, String fileUploadFormId) throws IOException {
 
         final UploadPortletRequest uploadPortletRequest = PortalUtil.getUploadPortletRequest(request);
 
@@ -362,7 +338,7 @@ private User dealWithUser(PortletRequest request, UserCSV userRec) {
         try {
             user = userRec.addLifeRayUser(request);
             if (user != null) {
-                UserUtils.synchronizeUserWithDatabase(userRec, thriftClients, userRec::getEmail, UserUtils::fillThriftUserFromUserCSV);
+                UserUtils.synchronizeUserWithDatabase(userRec, thriftClients, userRec::getEmail, userRec::getGid, UserUtils::fillThriftUserFromUserCSV);
             }
         } catch (SystemException | PortalException e) {
             log.error("Error creating a new user", e);

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/portlets/signup/SignupPortlet.java

@@ -101,7 +101,7 @@ private User createUser(Registrant registrant, PortletRequest request) {
         try {
             com.liferay.portal.model.User liferayUser = registrant.addLifeRayUser(request);
             if (liferayUser != null) {
-                user = UserUtils.synchronizeUserWithDatabase(registrant, thriftClients, registrant::getEmail, UserUtils::fillThriftUserFromThriftUser);
+                user = UserUtils.synchronizeUserWithDatabase(registrant, thriftClients, registrant::getEmail, registrant::getExternalid, UserUtils::fillThriftUserFromThriftUser);
             }
         } catch (PortalException | SystemException e) {
             log.error(e);

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/users/OrganizationHelper.java

@@ -53,7 +53,7 @@ public class OrganizationHelper {
     public void reassignUserToOrganizationIfNecessary(User user, Organization organization) throws PortalException, SystemException {
         if (organization != null && userIsNotInOrganization(user, organization.getOrganizationId())) {
             removeUserFromOtherOrganizations(user);
-            log.debug("OrganizationHelper adds user " + user.getEmailAddress() + " to the organization " + organization.getName());
+            log.info("OrganizationHelper adds user " + user.getEmailAddress() + " to the organization " + organization.getName());
             UserLocalServiceUtil.addOrganizationUsers(organization.getOrganizationId(), Collections.singletonList(user));
         }
     }

frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/users/PortletRequestAdapter.java

@@ -13,13 +13,10 @@
 import com.liferay.portal.kernel.exception.PortalException;
 import com.liferay.portal.kernel.exception.SystemException;
 import com.liferay.portal.kernel.servlet.SessionMessages;
-import com.liferay.portal.kernel.util.WebKeys;
 import com.liferay.portal.service.ServiceContext;
 import com.liferay.portal.service.ServiceContextFactory;
-import com.liferay.portal.theme.ThemeDisplay;
 
 import javax.portlet.PortletRequest;
-import javax.servlet.http.HttpServletRequest;
 import java.util.Optional;
 import java.util.function.Consumer;
 
@@ -33,8 +30,7 @@ class PortletRequestAdapter implements RequestAdapter {
 
     @Override
     public long getCompanyId() {
-        ThemeDisplay themeDisplay = (ThemeDisplay) request.getAttribute(WebKeys.THEME_DISPLAY);
-        return themeDisplay.getCompanyId();
+        return UserUtils.getCompanyId(request);
     }
 
     @Override