diff --git a/.github/workflows/push-to-docker-hub.yml b/.github/workflows/push-to-docker-hub.yml index 5f5b38e0..00caa881 100644 --- a/.github/workflows/push-to-docker-hub.yml +++ b/.github/workflows/push-to-docker-hub.yml @@ -6,6 +6,7 @@ on: - '*' branches: - master + - BR234 jobs: push_to_docker_hub: diff --git a/app/controllers/ApplicationController.java b/app/controllers/ApplicationController.java index c81b2df1..0eb0b934 100644 --- a/app/controllers/ApplicationController.java +++ b/app/controllers/ApplicationController.java @@ -176,12 +176,12 @@ public static Result javascriptRoutes() { @Security.Authenticated(SecuredController.class) public static Result home() { - String email = session().get("email"); - User user = User.findByEmail(email); - if (user.ddhaptUser && getDDHAPTStatus()) - return redirect(routes.WatchedTargets.overview(0, "target.title", "asc")); - else - return redirect(routes.ApplicationController.index()); + //String email = session().get("email"); + //User user = User.findByEmail(email); + //if (user.ddhaptUser && getDDHAPTStatus()) + // return redirect(routes.WatchedTargets.overview(0, "target.title", "asc")); + //else + return redirect(routes.ApplicationController.index()); } public static boolean getDDHAPTStatus(){ diff --git a/app/controllers/CollectionController.java b/app/controllers/CollectionController.java index 88192ad0..0cde51cf 100644 --- a/app/controllers/CollectionController.java +++ b/app/controllers/CollectionController.java @@ -129,6 +129,12 @@ public static Result filterByJson(String name) { return ok(jsonData); } + @Security.Authenticated(SecuredController.class) + @BodyParser.Of(BodyParser.Json.class) + public static Result json() { + return ok(Json.toJson(Collection.findAllCollections())); + } + @Security.Authenticated(SecuredController.class) @BodyParser.Of(BodyParser.Json.class) public static Result getByJson() { @@ -419,6 +425,8 @@ else if (collections.length > 1) { } else if (action.equals("delete")) { Collection collection = Collection.findById(id); flash("message", "Collection " + filledForm.get().name + " has been deleted"); + Ebean.createSqlUpdate("DELETE FROM taxonomy_parents_all where parent_id=:id") + .setParameter("id", id).execute(); collection.delete(); Logger.debug("invalidate cache on Collection Delete, key CollectionsData: "); getCache().remove("CollectionsData"); diff --git a/app/controllers/CrawlPermissionController.java b/app/controllers/CrawlPermissionController.java index a0c27751..11f6c3a3 100644 --- a/app/controllers/CrawlPermissionController.java +++ b/app/controllers/CrawlPermissionController.java @@ -53,8 +53,10 @@ public class CrawlPermissionController extends AbstractController { //final static Form crawlPermissionForm = new Form(CrawlPermission.class); - public static String EMAIL_REGEX = "^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@" - + "[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$"; + /** + * RFC5322 + */ + public static String EMAIL_REGEX = "^[a-zA-Z0-9_!#$%&'*+/=?`{|}~^.-]+@[a-zA-Z0-9.-]+$"; /** * Display the crawl permissions. @@ -917,6 +919,12 @@ public static Result send() throws ActException { int pageNo = Integer.parseInt(requestData.get("pageNo")); Logger.debug("pageNo: " + pageNo); + String sortBy = requestData.get("sortByValue"); + Logger.debug("sortByValue: " + sortBy); + + String orderBy = requestData.get("orderByValue"); + Logger.debug("orderByValue: " + orderBy); + Map formParams = request().body().asFormUrlEncoded(); String[] permissionValues = formParams.get("permissionsList"); @@ -1000,12 +1008,12 @@ public static Result send() throws ActException { if (action.equals("selectall")) { Logger.debug("select all listed in page crawl permissions"); res = redirect(routes.CrawlPermissionController.list( - pageNo, Const.NAME, Const.ASC, "", status, organisation, Const.SELECT_ALL)); + pageNo, sortBy, orderBy, "", status, organisation, Const.SELECT_ALL)); } if (action.equals("deselectall")) { Logger.debug("deselect all listed in page crawl permissions"); res = redirect(routes.CrawlPermissionController.list( - pageNo, Const.NAME, Const.ASC, "", status, organisation, Const.DESELECT_ALL)); + pageNo, sortBy, orderBy, "", status, organisation, Const.DESELECT_ALL)); } } return res; diff --git a/app/controllers/ReportController.java b/app/controllers/ReportController.java index 993dce8e..3ced63a2 100644 --- a/app/controllers/ReportController.java +++ b/app/controllers/ReportController.java @@ -2,13 +2,8 @@ import static play.data.Form.form; -import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.TimeZone; +import java.util.*; import models.CrawlPermission; import models.License.LicenseStatus; @@ -22,21 +17,17 @@ import play.Logger; import play.data.DynamicForm; -import play.db.ebean.Transactional; +import play.libs.Json; import play.mvc.Result; import play.mvc.Security; import uk.bl.Const; import uk.bl.Const.CrawlFrequency; import uk.bl.Const.CrawlPermissionStatus; import uk.bl.Const.NpldType; -import uk.bl.Const.RequestType; import uk.bl.Const.ScopeType; -import uk.bl.api.Utils; import uk.bl.exception.ActException; import views.html.reports.*; -import com.avaje.ebean.Ebean; -import com.avaje.ebean.ExpressionList; import com.avaje.ebean.Page; /** @@ -50,122 +41,137 @@ public class ReportController extends AbstractController { * @throws ActException */ public static Result index() throws ActException { - List resList = processFilterReports(null, null, Const.CrawlPermissionStatus.PENDING.name(),"","","","",""); - List resListGranted = processFilterReports(null, null, Const.CrawlPermissionStatus.GRANTED.name(),"","","","",""); - List resListRefused = processFilterReports(null, null, Const.CrawlPermissionStatus.REFUSED.name(),"","","","",""); - User user = User.findByEmail(request().username()); - List users = User.findAll(); - List organisations = Organisation.findAllSorted(); - RequestType[] requestTypes = Const.RequestType.values(); - - return ok( - reports.render( - "Reports", user, resList, resListGranted, resListRefused, null, null, "", "", "", "", "", users, organisations, requestTypes - ) - ); - } + return redirect(routes.ReportController.processFilterReports( + 0,-1L, -1L, + CrawlPermissionStatus.PENDING.name(), //default filter + "","","","")); + }; /** * This method enables searching for given URL and redirection in order to add new entry * if required. + * Helper for Targets -> Reports search/filter form * @return * @throws ActException */ public static Result search() throws ActException { - DynamicForm requestData = form().bindFromRequest(); - String action = requestData.get("action"); + DynamicForm requestData = form().bindFromRequest(); + String crawlPermissionsStatus22 = form().bindFromRequest().get("crawlPermissionsStatus22"); + String action = requestData.get("action"); + Logger.debug("action: " + action); + String curator = requestData.get("curator"); + String organisation = requestData.get("organisation"); + Long curatorId = null; + if (StringUtils.isNotBlank(curator)) { + curatorId = Long.parseLong(curator); + } + Long organisationId = null; + if (StringUtils.isNotBlank(organisation)) { + organisationId = Long.parseLong(organisation); + } + String generalFromDate = requestData.get("startDate"); + String generalToDate = requestData.get("endDate"); - Logger.debug("action: " + action); - String curator = requestData.get("curator"); - String organisation = requestData.get("organisation"); - Long curatorId = null; - if (StringUtils.isNotBlank(curator)) { - curatorId = Long.parseLong(curator); - } - Long organisationId = null; - if (StringUtils.isNotBlank(organisation)) { - organisationId = Long.parseLong(organisation); - } - - String request = requestData.get("request"); -// if (request_name != null && !request_name.toLowerCase().equals(Const.NONE) -// && !request_name.toLowerCase().equals(Const.ALL)) { -// request = request_name; -// } - String requestedFromDate = requestData.get("startDate"); - String requestedToDate = requestData.get("endDate"); - String grantedFromDate = requestData.get("grantedFromDate"); - String grantedToDate = requestData.get("grantedToDate"); - Logger.debug("Parameters: " + curatorId + ", " + organisationId + ", " + requestedFromDate + ", " + requestedToDate + ", " + grantedFromDate + ", " + grantedToDate); - - List resListRequest = processFilterReports(curatorId, organisationId, Const.CrawlPermissionStatus.PENDING.name(), request, requestedFromDate, requestedToDate, grantedFromDate, grantedToDate); - List resListGranted = processFilterReports(curatorId, organisationId, Const.CrawlPermissionStatus.GRANTED.name(), request, requestedFromDate, requestedToDate, grantedFromDate, grantedToDate); - List resListRefused = processFilterReports(curatorId, organisationId, Const.CrawlPermissionStatus.REFUSED.name(), request, requestedFromDate, requestedToDate, grantedFromDate, grantedToDate); - - Logger.debug("resListRequest: " + resListRequest); - Logger.debug("resListRequest"); -// Logger.debug("resListGranted: " + resListGranted); -// Logger.debug("resListGranted"); -// Logger.debug("resListRefused: " + resListRefused); -// Logger.debug("resListRefused"); - - List users = User.findAll(); -// Logger.debug("users: " + users); - List organisations = Organisation.findAllSorted(); -// Logger.debug("organisations: " + organisations); - RequestType[] requestTypes = Const.RequestType.values(); + //String requestedFromDate = requestData.get("startDate"); + //String requestedToDate = requestData.get("endDate"); + String grantedFromDate = requestData.get("grantedFromDate"); + String grantedToDate = requestData.get("grantedToDate"); + Logger.debug("Parameters: " + curatorId + ", " + organisationId + ", " + crawlPermissionsStatus22 + ", " + generalFromDate + ", " + generalToDate + ", " + grantedFromDate + ", " + grantedToDate); + + List users = User.findAll(); + List organisations = Organisation.findAllSorted(); + + User user = User.findByEmail(request().username()); + + String sort = requestData.get("s"); + String order = requestData.get("o"); + //Long curatorId = Long.parseLong(requestData.get("curator")); + //Long organisationId = Long.parseLong(requestData.get("organisation")); + + // default + String searchByCrawlPermissionsStatus = (crawlPermissionsStatus22 != null && !crawlPermissionsStatus22.trim().isEmpty()) ? crawlPermissionsStatus22 : CrawlPermissionStatus.PENDING.name(); + if (action.equals("searchRequested")) { + searchByCrawlPermissionsStatus = CrawlPermissionStatus.PENDING.name(); + } else if (action.equals("searchGranted")) { + searchByCrawlPermissionsStatus = CrawlPermissionStatus.GRANTED.name(); + } else if (action.equals("searchRefused")) { + searchByCrawlPermissionsStatus = CrawlPermissionStatus.REFUSED.name(); + } else { + searchByCrawlPermissionsStatus = crawlPermissionsStatus22; + } + + Page pages = Target.pageReports(0, curatorId, organisationId, + searchByCrawlPermissionsStatus, + generalFromDate, generalToDate, + grantedFromDate, grantedToDate); + String status = searchByCrawlPermissionsStatus; + if (StringUtils.isEmpty(action)) { + return badRequest("You must provide a valid action"); + } + else { + if (action.equals("search") || action.equals("searchRequested") || action.equals("searchGranted") || action.equals("searchRefused")) { + Logger.debug("action : search+"); + return ok( + reports.render( + "Reports", user, + pages, + sort, order, + searchByCrawlPermissionsStatus, + curatorId, organisationId, + generalFromDate, generalToDate, + //requestedFromDate, requestedToDate, + grantedFromDate, grantedToDate, + users, organisations//, requestTypes + ) + ); + } + else if (action.equals("exportcsv")) { + Logger.debug("export CSV: " + status); + if (StringUtils.isNotEmpty(status)) { + Logger.debug("export requested size: " + pages.getList().size()); + List pagesFull = Target.pageReportsFull(0, curatorId, organisationId, + searchByCrawlPermissionsStatus, + generalFromDate, generalToDate, + grantedFromDate, grantedToDate); + String file = exportCrawlPermissionsCsv(pagesFull, searchByCrawlPermissionsStatus); + response().setContentType("text/csv; charset=utf-8"); + response().setHeader("Content-disposition", "attachment; filename=\"" + searchByCrawlPermissionsStatus + "_" + (generalFromDate.equals("") ? "ALL" : generalFromDate) + "-" + (generalToDate.equals("") ? "ALL" : generalToDate) + "_" + Const.EXPORT_LICENCE_FILE_CSV + "\""); + return ok(file); + } + Logger.debug("returning in export CSV"); + } + else if (action.equals("exporttsv")) { + Logger.debug("export TSV: " + status); + if (StringUtils.isNotEmpty(status)) { + Logger.debug("export TSV size: " + pages.getList().size()); + List pagesFull = Target.pageReportsFull(0, curatorId, organisationId, + searchByCrawlPermissionsStatus, + generalFromDate, generalToDate, + grantedFromDate, grantedToDate); + String file = exportCrawlPermissionsTsv(pagesFull, searchByCrawlPermissionsStatus); + response().setContentType("text/tsv; charset=utf-8"); + response().setHeader("Content-disposition", "attachment; filename=\"" + searchByCrawlPermissionsStatus + "_" + (generalFromDate.equals("") ? "ALL" : generalFromDate) + "-" + (generalToDate.equals("") ? "ALL" : generalToDate) + "_" + Const.EXPORT_LICENCE_FILE_TSV + "\""); + return ok(file); + } + } + else if (action.equals("exportjson")) { + Logger.debug("export JSON: " + status); + if (StringUtils.isNotEmpty(status)) { + Logger.debug("export JSON size: " + pages.getList().size()); + List pagesFull = Target.pageReportsFull(0, curatorId, organisationId, + searchByCrawlPermissionsStatus, + generalFromDate, generalToDate, + grantedFromDate, grantedToDate); + response().setContentType("application/json; charset=utf-8"); + response().setHeader("Content-disposition", "attachment; filename=\"" + searchByCrawlPermissionsStatus + "_" + (generalFromDate.equals("") ? "ALL" : generalFromDate) + "-" + (generalToDate.equals("") ? "ALL" : generalToDate) + "_" + Const.EXPORT_LICENCE_FILE_JSON + "\""); + return ok(Json.toJson(exportCrawlPermissionsJson(pagesFull, searchByCrawlPermissionsStatus))); + } + Logger.debug("returning in export JSON"); + } + } + return badRequest("This action is not allowed"); + } - User user = User.findByEmail(request().username()); - if (StringUtils.isEmpty(action)) { - return badRequest("You must provide a valid action"); - } else { - if (action.equals("search")) { - return ok( - reports.render( - "Reports", user, resListRequest, resListGranted, - resListRefused, curatorId, organisationId, requestedFromDate, requestedToDate, request, grantedFromDate, grantedToDate, users, organisations, requestTypes - ) - ); - } else if (action.equals("export")) { - String status = requestData.get("status"); - Logger.debug("in export: " + status); - - - if (StringUtils.isNotEmpty(status)) { - Logger.debug("status: " + status); - if (status.equals("tabRequested")) { - Logger.debug("export requested size: " + resListRequest.size()); - String file = export(resListRequest); - response().setContentType("text/csv; charset=utf-8"); - response().setHeader("Content-disposition","attachment; filename=\"" + Const.EXPORT_REQUESTED_LICENCE_FILE + "\""); - return ok(file); - } else if (status.equals("tabGranted")) { - Logger.debug("export refused size: " + resListGranted.size()); - String file = export(resListGranted); - response().setContentType("text/csv; charset=utf-8"); - response().setHeader("Content-disposition","attachment; filename=\"" + Const.EXPORT_GRANTED_LICENCE_FILE + "\""); - return ok(file); - - } else if (status.equals("tabRefused")) { - Logger.debug("export refused size: " + resListRefused.size()); - String file = export(resListRefused); - response().setContentType("text/csv; charset=utf-8"); - response().setHeader("Content-disposition","attachment; filename=\"" + Const.EXPORT_REFUSED_LICENCE_FILE + "\""); - return ok(file); - } - return ok( - reports.render( - "Reports", user, resListRequest, resListGranted, - resListRefused, curatorId, organisationId, requestedFromDate, requestedToDate, request, grantedFromDate, grantedToDate, users, organisations, requestTypes - ) - ); - } - Logger.debug("returning in export"); - - } - return badRequest("This action is not allowed"); - } - } // public static void exportLicenses(String name) { // return ok( @@ -182,38 +188,144 @@ public static Result search() throws ActException { * @param file name * @return */ - public static String export(List permissionList) { -// public static void export(List permissionList, String fileName) { - Logger.debug("export() permissionList size: " + permissionList.size()); - -// StringWriter sw = new StringWriter(); - StringBuilder sw = new StringBuilder(); - sw.append("Target title"); + public static String export(List permissionList) { + Logger.debug("export() permissionList size: " + permissionList.size()); + StringBuilder sw = new StringBuilder(); + sw.append("Target title"); sw.append(Const.CSV_SEPARATOR); - sw.append("Target URL"); + sw.append("Target URL"); sw.append(Const.CSV_SEPARATOR); - sw.append("Date requested"); + sw.append("Date requested"); sw.append(Const.CSV_SEPARATOR); - sw.append(Const.CSV_LINE_END); - - if (permissionList != null && permissionList.size() > 0) { - Iterator itr = permissionList.iterator(); - while (itr.hasNext()) { - Target target = itr.next(); - sw.append(StringEscapeUtils.escapeCsv(target.title)); - sw.append(Const.CSV_SEPARATOR); - sw.append(StringEscapeUtils.escapeCsv(target.fieldUrl())); - sw.append(Const.CSV_SEPARATOR); - sw.append(StringEscapeUtils.escapeCsv(target.createdAt.toString())); - sw.append(Const.CSV_SEPARATOR); - sw.append(Const.CSV_LINE_END); - } - } + sw.append(Const.CSV_LINE_END); + + if (permissionList != null && permissionList.size() > 0) { + Iterator itr = permissionList.iterator(); + while (itr.hasNext()) { + Target target = itr.next(); + sw.append(StringEscapeUtils.escapeCsv(target.title)); + sw.append(Const.CSV_SEPARATOR); + sw.append(StringEscapeUtils.escapeCsv(target.fieldUrl())); + sw.append(Const.CSV_SEPARATOR); + sw.append(StringEscapeUtils.escapeCsv(target.createdAt.toString())); + sw.append(Const.CSV_SEPARATOR); + sw.append(Const.CSV_LINE_END); + } + } // File file = Utils.INSTANCE.generateCsvFile(fileName, sw.toString()); - return sw.toString(); - } - - /** + return sw.toString(); + } + + /** + * Exports CrawlPermissions in CSV format + * @param permissionList + * @param crawlPermissionStatus + * @return + */ + public static String exportCrawlPermissionsCsv(List permissionList, String crawlPermissionStatus) { + StringBuilder sw = new StringBuilder(); + sw.append("Target title"); + sw.append(Const.CSV_SEPARATOR); + sw.append("Target URL"); + sw.append(Const.CSV_SEPARATOR); + sw.append("Date requested"); + sw.append(Const.CSV_SEPARATOR); + if (crawlPermissionStatus.equals("GRANTED")) { + sw.append("Date granted"); + //sw.append(Const.CSV_SEPARATOR); + } + else if (crawlPermissionStatus.equals("REFUSED")) { + sw.append("Date refused"); + //sw.append(Const.CSV_SEPARATOR); + } + sw.append(Const.CSV_LINE_END); + if (permissionList != null && permissionList.size() > 0) { + Iterator itr = permissionList.iterator(); + while (itr.hasNext()) { + Target target = itr.next(); + sw.append(StringEscapeUtils.escapeCsv(target.title)); + sw.append(Const.CSV_SEPARATOR); + sw.append(StringEscapeUtils.escapeCsv(target.fieldUrl())); + sw.append(Const.CSV_SEPARATOR); + sw.append(StringEscapeUtils.escapeCsv(target.crawlPermissions.get(0).requestedAt!=null?(target.crawlPermissions.get(0).requestedAt.toString()):"")); + sw.append(Const.CSV_SEPARATOR); + if (crawlPermissionStatus.equals("GRANTED")) + sw.append(StringEscapeUtils.escapeCsv(target.crawlPermissions.get(0).grantedAt!=null?(target.crawlPermissions.get(0).grantedAt.toString()):"")); + else if (crawlPermissionStatus.equals("REFUSED")) + sw.append(StringEscapeUtils.escapeCsv(target.isRefused()==true?"":"")); + sw.append(Const.CSV_LINE_END); + } + } + return sw.toString(); + } + + /** + * exports CrawlPermissions in TSV format + * @param permissionList + * @param crawlPermissionStatus + * @return + */ + public static String exportCrawlPermissionsTsv(List permissionList, String crawlPermissionStatus) { + StringBuilder sw = new StringBuilder(); + sw.append("Target title"); + sw.append(Const.TSV_SEPARATOR); + sw.append("Target URL"); + sw.append(Const.TSV_SEPARATOR); + sw.append("Date requested"); + sw.append(Const.TSV_SEPARATOR); + if (crawlPermissionStatus.equals("GRANTED")) + sw.append("Date granted"); + else if (crawlPermissionStatus.equals("REFUSED")) + sw.append("Date refused"); + sw.append(Const.CSV_LINE_END); //same as TSV + if (permissionList != null && permissionList.size() > 0) { + Iterator itr = permissionList.iterator(); + while (itr.hasNext()) { + Target target = itr.next(); + sw.append(StringEscapeUtils.escapeCsv(target.title)); + sw.append(Const.TSV_SEPARATOR); + sw.append(StringEscapeUtils.escapeCsv(target.fieldUrl())); + sw.append(Const.TSV_SEPARATOR); + sw.append(StringEscapeUtils.escapeCsv(target.crawlPermissions.get(0).requestedAt!=null?(target.crawlPermissions.get(0).requestedAt.toString()):"")); + sw.append(Const.TSV_SEPARATOR); + if (crawlPermissionStatus.equals("GRANTED")) + sw.append(StringEscapeUtils.escapeCsv(target.crawlPermissions.get(0).grantedAt!=null?(target.crawlPermissions.get(0).grantedAt.toString()):"")); + else if (crawlPermissionStatus.equals("REFUSED")) + sw.append(StringEscapeUtils.escapeCsv(target.isRefused()==true?"":"")); + sw.append(Const.CSV_LINE_END); + } + } + return sw.toString(); + } + + /** + * export Crawl Permissions in List> for export in Json + * @param permissionList + * @param crawlPermissionStatus + * @return + */ + public static List> exportCrawlPermissionsJson(List permissionList, String crawlPermissionStatus) { + Map reportMap; + List> reportListOfMaps = new ArrayList<>(); + if (permissionList != null && permissionList.size() > 0) { + Iterator itr = permissionList.iterator(); + while (itr.hasNext()) { + Target target = itr.next(); + reportMap = new HashMap(); + reportMap.put("TargetTitle", StringEscapeUtils.escapeCsv(target.title)); + reportMap.put("TargetURL", StringEscapeUtils.escapeCsv(target.fieldUrl())); + reportMap.put("DateRequested", target.crawlPermissions.get(0).requestedAt!=null?(target.crawlPermissions.get(0).requestedAt.toString()):""); + if (crawlPermissionStatus.equals("GRANTED")) + reportMap.put("DateGranted", target.crawlPermissions.get(0).grantedAt!=null?(target.crawlPermissions.get(0).grantedAt.toString()):""); + else if (crawlPermissionStatus.equals("REFUSED")) + reportMap.put("DateRefused", target.isRefused()==true?"":""); + reportListOfMaps.add(reportMap); + } + } + return reportListOfMaps; + } + + /** * This method applies filters to the list of crawl reports. * @param curator The curator URL * @param organisation The organisation URL @@ -222,104 +334,57 @@ public static String export(List permissionList) { * @return * @throws ActException */ - public static List processFilterReports(Long curatorId, Long organisationId, - String crawlPermissionsStatus, String request, String requestedFromDate, String requestedToDate, String grantedFromDate, String grantedToDate) throws ActException { -// boolean isProcessed = false; - ExpressionList exp = Target.find.where(); - exp = exp.eq("active", true); - - + + //List resListRequest = processFilterReports(null, null, Const.CrawlPermissionStatus.PENDING.name(),"","","","",""); + + /** + * Process and Filter Reports + * @param pageNo + * @param curatorId + * @param organisationId + * @param crawlPermissionsStatus + * @param request + * @param requestedFromDate + * @param requestedToDate + * @param grantedFromDate + * @param grantedToDate + * @return + * @throws ActException + */ + public static Result processFilterReports(int pageNo, Long curatorId, Long organisationId, + String crawlPermissionsStatus, //String request, + String requestedFromDate, + String requestedToDate, + String grantedFromDate, + String grantedToDate) throws ActException { + User user = User.findByEmail(request().username()); + List users = User.findAll(); + List organisations = Organisation.findAllSorted(); + //RequestType[] requestTypes = Const.RequestType.values(); + Logger.debug("pageNo: " + pageNo); Logger.debug("curatorId: " + curatorId); Logger.debug("organisationId: " + organisationId); Logger.debug("crawlPermissionsStatus: " + crawlPermissionsStatus); - Logger.debug("request: " + request); + //Logger.debug("request: " + request); Logger.debug("requestedFromDate: " + requestedFromDate); Logger.debug("requestedToDate: " + requestedToDate); Logger.debug("grantedFromDate: " + grantedFromDate); Logger.debug("grantedToDate: " + grantedToDate); - - if (curatorId != null) { - exp = exp.eq("authorUser.id", curatorId); - } - if (organisationId != null) { - exp = exp.eq("organisation.id", organisationId); - } - if (StringUtils.isNotEmpty(crawlPermissionsStatus)) { - exp = exp.eq("crawlPermissions.status", crawlPermissionsStatus); - } - if (StringUtils.isNotEmpty(requestedFromDate)) { - try { - Date date = Utils.INSTANCE.convertDate(requestedFromDate); - exp = exp.ge("crawlPermissions.requestedAt", date); - } catch (ParseException e) { - throw new ActException(e); - } - } - - if (StringUtils.isNotEmpty(requestedToDate)) { - try { - String modRequestedToDate = Utils.INSTANCE.getaddDayToDate(requestedToDate); - Date date = Utils.INSTANCE.convertDate(modRequestedToDate); - exp = exp.le("crawlPermissions.requestedAt", date); - } catch (ParseException e) { - throw new ActException(e); - } - } - - if (StringUtils.isNotEmpty(grantedFromDate)) { - try { - Date date = Utils.INSTANCE.convertDate(grantedFromDate); - exp = exp.ge("crawlPermissions.grantedAt", date); - } catch (ParseException e) { - throw new ActException(e); - } - } - - if (StringUtils.isNotEmpty(grantedToDate)) { - try { - String modGrantedToDate = Utils.INSTANCE.getaddDayToDate(grantedToDate); - Date date = Utils.INSTANCE.convertDate(modGrantedToDate); - exp = exp.le("crawlPermissions.grantedAt", date); - } catch (ParseException e) { - throw new ActException(e); - } - } - - Logger.debug("exp: " + exp.toString()); - - List res = exp.query().findList(); - - Logger.debug("res size: " + res.size()); -// Logger.debug("processFilterReports() Expression list size: " + res.size() + ", isProcessed: " + isProcessed); -// if (request != null && !request.toLowerCase().equals(Const.ALL) && request.length() > 0) { -// Logger.debug("request: " + request); -// List resByRequest = new ArrayList(); -// Iterator resIter = res.iterator(); -// while (resIter.hasNext()) { -// Target target = resIter.next(); -// if (target.fieldUrl() != null && target.fieldUrl().length() > 0) { -// List permissionList = CrawlPermission.filterByTarget(target.fieldUrl()); -// if (permissionList != null && permissionList.size() > 0) { -// CrawlPermission permission = permissionList.get(0); -// Logger.debug("permission: " + permission); -// Logger.debug("permission requestFollowup: " + permission.requestFollowup + ", request: " + request); -// try { -// if (permission.requestFollowup && request.equals(Const.RequestTypes.FOLLOW_UP.name()) -// || !permission.requestFollowup && request.equals(Const.RequestTypes.FIRST_REQUEST.name())) { -// resByRequest.add(target); -// } -// } catch (Exception e) { -// if (request.equals(Const.RequestTypes.FIRST_REQUEST.name())) { -// resByRequest.add(target); -// } -// } -// } -// } -// } -// return resByRequest; -// } - return res; + Page pages = Target.pageReports(pageNo, curatorId, organisationId, + crawlPermissionsStatus, + requestedFromDate, requestedToDate, grantedFromDate, grantedToDate); + + return ok(reports.render( + "Reports", user, + pages, + "", "", + crawlPermissionsStatus, + curatorId, organisationId, + requestedFromDate, requestedToDate, grantedFromDate, grantedToDate, + users, organisations + //, requestTypes + )); } /** @@ -334,7 +399,7 @@ public static Result summary() { public static Result openLicences() { return redirect( routes.ReportController.index() - ); + ); } public static Result recordCreation() { @@ -368,9 +433,9 @@ public static Result targets(int pageNo, String sortBy, String order, Long curat Logger.debug("ReportsCreation.targets()"); User user = User.findByEmail(request().username()); - Page pages = Target.pageReportsCreation(pageNo, 10, sortBy, order, curatorId, organisationId, + + Page pages = Target.pageReportsCreation(pageNo, Const.PAGINATION_OFFSET, sortBy, order, curatorId, organisationId, startDate, endDate, npld, crawlFrequency, tld); - List users = User.findAll(); List organisations = Organisation.findAll(); @@ -397,18 +462,15 @@ public static Result targets(int pageNo, String sortBy, String order, Long curat nplds) ); } - - /** + + /** * This method enables searching for given URL and redirection in order to add new entry * if required. * @return */ public static Result searchCreation() throws ActException { - DynamicForm requestData = form().bindFromRequest(); - + DynamicForm requestData = form().bindFromRequest(); String action = requestData.get("action"); - - int pageNo = Integer.parseInt(requestData.get("p")); String sort = requestData.get("s"); String order = requestData.get("o"); Long curatorId = Long.parseLong(requestData.get("curator")); @@ -441,11 +503,11 @@ public static Result searchCreation() throws ActException { } else { if (action.equals("export")) { List exportTargets = new ArrayList(); - Page page = Target.pageReportsCreation(pageNo, 10, sort, order, curatorId, organisationId, startDate, endDate, npld, crawlFrequencyName, tld); + Page page = Target.pageReportsCreation(0, Const.PAGINATION_OFFSET, sort, order, curatorId, organisationId, startDate, endDate, npld, crawlFrequencyName, tld); int rowCount = page.getTotalRowCount(); - Page pageAll = Target.pageReportsCreation(pageNo, rowCount, sort, order, curatorId, organisationId, startDate, endDate, npld, crawlFrequencyName, tld); + Page pageAll = Target.pageReportsCreation(0, rowCount, sort, order, curatorId, organisationId, startDate, endDate, npld, crawlFrequencyName, tld); exportTargets.addAll(pageAll.getList()); Logger.debug("export report creation size: " + exportTargets.size()); String file = export(exportTargets); @@ -453,7 +515,7 @@ public static Result searchCreation() throws ActException { response().setHeader("Content-disposition","attachment; filename=\"" + Const.EXPORT_TARGETS_REPORT_CREATION + "\""); return ok(file); } else if (action.equals("search")) { - return redirect(routes.ReportController.targets(pageNo, sort, order, curatorId, organisationId, + return redirect(routes.ReportController.targets(0, sort, order, curatorId, organisationId, startDate, endDate, npld, crawlFrequencyName, tld)); } else { return badRequest("This action is not allowed"); @@ -465,7 +527,7 @@ public static Result searchCreation() throws ActException { * Display the report. */ public static Result indexCreation() { - return redirect(routes.ReportController.targets(0, "createdAt", "desc", -1l, -1l, + return redirect(routes.ReportController.targets(0, "createdAt", "desc", -1l, -1l, "", "", "", "", "either")); } diff --git a/app/controllers/SubjectController.java b/app/controllers/SubjectController.java index d1fbdf86..7fb88164 100644 --- a/app/controllers/SubjectController.java +++ b/app/controllers/SubjectController.java @@ -101,18 +101,26 @@ public static Result search() { } } + @Security.Authenticated(SecuredController.class) @BodyParser.Of(BodyParser.Json.class) - public static Result filterByJson(String name) { - JsonNode jsonData = null; - if (name != null) { - List subjects = Taxonomy.filterSubjectsByName(name); - jsonData = Json.toJson(subjects); - } - return ok(jsonData); - } - - - public static Result view(Long id) { + public static Result filterByJson(String name) { + JsonNode jsonData = null; + if (name != null) { + List subjects = Taxonomy.filterSubjectsByName(name); + jsonData = Json.toJson(subjects); + } + return ok(jsonData); + } + + @Security.Authenticated(SecuredController.class) + @BodyParser.Of(BodyParser.Json.class) + public static Result json() { + return ok(Json.toJson(Subject.findAllSubjects())); + } + + + + public static Result view(Long id) { Subject subject = Subject.findById(id); if (subject == null) return notFound("There is no Subject with ID " + id); diff --git a/app/controllers/TargetController.java b/app/controllers/TargetController.java index 6e160b7b..6a6cd5f4 100644 --- a/app/controllers/TargetController.java +++ b/app/controllers/TargetController.java @@ -402,26 +402,36 @@ public static Result searchTargets() { return GO_HOME; } else { - if(action.equals("export")) { + if(action.equals("exportcsv")) { List exportTargets = new ArrayList(); -// Page page = Target.pageTargets(0, pageSize, sort, order, query, curator, organisation, subject, crawlFrequency, depth, collection, license, flag); - // TODO: no time to do it now but this needs redoing with straight SQL/JPA to get the count - Page page = Target.pageTargets(pageNo, pageSize, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, flagId); - int rowCount = page.getTotalRowCount(); -// Page pageAll = Target.pageTargets(0, rowCount, sort, order, query, curator, organisation, subject, crawlFrequency, depth, collection, license, flag); + int rowCount = Target.filteredTargetListCount(filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, flagId); Page pageAll = Target.pageTargets(pageNo, rowCount, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, flagId); exportTargets.addAll(pageAll.getList()); - Logger.debug("export size: " + exportTargets.size()); - String csvData = Utils.INSTANCE.export(exportTargets); -// return redirect(routes.TargetController.list(pageNo, sort, order, query, curator, organisation, -// subject, crawlFrequency, depth, collection, license, pageSize, flag)); - + Logger.debug("export size for CSV: " + exportTargets.size()); + String csvData = Utils.INSTANCE.exportCsv(exportTargets); response().setContentType("text/csv; charset=utf-8"); response().setHeader("Content-disposition", "attachment; filename=\"target-export.csv"); return ok(csvData); -// return redirect(routes.TargetController.list(pageNo, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, pageSize, flagId)); - } - else { + } else if (action.equals("exporttsv")) { + List exportTargets = new ArrayList(); + int rowCount = Target.filteredTargetListCount(filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, flagId); + Page pageAll = Target.pageTargets(pageNo, rowCount, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, flagId); + exportTargets.addAll(pageAll.getList()); + Logger.debug("export size for TSV: " + exportTargets.size()); + String tsvData = Utils.INSTANCE.exportTsv(exportTargets); + response().setContentType("text/tsv; charset=utf-8"); + response().setHeader("Content-disposition", "attachment; filename=\"target-export.tsv"); + return ok(tsvData); + } else if (action.equals("exportjson")) { + List exportTargets = new ArrayList(); + int rowCount = Target.filteredTargetListCount(filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, flagId); + Logger.debug("export size (for JSON): " + rowCount); + Page pageAll = Target.pageTargets(pageNo, rowCount, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, flagId); + exportTargets.addAll(pageAll.getList()); + response().setContentType("application/json; charset=utf-8"); + response().setHeader("Content-disposition", "attachment; filename=\"target-export.json"); + return ok(Json.toJson(Utils.INSTANCE.exportJson(exportTargets))); + } else { if(action.equals("search") || action.equals("apply")) { return redirect(routes.TargetController.list(pageNo, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, pageSize, flagId)); } @@ -441,25 +451,19 @@ public static Result searchTargets() { * @return */ public static Result search() { - DynamicForm form = DynamicForm.form().bindFromRequest(); - String action = form.get("action"); String query = form.get("filter"); int pageNo = Integer.parseInt(form.get("p")); String sort = form.get("s"); String order = form.get("o"); - if(StringUtils.isEmpty(action)) { return badRequest("You must provide a valid action"); } else { - // check url - Logger.debug("Query: " + query); boolean isValidUrl = Utils.INSTANCE.validUrl(query); - Logger.debug("valid? " + isValidUrl); if(!isValidUrl) { ValidationError ve = new ValidationError("formUrl", "Invalid URL"); diff --git a/app/models/Target.java b/app/models/Target.java index 0b7a1d02..617c250b 100644 --- a/app/models/Target.java +++ b/app/models/Target.java @@ -159,7 +159,9 @@ public void setEdit_url(String edit_url) { //@JsonProperty("crawl_permissions") @OneToMany(mappedBy = "target", cascade = CascadeType.ALL) + //TODO: orderBy? // @OrderBy("createdAt DESC") + @OrderBy("requestedAt ASC") public List crawlPermissions; @JsonIgnore @@ -1123,35 +1125,240 @@ public static Page pageTargets(int page, int pageSize, return res; } + + public static Page pageReports(int pageNo, Long curatorId, Long organisationId, + String crawlPermissionsStatus, + //String request, + String generalFromDate, String generalToDate, + //String requestedFromDate, String requestedToDate, + String grantedFromDate, String grantedToDate) throws ActException { + + ExpressionList exp = Target.find + .fetch("crawlPermissions").where(); + + exp = exp.eq("active", true); + + Logger.debug("curatorId: " + curatorId); + Logger.debug("organisationId: " + organisationId); + Logger.debug("crawlPermissionsStatus: " + crawlPermissionsStatus); + Logger.debug("generalFromDate: " + generalFromDate); + Logger.debug("generalToDate: " + generalToDate); +// Logger.debug("requestedFromDate: " + requestedFromDate); +// Logger.debug("requestedToDate: " + requestedToDate); + Logger.debug("grantedFromDate: " + grantedFromDate); + Logger.debug("grantedToDate: " + grantedToDate); + + if (curatorId != -1) { + exp = exp.eq("authorUser.id", curatorId); + } + if (organisationId != -1) { + exp = exp.eq("organisation.id", organisationId); + } + + if (StringUtils.isNotEmpty(crawlPermissionsStatus)) + exp = exp.eq("crawlPermissions.status", crawlPermissionsStatus); + + //TODO: IF REJECTED + if (StringUtils.isNotEmpty(generalFromDate)) { + try { + Date date = Utils.INSTANCE.convertDate(generalFromDate); + //ge - greater or equal + exp = crawlPermissionsStatus.equals("PENDING")? + exp.ge("crawlPermissions.requestedAt", date): + exp.ge("crawlPermissions.grantedAt", date); + } catch (ParseException e) { + throw new ActException(e); + } + } + + if (StringUtils.isNotEmpty(generalToDate)) { + try { + String modRequestedToDate = Utils.INSTANCE.getaddDayToDate(generalToDate); + Date date = Utils.INSTANCE.convertDate(modRequestedToDate); + //le - less or equal + exp = crawlPermissionsStatus.equals("PENDING")? + exp.le("crawlPermissions.requestedAt", date): + exp.le("crawlPermissions.grantedAt", date); + + + } catch (ParseException e) { + throw new ActException(e); + } + } + + /* + if (StringUtils.isNotEmpty(grantedFromDate)) { + try { + Date date = Utils.INSTANCE.convertDate(grantedFromDate); + //GE - GREATER OT EQUAL TO VALUE + exp = exp.ge("crawlPermissions.grantedAt", date); + } catch (ParseException e) { + throw new ActException(e); + } + } + + if (StringUtils.isNotEmpty(grantedToDate)) { + try { + String modGrantedToDate = Utils.INSTANCE.getaddDayToDate(grantedToDate); + Date date = Utils.INSTANCE.convertDate(modGrantedToDate); + //LE - LESS OT EQUAL TO VALUE + exp = exp.le("crawlPermissions.grantedAt", date); + } catch (ParseException e) { + throw new ActException(e); + } + } + */ + + //TODO: why hardcoding?! + + String sortBy = crawlPermissionsStatus.equals("PENDING")? + "crawlPermissions.requestedAt": + "crawlPermissions.grantedAt"; + //= "crawlPermissions.requestedAt"; + //String sortBy = "crawlPermissions.grantedAt";//"u1.granted_at";//" MIN(u1.granted_at) "; + String order = " asc "; + //.orderBy(sortBy) + //exp = exp.orderBy("crawlPermissions.grantedAt "); + + Logger.debug("sortBy : " + sortBy); + + Page res; + + Query query = exp.query().orderBy(sortBy + " " + order); + + res = query//.orderBy(sortBy ) + .findPagingList(Const.PAGINATION_OFFSET) + .setFetchAhead(false) + .getPage(pageNo); + Logger.debug("pageReports - Expression list for targets created size: " + + res.getTotalRowCount()); + + return res; + } + + + public static List pageReportsFull(int pageNo, Long curatorId, Long organisationId, + String crawlPermissionsStatus, + //String request, + String generalFromDate, String generalToDate, + String grantedFromDate, String grantedToDate) throws ActException { + ExpressionList exp = Target.find.fetch("crawlPermissions").where(); + exp = exp.eq("active", true); + + Logger.debug("curatorId: " + curatorId); + Logger.debug("organisationId: " + organisationId); + Logger.debug("crawlPermissionsStatus: " + crawlPermissionsStatus); + Logger.debug("generalFromDate: " + generalFromDate); + Logger.debug("generalToDate: " + generalToDate); + Logger.debug("grantedFromDate: " + grantedFromDate); + Logger.debug("grantedToDate: " + grantedToDate); + + if (curatorId != -1) { + exp = exp.eq("authorUser.id", curatorId); + } + if (organisationId != -1) { + exp = exp.eq("organisation.id", organisationId); + } + + if (StringUtils.isNotEmpty(crawlPermissionsStatus)) + exp = exp.eq("crawlPermissions.status", crawlPermissionsStatus); + + if (StringUtils.isNotEmpty(generalFromDate)) { + try { + Date date = Utils.INSTANCE.convertDate(generalFromDate); + exp = crawlPermissionsStatus.equals("PENDING")? + exp.ge("crawlPermissions.requestedAt", date): + exp.ge("crawlPermissions.grantedAt", date); + } catch (ParseException e) { + throw new ActException(e); + } + } + + if (StringUtils.isNotEmpty(generalToDate)) { + try { + String modRequestedToDate = Utils.INSTANCE.getaddDayToDate(generalToDate); + Date date = Utils.INSTANCE.convertDate(modRequestedToDate); + exp = crawlPermissionsStatus.equals("PENDING")? + exp.le("crawlPermissions.requestedAt", date): + exp.le("crawlPermissions.grantedAt", date); + } catch (ParseException e) { + throw new ActException(e); + } + } + + /* + if (StringUtils.isNotEmpty(grantedFromDate)) { + try { + Date date = Utils.INSTANCE.convertDate(grantedFromDate); + exp = exp.ge("crawlPermissions.grantedAt", date); + } catch (ParseException e) { + throw new ActException(e); + } + } + + if (StringUtils.isNotEmpty(grantedToDate)) { + try { + String modGrantedToDate = Utils.INSTANCE.getaddDayToDate(grantedToDate); + Date date = Utils.INSTANCE.convertDate(modGrantedToDate); + exp = exp.le("crawlPermissions.grantedAt", date); + } catch (ParseException e) { + throw new ActException(e); + } + } + */ + + //Logger.debug("exp: " + exp.toString()); + + //TODO: in Target class - need method returns Page + List res;// = null; + + String sortBy = crawlPermissionsStatus.equals("PENDING")? + "crawlPermissions.requestedAt": + "crawlPermissions.grantedAt"; +// String sortBy = " crawlPermissions.grantedAt "; + String order = "asc"; + Query query = exp.query(); + //Logger.info("query.toString = " + query.toString()); + //.orderBy(sortBy + " " + order); + //TODO: why hardcoding?! + //find.where().orderBy("fieldname desc").findList() + + res = query.orderBy(sortBy + " " + order).findList(); + //res = query.findList(); + Logger.debug("pageReports - Expression list for targets created size: " + + res.size()); + + return res; + } + + /** - * Return a page of Target objects. - * - * @param page - * Page to display - * @param pageSize - * Number of targets per page - * @param sortBy - * Target property used for sorting - * @param order - * Sort order (either or asc or desc) - * @param status - * The type of report QA e.g. awaiting QA, with no QA issues... - * @param curatorUrl - * @param organisationUrl - * @param startDate - * The start date for filtering - * @param endDate - * The end date for filtering - * @param collectionCategoryUrl - * @return - */ + * Return a page of Target objects. + * + * @param page + * Page to display + * @param pageSize + * Number of targets per page + * @param sortBy + * Target property used for sorting + * @param order + * Sort order (either or asc or desc) + * @param status + * The type of report QA e.g. awaiting QA, with no QA issues... + * @param curatorUrl + * @param organisationUrl + * @param startDate + * The start date for filtering + * @param endDate + * The end date for filtering + * @param collectionCategoryUrl + * @return + */ public static Page pageReportsQa(int page, int pageSize, String sortBy, String order, String status, Long curatorId, Long organisationId, String startDate, String endDate, Long collectionId) { - -// List instanceList = Instance.processReportsQa(status, startDate, endDate); - + ExpressionList exp = Target.find.fetch("collections").fetch("fieldUrls").where(); Page res = null; exp = exp.eq(Const.ACTIVE, true); @@ -1214,7 +1421,12 @@ public static Page pageReportsCreation(int page, int pageSize, String sortBy, String order, Long curatorId, Long organisationId, String startDate, String endDate, String npld, String crawlFrequencyName, String tld) throws ActException { - ExpressionList exp = Target.find.fetch("fieldUrls").fetch("flags").fetch("licenses").fetch("subjects").fetch("collections").where(); + ExpressionList exp = Target.find + .fetch("fieldUrls") + .fetch("flags") + .fetch("licenses") + .fetch("subjects") + .fetch("collections").where(); Page res = null; exp = exp.eq(Const.ACTIVE, true); @@ -1299,7 +1511,6 @@ public static Page pageReportsCreation(int page, int pageSize, } // TODO: NONE SELECTED??? - Logger.debug("pageReportsCreation() NPLD: " + npld); /** @@ -1346,16 +1557,85 @@ public static Page pageReportsCreation(int page, int pageSize, Query query = exp.query(); - res = query.orderBy(sortBy + " " + order).findPagingList(pageSize) - .setFetchAhead(false).getPage(page); + res = query.orderBy(sortBy + " " + order) + .findPagingList(pageSize) + .setFetchAhead(false) + .getPage(page); Logger.debug("Expression list for targets created size: " + res.getTotalRowCount()); return res; + } + /** + * Target List Count after filtering + * + * @param filterUrl + * @param curatorId + * @param organisationId + * @param subjectSelect + * @param crawlFrequencyName + * @param depthName + * @param collectionSelect + * @param licenseId + * @param flagId + * @return + */ + public static int filteredTargetListCount(String filterUrl, Long curatorId, Long organisationId, + String subjectSelect, String crawlFrequencyName, + String depthName, String collectionSelect, + Long licenseId, Long flagId) { + ExpressionList exp = Target.find.fetch("fieldUrls").where(); + exp = exp.eq(Const.ACTIVE, true); + exp = exp.add(Expr.or( + Expr.icontains("fieldUrls.url", filterUrl), + Expr.icontains("title", filterUrl)) + ); + if (curatorId != 0) { + exp = exp.eq("authorUser.id", curatorId); + } + if (organisationId != 0) { + exp = exp.eq("organisation.id", organisationId); + } + if (StringUtils.isNotEmpty(crawlFrequencyName)) { + exp = exp.eq("crawlFrequency", crawlFrequencyName); + } + if (StringUtils.isNotEmpty(depthName)) { + exp = exp.eq("depth", depthName); + } + if (licenseId != 0) { + if (licenseId == -1) + exp = exp.eq("t0.license_status", null); + else + exp = exp.eq("licenses.id", licenseId); + } + if (flagId != 0) { + exp = exp.eq("flags.id", flagId); + } + if (StringUtils.isNotEmpty(subjectSelect)) { + List subjectIds = new ArrayList(); + String[] subjects = subjectSelect.split(Const.TREE_LIST_ID_DELIMITER); + for (String sId : subjects) { + Long subjectId = Long.valueOf(sId); + subjectIds.add(subjectId); + } + exp = exp.in("subjects.id", subjectIds); + } + if (StringUtils.isNotEmpty(collectionSelect)) { + List collectionIds = new ArrayList(); + String[] collections = collectionSelect.split(Const.TREE_LIST_ID_DELIMITER); + for (String cId : collections) { + Long collectionId = Long.valueOf(cId); + Collection collection = Collection.findById(collectionId); + collectionIds.add(collection); + } + exp = exp.in("collections", collectionIds); + } + return exp.query().select("title").findRowCount(); } + /** * Return a page of Target * diff --git a/app/uk/bl/Const.java b/app/uk/bl/Const.java index dfbb89ac..3646c633 100644 --- a/app/uk/bl/Const.java +++ b/app/uk/bl/Const.java @@ -1,11 +1,7 @@ package uk.bl; -import play.Logger; - import java.io.File; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; public final class Const { @@ -64,7 +60,7 @@ private Const() {} public static final int STRING_LIMIT = 50; // Help constants - public static final int PAGINATION_OFFSET = 10; // offset is a page step from current page for pagination + public static final int PAGINATION_OFFSET = 15; // offset is a page step from current page for pagination public static final int ROWS_PER_PAGE = 50; public static final int MAX_USER_COUNT = 100; public static final int BUFFER_SIZE = 1024; // 1KB @@ -353,9 +349,13 @@ private Const() {} public static final String EXPORT_FILE = "export.csv"; public static final String EXPORT_INSTANCE_FILE = "exportinstances.csv"; public static final String EXPORT_REQUESTED_LICENCE_FILE = "export_requested_licences.csv"; + public static final String EXPORT_LICENCE_FILE_CSV = "export_licences.csv"; + public static final String EXPORT_LICENCE_FILE_TSV = "export_licences.tsv"; + public static final String EXPORT_LICENCE_FILE_JSON = "export_licences.json"; public static final String EXPORT_GRANTED_LICENCE_FILE = "export_granted_licences.csv"; public static final String EXPORT_REFUSED_LICENCE_FILE = "export_refused_licences.csv"; public static final String CSV_SEPARATOR = ","; + public static final String TSV_SEPARATOR = "\t"; //tab - for TAB separated values public static final String CSV_LINE_END = "\n"; public static final String TWO_POINTS = ": "; public static final String SEARCH = "search"; @@ -383,7 +383,7 @@ private Const() {} public static final String QUERY = "q"; public static final String QUERY_COLLECTION = "query_collection"; public static final String QUERY_QA_STATUS = "query_qa_status"; - + /** * Export settings */ @@ -1052,7 +1052,7 @@ public static Map options() { targetMap.put("field_notes", 7); targetMap.put("field_instances", 8); } - + public static final Map targetExportMap = new HashMap(); static { targetExportMap.put("nid", 0); @@ -1062,8 +1062,18 @@ public static Map options() { targetExportMap.put("field_crawl_frequency", 4); targetExportMap.put("created", 5); } - - public static final Map permissionExportMap = new HashMap(); + + public static final List targetExportList = new ArrayList<>(); + static { + targetExportList.add("nid"); + targetExportList.add("title"); + targetExportList.add("field_url"); + targetExportList.add("author"); + targetExportList.add("field_crawl_frequency"); + targetExportList.add("created"); + } + + public static final Map permissionExportMap = new HashMap(); static { permissionExportMap.put("target", 0); permissionExportMap.put("licenseDate", 1); diff --git a/app/uk/bl/api/Utils.java b/app/uk/bl/api/Utils.java index 3ca678c5..6067ad53 100644 --- a/app/uk/bl/api/Utils.java +++ b/app/uk/bl/api/Utils.java @@ -16,16 +16,7 @@ import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.UUID; +import java.util.*; import org.apache.commons.lang3.StringUtils; import org.postgresql.util.PGInterval; @@ -887,36 +878,47 @@ public List processParentsCollections(List collections, } return collections; } + + /** + * Problematic characters should be handled may vary with the use case. + * Our method calls the escapeSpecialCharacters method on some data as it's building up a String. + * @param field + * @return + */ + public String escapeSpecialCharacters(String field) { + String escapedData = field.replaceAll("\\R", " "); + if (field.contains(",") || field.contains("\"") || field.contains("'")) { + field = field.replace("\"", "\"\""); + escapedData = "\"" + field + "\""; + } + return escapedData; + } /** * This method exports selected targets to CSV file. * @param list of Target objects * @return */ - public String export(List targetList) { + public String exportCsv(List targetList) { Logger.debug("export() targetList size: " + targetList.size()); StringBuilder builder = new StringBuilder(); -// StringWriter sw = new StringWriter(); for (int i = 0; i < Const.targetExportMap.size(); i++) { { for (Map.Entry entry : Const.targetExportMap.entrySet()) -// Logger.debug("export key: " + entry.getKey()); if (entry.getValue() == i) { builder.append(entry.getKey()); builder.append(Const.CSV_SEPARATOR); } } } - builder.append(Const.CSV_LINE_END); - if (targetList != null && targetList.size() > 0) { // "nid", "title", "field_url", "author", "field_crawl_frequency", "created" for (Target target : targetList) { builder.append(String.valueOf(target.id)); builder.append(Const.CSV_SEPARATOR); - builder.append(target.title); + builder.append(escapeSpecialCharacters(target.title)); builder.append(Const.CSV_SEPARATOR); builder.append(target.fieldUrl()); builder.append(Const.CSV_SEPARATOR); @@ -935,8 +937,79 @@ public String export(List targetList) { // Utils.INSTANCE.generateCsvFile(Const.EXPORT_FILE, sw.toString()); return builder.toString(); } - - public boolean isDuplicate(String url, String dbUrl) throws ActException { + + /** + * This method exports selected targets to TSV file. + * @param targetList + * @return + */ + public String exportTsv(List targetList) { + Logger.debug("export() targetList size: " + targetList.size()); + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < Const.targetExportMap.size(); i++) { + { + for (Map.Entry entry : Const.targetExportMap.entrySet()) + if (entry.getValue() == i) { + builder.append(entry.getKey()); + builder.append(Const.TSV_SEPARATOR); + } + } + } + builder.append(Const.CSV_LINE_END); + if (targetList != null && targetList.size() > 0) { +// "nid", "title", "field_url", "author", "field_crawl_frequency", "created" + for (Target target : targetList) { + builder.append(String.valueOf(target.id)); + builder.append(Const.TSV_SEPARATOR); + builder.append(escapeSpecialCharacters(target.title)); + builder.append(Const.TSV_SEPARATOR); + builder.append(target.fieldUrl()); + builder.append(Const.TSV_SEPARATOR); + String authorName = ""; + if (target.authorUser != null) { + authorName = target.authorUser.name; + } + builder.append(authorName); + builder.append(Const.TSV_SEPARATOR); + builder.append(target.crawlFrequency); + builder.append(Const.TSV_SEPARATOR); + builder.append(convertToDateTimeISO(target.createdAt)); + builder.append(Const.CSV_LINE_END); + } + } + return builder.toString(); + } + + /** + * Transform Targets' data into reduced data list of maps, where fields defined in Const.targetExportList + * @param targetList + * @return + */ + public List exportJson(List targetList) { + Logger.debug("export(JSON) targetList size: " + targetList.size()); + List listOfMaps = new ArrayList<>(); + Map targetFields; + if (targetList != null && targetList.size() > 0) { +// "nid", "title", "field_url", "author", "field_crawl_frequency", "created" + for (Target target : targetList) { + targetFields = new HashMap<>(); + targetFields.put(Const.targetExportList.get(0), String.valueOf(target.id)); + targetFields.put(Const.targetExportList.get(1), escapeSpecialCharacters(target.title)); + targetFields.put(Const.targetExportList.get(2), target.fieldUrl()); + String authorName = ""; + if (target.authorUser != null) { + authorName = target.authorUser.name; + } + targetFields.put(Const.targetExportList.get(3), authorName); + targetFields.put(Const.targetExportList.get(4), target.crawlFrequency); + targetFields.put(Const.targetExportList.get(5), convertToDateTimeISO(target.createdAt)); + listOfMaps.add(targetFields); + } + } + return listOfMaps; + } + + public boolean isDuplicate(String url, String dbUrl) throws ActException { url = Utils.INSTANCE.getPath(url); dbUrl = Utils.INSTANCE.getPath(dbUrl); boolean match = (url.equalsIgnoreCase(dbUrl)); diff --git a/app/views/crawlpermissions/list.scala.html b/app/views/crawlpermissions/list.scala.html index 5729ff57..79ad97ba 100644 --- a/app/views/crawlpermissions/list.scala.html +++ b/app/views/crawlpermissions/list.scala.html @@ -250,6 +250,9 @@

Archivist Tasks > Cr + + +
@if(currentPage.getTotalRowCount() > 0 && CrawlPermissionController.haveContactPerson(currentPage.getList())) { diff --git a/app/views/crawlpermissions/view.scala.html b/app/views/crawlpermissions/view.scala.html index 3949ac32..7ccbf96b 100644 --- a/app/views/crawlpermissions/view.scala.html +++ b/app/views/crawlpermissions/view.scala.html @@ -220,7 +220,7 @@

Archivist Tasks > Crawl Permissions > Crawl Permission @permission.name

@if(user.hasRole("sys_admin") || user.hasRole("archivist")) { - Send + Back to Crawl Permissions } @if(permission.target.title != null && permission.target.title.length() > 0) { Back to Target Record diff --git a/app/views/reports/reports.scala.html b/app/views/reports/reports.scala.html index b23eeb23..e35c34dc 100644 --- a/app/views/reports/reports.scala.html +++ b/app/views/reports/reports.scala.html @@ -1,4 +1,6 @@ -@(title: String, user: User, permissions: List[Target], permissionsGranted: List[Target], permissionsRefused: List[Target], curatorId: Long, organisationId: Long, startDate: String, endDate: String, requestType: String, grantedFromDate: String, grantedToDate: String, users: List[User], organisations: List[Organisation], requestTypes: Array[uk.bl.Const.RequestType]) +@import uk.bl.Const +@import uk.bl.Const +@(title: String, user: User, currentPage: com.avaje.ebean.Page[Target], currentSortBy: String, currentOrder: String, crawlPermissionsStatus2: String,curatorId: Long, organisationId: Long, startDate: String, endDate: String, grantedFromDate: String, grantedToDate: String, users: List[User], organisations: List[Organisation]) @styles = { @@ -6,19 +8,19 @@ @scripts = { - - + + } + @**************************************** + * Helper generating navigation links * + ****************************************@ +@link(newPage:Int, newSortBy:String) = @{ + +// var sortBy = currentSortBy +// var order = currentOrder +// +// if(newSortBy != null) { +// sortBy = newSortBy +// if(currentSortBy == newSortBy) { +// if(currentOrder == "asc") { +// order = "desc" +// } else { +// order = "asc" +// } +// } else { +// order = "asc" +// } +// } + + // Generate the link + + routes.ReportController.processFilterReports(newPage, curatorId, organisationId, + crawlPermissionsStatus2, + startDate, endDate, grantedFromDate, grantedToDate) +} + + @********************************** + * Helper generating table headers * + ***********************************@ + +@pagination = { +
+ +
+} + + @if(user.hasRole("sys_admin") || user.hasRole("archivist") || user.hasRole("expert_user") || user.hasRole("user")) { @main(title, user, scripts, styles) { @@ -57,32 +131,55 @@

Targets > Reports

- + - -
- -
- +
- @helper.form(action=routes.ReportController.search) { + @helper.form(action=routes.ReportController.search()) { +
- +
+@* *@ + + + + + +
+ + +
+
+ +
+
+
+ Current selection +
+
+
+
+ @if(crawlPermissionsStatus2.equals(Const.CrawlPermissionStatus.PENDING.toString())){ + REQUESTED + } else { + @crawlPermissionsStatus2 + } + +
+
+
+
▼ Filters ▲
- +
@@ -92,7 +189,7 @@

Targets > Reports

- + @for(organisation <- organisations) { @if(organisation.id == organisationId) { } else { } - } + }
+
- +
-
- +
@@ -143,76 +240,28 @@

Targets > Reports

-
-
-
-
-
-
- -
- -
-
-
-
- +
- -
-
-
- -
-
- -
-
-
-
- -
-
- -
-
- -
-
-
-
-
- - - - + + @if(user.hasRole("sys_admin") || user.hasRole("archivist")) {
-
-
+
+
- @if(permissions.size() > 0 || permissionsGranted.size() > 0 || permissionsRefused.size() > 0) { - - } else { - - } -
+ + + + +
- } + } +
+ @pagination
-
@@ -221,52 +270,19 @@

Targets > Reports

+ @if(crawlPermissionsStatus2.equals(Const.CrawlPermissionStatus.GRANTED.toString())){ + + } + @if(crawlPermissionsStatus2.equals(Const.CrawlPermissionStatus.REFUSED.toString())){ + + } - @permissions.map { target => - - - - @if(target.url != null && target.getLatestCrawlPermission.requestedAt != null) { - - } else { - - } - - - } - -
Target Title Target URL Date requestedDate grantedDate refused
- @if(target.title != null || target.title.length() > 0) { - @target.title - } else { - none - } - - @for(fieldUrl <- target.fieldUrls) { - @fieldUrl.url - } - @target.getLatestCrawlPermission.requestedAt
-
-
- -
-
- - - - - - - - - - - @permissionsGranted.map { target => + @for(target <- currentPage.getList) { @if(target.url != null && target.getLatestCrawlPermission.requestedAt != null) { } else { - } + } @if(target.url != null && target.getLatestCrawlPermission.grantedAt != null) { } else { - } - - } - -
Target TitleTarget URLDate requestedDate granted
- @if(!target.title.equals("none")) { + @if(target.title != null || target.title.length() > 0) { @target.title } else { none @@ -275,65 +291,27 @@

Targets > Reports

 @for(fieldUrl <- target.fieldUrls) { @fieldUrl.url - } + } @target.getLatestCrawlPermission.requestedAt@target.getLatestCrawlPermission.grantedAt
-
-
- -
-
- - - - - - - - - - @permissionsRefused.map { target => - - - - @if(target.url != null && target.getLatestCrawlPermission.requestedAt != null) { - - } else { - - } - + } + }
Target TitleTarget URLDate requested
- @if(!target.title.equals("none")) { - @target.title - } else { - none - } - - @for(fieldUrl <- target.fieldUrls) { - @fieldUrl.url - } - @target.getLatestCrawlPermission.requestedAt
-
+
- - - + + @pagination } } diff --git a/app/views/reports/reportscreation.scala.html b/app/views/reports/reportscreation.scala.html index ebc140d6..b3616f7f 100644 --- a/app/views/reports/reportscreation.scala.html +++ b/app/views/reports/reportscreation.scala.html @@ -267,7 +267,7 @@

Targets > Reports

@header("title", "Target Title") @header("url", "Target URL") - @header("created", "Date created") + @header("createdAt", "Date created") diff --git a/app/views/targets/list.scala.html b/app/views/targets/list.scala.html index 10f56177..3642e851 100644 --- a/app/views/targets/list.scala.html +++ b/app/views/targets/list.scala.html @@ -226,7 +226,10 @@

Targets > List

@if(user.hasRole("sys_admin") || user.hasRole("archivist") || user.hasRole("expert_user") || user.hasRole("user")) { - + + + + } @@ -417,8 +420,11 @@

Targets > List

@if(user.hasRole("sys_admin") || user.hasRole("archivist") || user.hasRole("expert_user") || user.hasRole("user")) { - - } + + + + + } diff --git a/conf/routes b/conf/routes index 821c32cf..0bb96e5c 100644 --- a/conf/routes +++ b/conf/routes @@ -49,6 +49,7 @@ GET /collections controllers.CollectionController.ind GET /collections/list controllers.CollectionController.list(p:Int ?= 0, s ?= "name", o ?= "asc", f ?= "") GET /collections/search controllers.CollectionController.search() GET /collections/filterbyjson/:name controllers.CollectionController.filterByJson(name) +GET /collections/json controllers.CollectionController.json() GET /collections/new controllers.CollectionController.newForm() GET /collections/:id controllers.CollectionController.view(id:Long) #GET /collections/:id/json controllers.CollectionController.viewAsJson(id:Long) @@ -63,6 +64,7 @@ GET /subjects controllers.SubjectController.index( GET /subjects/list controllers.SubjectController.list(p:Int ?= 0, s ?= "name", o ?= "asc", f ?= "") GET /subjects/search controllers.SubjectController.search() GET /subjects/filterbyjson/:name controllers.SubjectController.filterByJson(name) +GET /subjects/json controllers.SubjectController.json() GET /subjects/new controllers.SubjectController.newForm() GET /subjects/:id controllers.SubjectController.view(id:Long) GET /subjects/:id/edit controllers.SubjectController.edit(id:Long) @@ -210,6 +212,7 @@ POST /crawlpermissions/save controllers.CrawlPermissionContr POST /crawlpermissions/update/:id controllers.CrawlPermissionController.update(id:Long) GET /reports controllers.ReportController.index() +GET /reports/crawlstatus controllers.ReportController.processFilterReports(p:Int ?= 0, curator:Long ?= -1L, organisation:Long ?= -1L, crawlPermissionsStatus ?= "", requestedFromDate ?= "", requestedToDate ?= "", grantedFromDate ?= "", grantedToDate ?= "") GET /reports/search controllers.ReportController.search() GET /reports/summary controllers.ReportController.summary() GET /reports/openLicences controllers.ReportController.openLicences() diff --git a/conf/templates/2_NLS_acknowledgement.txt b/conf/templates/2_NLS_acknowledgement.txt index 25f368c3..6aef2e53 100644 --- a/conf/templates/2_NLS_acknowledgement.txt +++ b/conf/templates/2_NLS_acknowledgement.txt @@ -18,7 +18,7 @@ Web Archivist | Collections Management National Library of Scotland George IV Bridge, Edinburgh EH1 1EW -Tel: +44 (0) 131 623 3796 +Tel: +44 (0) 131 623 3934 Email: e.macglone@nls.uk Website: www.nls.uk Twitter: @natlibscot diff --git a/conf/templates/4_NLS_LD_openAccess.txt b/conf/templates/4_NLS_LD_openAccess.txt index 89feed73..079d252a 100644 --- a/conf/templates/4_NLS_LD_openAccess.txt +++ b/conf/templates/4_NLS_LD_openAccess.txt @@ -22,7 +22,7 @@ Web Archivist | Collections Management National Library of Scotland George IV Bridge, Edinburgh EH1 1EW -Tel: +44 (0) 131 623 3796 +Tel: +44 (0) 131 623 3934 Email: e.macglone@nls.uk Website: www.nls.uk Twitter: @natlibscot diff --git a/conf/templates/6_NLS_nonUK_nonLD.txt b/conf/templates/6_NLS_nonUK_nonLD.txt index 6fc9cbff..7bb263c7 100644 --- a/conf/templates/6_NLS_nonUK_nonLD.txt +++ b/conf/templates/6_NLS_nonUK_nonLD.txt @@ -13,7 +13,7 @@ Web Archivist | Collections Management National Library of Scotland George IV Bridge, Edinburgh EH1 1EW -Tel: +44 (0) 131 623 3796 +Tel: +44 (0) 131 623 3934 Email: e.macglone@nls.uk Website: www.nls.uk Twitter: @natlibscot