diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Clear.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Clear.java index 6a314b21583..9008f049763 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Clear.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Clear.java @@ -1,5 +1,5 @@ //============================================================================= -//=== Copyright (C) 2001-2013 Food and Agriculture Organization of the +//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the //=== United Nations (FAO-UN), United Nations World Food Programme (WFP) //=== and United Nations Environment Programme (UNEP) //=== @@ -39,6 +39,7 @@ * * @author delawen */ +@Deprecated public class Clear implements Service { //-------------------------------------------------------------------------- //--- diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/CreateClone.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/CreateClone.java index 49139c0ec84..7ae59b44fad 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/CreateClone.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/CreateClone.java @@ -1,5 +1,5 @@ //============================================================================= -//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the +//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the //=== United Nations (FAO-UN), United Nations World Food Programme (WFP) //=== and United Nations Environment Programme (UNEP) //=== @@ -37,6 +37,7 @@ /** * TODO javadoc. */ +@Deprecated public class CreateClone implements Service { //-------------------------------------------------------------------------- //--- diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Get.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Get.java index 08fe11127ec..f40e4acfbf8 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Get.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Get.java @@ -51,6 +51,7 @@ /** * */ +@Deprecated public class Get implements Service { //-------------------------------------------------------------------------- //--- diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/HistoryDelete.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/HistoryDelete.java index 34b60905fdb..890736e6a2c 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/HistoryDelete.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/HistoryDelete.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2001-2016 Food and Agriculture Organization of the + * Copyright (C) 2001-2023 Food and Agriculture Organization of the * United Nations (FAO-UN), United Nations World Food Programme (WFP) * and United Nations Environment Programme (UNEP) * @@ -47,6 +47,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +@Deprecated public class HistoryDelete implements Service { //-------------------------------------------------------------------------- //--- diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Invoke.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Invoke.java index 7fcc052d6bf..4b68d20ff0d 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Invoke.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Invoke.java @@ -1,5 +1,5 @@ //============================================================================= -//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the +//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the //=== United Nations (FAO-UN), United Nations World Food Programme (WFP) //=== and United Nations Environment Programme (UNEP) //=== @@ -44,6 +44,7 @@ * "Invoke" service will run a harvester directly, i.e. synchronously. The main purpose for this * service is testing, i.e. get results immmediately returned in a test script. */ +@Deprecated public class Invoke implements Service { //-------------------------------------------------------------------------- //--- diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Log.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Log.java index b134c0788c8..19d0cd8c002 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Log.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Log.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2001-2016 Food and Agriculture Organization of the + * Copyright (C) 2001-2023 Food and Agriculture Organization of the * United Nations (FAO-UN), United Nations World Food Programme (WFP) * and United Nations Environment Programme (UNEP) * @@ -42,6 +42,7 @@ * * @author delawen */ +@Deprecated public class Log implements Service { public void init(Path appPath, ServiceConfig config) throws Exception { diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Remove.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Remove.java index e870e7c4f2f..17f7aa60491 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Remove.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Remove.java @@ -1,5 +1,5 @@ //============================================================================= -//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the +//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the //=== United Nations (FAO-UN), United Nations World Food Programme (WFP) //=== and United Nations Environment Programme (UNEP) //=== @@ -35,6 +35,7 @@ //============================================================================= +@Deprecated public class Remove implements Service { //-------------------------------------------------------------------------- //--- diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Run.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Run.java index c50f7a03ed0..5aec260ff36 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Run.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Run.java @@ -1,5 +1,5 @@ //============================================================================= -//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the +//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the //=== United Nations (FAO-UN), United Nations World Food Programme (WFP) //=== and United Nations Environment Programme (UNEP) //=== @@ -35,6 +35,7 @@ //============================================================================= +@Deprecated public class Run implements Service { //-------------------------------------------------------------------------- //--- diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Start.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Start.java index e8f4c13c04f..6e02e54663b 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Start.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Start.java @@ -1,5 +1,5 @@ //============================================================================= -//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the +//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the //=== United Nations (FAO-UN), United Nations World Food Programme (WFP) //=== and United Nations Environment Programme (UNEP) //=== @@ -35,6 +35,7 @@ //============================================================================= +@Deprecated public class Start implements Service { //-------------------------------------------------------------------------- //--- diff --git a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Stop.java b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Stop.java index 3ac887c41fe..0a9e3783a74 100644 --- a/harvesters/src/main/java/org/fao/geonet/services/harvesting/Stop.java +++ b/harvesters/src/main/java/org/fao/geonet/services/harvesting/Stop.java @@ -1,5 +1,5 @@ //============================================================================= -//=== Copyright (C) 2001-2007 Food and Agriculture Organization of the +//=== Copyright (C) 2001-2023 Food and Agriculture Organization of the //=== United Nations (FAO-UN), United Nations World Food Programme (WFP) //=== and United Nations Environment Programme (UNEP) //=== @@ -36,6 +36,7 @@ //============================================================================= +@Deprecated public class Stop implements Service { //-------------------------------------------------------------------------- //--- diff --git a/services/src/main/java/org/fao/geonet/api/harvesting/HarvestersApi.java b/services/src/main/java/org/fao/geonet/api/harvesting/HarvestersApi.java index ca2b45d30f7..e1e0ea5dc4e 100644 --- a/services/src/main/java/org/fao/geonet/api/harvesting/HarvestersApi.java +++ b/services/src/main/java/org/fao/geonet/api/harvesting/HarvestersApi.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2001-2016 Food and Agriculture Organization of the + * Copyright (C) 2001-2023 Food and Agriculture Organization of the * United Nations (FAO-UN), United Nations World Food Programme (WFP) * and United Nations Environment Programme (UNEP) * @@ -23,11 +23,14 @@ package org.fao.geonet.api.harvesting; +import com.google.common.collect.Lists; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; import jeeves.server.context.ServiceContext; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.fao.geonet.api.ApiParams; import org.fao.geonet.api.ApiUtils; import org.fao.geonet.api.exception.NoResultsFoundException; @@ -36,31 +39,43 @@ import org.fao.geonet.domain.HarvestHistory; import org.fao.geonet.domain.ISODate; import org.fao.geonet.domain.Source; +import org.fao.geonet.exceptions.ObjectNotFoundEx; import org.fao.geonet.kernel.DataManager; +import org.fao.geonet.kernel.GeonetworkDataDirectory; import org.fao.geonet.kernel.datamanager.IMetadataManager; import org.fao.geonet.kernel.datamanager.IMetadataUtils; +import org.fao.geonet.kernel.harvest.Common; import org.fao.geonet.kernel.harvest.HarvestManager; import org.fao.geonet.kernel.harvest.harvester.AbstractHarvester; +import org.fao.geonet.kernel.setting.SettingManager; +import org.fao.geonet.kernel.setting.Settings; import org.fao.geonet.repository.HarvestHistoryRepository; import org.fao.geonet.repository.SourceRepository; +import org.fao.geonet.repository.specification.HarvestHistorySpecs; +import org.fao.geonet.services.harvesting.Util; +import org.fao.geonet.utils.Xml; +import org.jdom.Content; import org.jdom.Element; +import org.jdom.JDOMException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.jpa.domain.Specification; import org.springframework.http.HttpEntity; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.*; +import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; -import java.util.ArrayList; -import java.util.List; +import javax.servlet.http.HttpServletResponse; +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; @RequestMapping(value = { "/{portal}/api/harvesters" @@ -82,18 +97,16 @@ public class HarvestersApi { private IMetadataManager metadataManager; @Autowired private DataManager dataManager; + @Autowired + private SettingManager settingManager; @io.swagger.v3.oas.annotations.Operation( summary = "Assign harvester records to a new source", description = "" -// authorizations = { -// @Authorization(value = "basicAuth") -// }) ) - @RequestMapping( + @PostMapping( value = "/{harvesterUuid}/assign", - produces = MediaType.APPLICATION_JSON_VALUE, - method = RequestMethod.POST + produces = MediaType.APPLICATION_JSON_VALUE ) @ResponseStatus(value = HttpStatus.NO_CONTENT) @PreAuthorize("hasAuthority('UserAdmin')") @@ -108,12 +121,12 @@ public HttpEntity assignHarvestedRecordToSource( description = "The harvester UUID" ) @PathVariable - String harvesterUuid, + String harvesterUuid, @Parameter( description = "The target source UUID" ) @RequestParam - String source) throws Exception { + String source) throws Exception { final long elapsedTime = System.currentTimeMillis(); final AbstractHarvester harvester = harvestManager.getHarvester(harvesterUuid); if (harvester == null) { @@ -132,20 +145,20 @@ public HttpEntity assignHarvestedRecordToSource( final List allHarvestedRecords = metadataRepository.findAllByHarvestInfo_Uuid(harvesterUuid); List records = new ArrayList<>(allHarvestedRecords.size()); - if (allHarvestedRecords.size() < 1) { + if (allHarvestedRecords.isEmpty()) { throw new NoResultsFoundException(String.format( "Harvester with UUID '%s' has no record to assign to source '%s'.", harvesterUuid, source)); } - for (AbstractMetadata record : allHarvestedRecords) { - record.getSourceInfo().setSourceId(source); - record.getHarvestInfo().setHarvested(false) + for (AbstractMetadata metadataRecord : allHarvestedRecords) { + metadataRecord.getSourceInfo().setSourceId(source); + metadataRecord.getHarvestInfo().setHarvested(false) .setUri(null) .setUuid(null); - metadataManager.save(record); - records.add(record.getId() + ""); + metadataManager.save(metadataRecord); + records.add(metadataRecord.getId() + ""); } dataManager.indexMetadata(records); @@ -174,13 +187,9 @@ public HttpEntity assignHarvestedRecordToSource( @io.swagger.v3.oas.annotations.Operation( summary = "Check if a harvester name or host already exist", description = "" - // authorizations = { - // @Authorization(value = "basicAuth") - // }) ) - @RequestMapping( - value = "/properties/{property}", - method = RequestMethod.GET + @GetMapping( + value = "/properties/{property}" ) @ResponseStatus(value = HttpStatus.OK) @PreAuthorize("hasAuthority('UserAdmin')") @@ -194,12 +203,12 @@ public ResponseEntity checkHarvesterPropertyExist( description = "The harvester property to check" ) @PathVariable - String property, + String property, @Parameter( description = "The value to search" ) @RequestParam - String exist, + String exist, HttpServletRequest request) throws Exception { ServiceContext context = ApiUtils.createServiceContext(request); final Element list = harvestManager.get(null, context, "site[1]/name[1]"); @@ -212,4 +221,381 @@ public ResponseEntity checkHarvesterPropertyExist( return new ResponseEntity<>(HttpStatus.NOT_FOUND); } + + @io.swagger.v3.oas.annotations.Operation( + summary = "Remove a harvester", + description = "") + @DeleteMapping( + value = "/{harvesterIdentifier}") + @ResponseStatus(HttpStatus.NO_CONTENT) + @ApiResponses(value = { + @ApiResponse(responseCode = "204", description = "Harvester removed."), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN) + }) + @PreAuthorize("hasAuthority('UserAdmin')") + public ResponseEntity deleteHarvester( + @Parameter( + description = "Harvester identifier", + required = true + ) + @PathVariable + Integer harvesterIdentifier, + HttpServletRequest request + ) throws Exception { + ServiceContext context = ApiUtils.createServiceContext(request); + Element params = new Element("params"); + params.addContent(new Element("id").setText(harvesterIdentifier.toString())); + + Util.exec(params, context, (hm, id) -> hm.remove(id)); + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + @io.swagger.v3.oas.annotations.Operation( + summary = "Removes the harvester metadata", + description = "") + @PutMapping( + value = "/{harvesterIdentifier}/clear") + @ResponseStatus(HttpStatus.NO_CONTENT) + @ApiResponses(value = { + @ApiResponse(responseCode = "204", description = "Harvester metadata removed."), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN) + }) + @PreAuthorize("hasAuthority('UserAdmin')") + public ResponseEntity clearHarvester( + @Parameter( + description = "Harvester identifier", + required = true + ) + @PathVariable + Integer harvesterIdentifier, + HttpServletRequest request + ) throws Exception { + ServiceContext context = ApiUtils.createServiceContext(request); + Element params = new Element("params"); + params.addContent(new Element("id").setText(harvesterIdentifier.toString())); + + Util.exec(params, context, (hm, id) -> hm.clearBatch(id)); + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + @io.swagger.v3.oas.annotations.Operation( + summary = "Activate a harvester", + description = "") + @PutMapping( + value = "/{harvesterIdentifier}/start") + @ResponseStatus(HttpStatus.NO_CONTENT) + @ApiResponses(value = { + @ApiResponse(responseCode = "204", description = "Harvester activated."), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN) + }) + @PreAuthorize("hasAuthority('UserAdmin')") + public ResponseEntity startHarvester( + @Parameter( + description = "Harvester identifier", + required = true + ) + @PathVariable + Integer harvesterIdentifier, + HttpServletRequest request + ) throws Exception { + ServiceContext context = ApiUtils.createServiceContext(request); + Element params = new Element("params"); + params.addContent(new Element("id").setText(harvesterIdentifier.toString())); + + Util.exec(params, context, (hm, id) -> hm.start(id)); + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + @io.swagger.v3.oas.annotations.Operation( + summary = "Deactivate a harvester", + description = "") + @PutMapping( + value = "/{harvesterIdentifier}/stop") + @ResponseStatus(HttpStatus.NO_CONTENT) + @ApiResponses(value = { + @ApiResponse(responseCode = "204", description = "Harvester activated."), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN) + }) + @PreAuthorize("hasAuthority('UserAdmin')") + public ResponseEntity stopHarvester( + @Parameter( + description = "Harvester identifier", + required = true + ) + @PathVariable + Integer harvesterIdentifier, + @Parameter( + description = "Harvester status" + ) + @RequestParam(required = false) + String status, + HttpServletRequest request + ) throws Exception { + ServiceContext context = ApiUtils.createServiceContext(request); + Element params = new Element("params"); + params.addContent(new Element("id").setText(harvesterIdentifier.toString())); + + if (StringUtils.isEmpty(status)) { + status = Common.Status.INACTIVE.toString(); + } + + final Common.Status newStatus = Common.Status.parse(status); + Util.exec(params, context, (hm, id) -> hm.stop(id, newStatus)); + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + + @io.swagger.v3.oas.annotations.Operation( + summary = "Activate and run a harvester", + description = "") + @PutMapping( + value = "/{harvesterIdentifier}/run") + @ResponseStatus(HttpStatus.NO_CONTENT) + @ApiResponses(value = { + @ApiResponse(responseCode = "204", description = "Harvester executed."), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN) + }) + @PreAuthorize("hasAuthority('UserAdmin')") + public ResponseEntity runHarvester( + @Parameter( + description = "Harvester identifier", + required = true + ) + @PathVariable + Integer harvesterIdentifier, + HttpServletRequest request + ) throws Exception { + ServiceContext context = ApiUtils.createServiceContext(request); + Element params = new Element("params"); + params.addContent(new Element("id").setText(harvesterIdentifier.toString())); + + Util.exec(params, context, (hm, id) -> hm.run(id)); + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + @io.swagger.v3.oas.annotations.Operation( + summary = "Create a clone of a harvester", + description = "") + @PutMapping( + value = "/{harvesterIdentifier}/clone") + @ResponseStatus(HttpStatus.CREATED) + @ApiResponses(value = { + @ApiResponse(responseCode = "201", description = "Harvester cloned."), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN), + @ApiResponse(responseCode = "404", description = ApiParams.API_RESPONSE_RESOURCE_NOT_FOUND) + }) + @PreAuthorize("hasAuthority('UserAdmin')") + @ResponseBody + public ResponseEntity cloneHarvester( + @Parameter( + description = "Harvester identifier", + required = true + ) + @PathVariable + Integer harvesterIdentifier, + HttpServletRequest request + ) throws Exception { + ServiceContext context = ApiUtils.createServiceContext(request); + + String newId = context.getBean(HarvestManager.class).createClone(harvesterIdentifier.toString(), context.getUserSession().getUserId(), context); + + if (newId != null) { + return new ResponseEntity<>(Integer.parseInt(newId), HttpStatus.CREATED); + } else { + //--- we get here only if the 'id' was not present or node was not found + throw new ResourceNotFoundException(String.format( + "Harvester with identifier '%d' not found. Cannot clone the harvester.", + harvesterIdentifier)); + } + } + + @io.swagger.v3.oas.annotations.Operation( + summary = "Delete a harvester history", + description = "") + @DeleteMapping( + value = "/{harvesterUuid}/history") + @ResponseStatus(HttpStatus.NO_CONTENT) + @ApiResponses(value = { + @ApiResponse(responseCode = "204", description = "Harvester history removed."), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN) + }) + @PreAuthorize("hasAuthority('UserAdmin')") + public ResponseEntity deleteHarvesterHistory( + @Parameter( + description = "Harvester identifier", + required = true + ) + @PathVariable + String harvesterUuid + ) { + final Specification hasHarvesterUuid = HarvestHistorySpecs.hasHarvesterUuid(harvesterUuid); + historyRepository.deleteAll(hasHarvesterUuid); + + return new ResponseEntity<>(HttpStatus.NO_CONTENT); + } + + + @io.swagger.v3.oas.annotations.Operation( + summary = "Download a logfile from harvesting", + description = "") + @GetMapping( + value = "/{harvesterHistoryIdentifier}/log", + produces = { + "text/plain" + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "Log file returned."), + @ApiResponse(responseCode = "400", description = "Bad parameters."), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN) + }) + @ResponseBody + public void getLog( + @Parameter( + description = "Harvester identifier", + required = true + ) + @PathVariable + Integer harvesterHistoryIdentifier, + HttpServletResponse response) throws IOException, JDOMException, ResourceNotFoundException { + + Optional harvestHistoryOptional = historyRepository.findById(harvesterHistoryIdentifier); + String logFile = ""; + + if (harvestHistoryOptional.isPresent()) { + Element info = harvestHistoryOptional.get().getInfoAsXml(); + logFile = info.getChildText("logfile"); + } + + if (StringUtils.isEmpty(logFile)) { + throw new ResourceNotFoundException( + "Couldn't find or read the logfile in catalogue log directory for the harvester history entry. Check log file configuration."); + } + + File mainLogFile = GeonetworkDataDirectory.getLogfile(); + Path pathLogFile; + if (mainLogFile != null) { + pathLogFile = mainLogFile.toPath().getParent().resolve(logFile); + } + else { + pathLogFile = Paths.get(logFile); + } + + if (!Files.exists(pathLogFile) || !Files.isReadable(pathLogFile)) { + throw new ResourceNotFoundException(String.format( + "Couldn't find or read the logfile %s in catalogue log directory. Check log file configuration.", + logFile)); + } + + response.setContentType("text/plain"); + response.setHeader("Content-Disposition", "attachment;filename=" + logFile); + ServletOutputStream out = response.getOutputStream(); + try (BufferedReader reader1 = new BufferedReader(new InputStreamReader(new FileInputStream(pathLogFile.toFile()), StandardCharsets.UTF_8))) { + IOUtils.copy(reader1, out, StandardCharsets.UTF_8); + } finally { + out.flush(); + out.close(); + } + } + + @io.swagger.v3.oas.annotations.Operation( + summary = "List harvesters", + description = "" + ) + @GetMapping( + produces = { + MediaType.APPLICATION_JSON_VALUE + } + ) + @ResponseStatus(value = HttpStatus.OK) + @PreAuthorize("hasAuthority('UserAdmin')") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "List of harvesters."), + @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_ONLY_USER_ADMIN) + }) + @ResponseBody + public String list( + @Parameter( + description = "The harvesters identifiers" + ) + @RequestParam(name = "id", required = false) + Integer[] harvesterIds, + @Parameter( + description = "Return only information" + ) + @RequestParam(required = false, defaultValue = "false") + Boolean onlyInfo, + @RequestParam(required = false, defaultValue = "site[1]/name[1]") + String sortField, + HttpServletRequest request) throws Exception { + + ServiceContext context = ApiUtils.createServiceContext(request); + + //--- if 'id' is null all entries are returned + String[] disabledTypes = StringUtils.split( + StringUtils.defaultIfBlank( + settingManager.getValue(Settings.SYSTEM_HARVESTER_DISABLED_HARVESTER_TYPES), + "").toLowerCase().replace(',', ' '), + " "); + + List ids; + if (harvesterIds.length == 0) { + ids = Collections.singletonList(-1); + } else { + ids = Arrays.asList(harvesterIds); + } + Element result = new Element("nodes"); + for (Integer id : ids) { + Element node = harvestManager.get(String.valueOf(id), context, sortField); + + if (node != null) { + if (id == -1) { + List childNodes = node.getChildren(); + for (Element childNode : childNodes) { + String harvesterType = childNode.getAttributeValue("type"); + if (Arrays.stream(disabledTypes).noneMatch(disabledType -> disabledType.equalsIgnoreCase(harvesterType))) { + result.addContent((Content) childNode.clone()); + } + } + } else { + String harvesterType = node.getAttributeValue("type"); + if (Arrays.stream(disabledTypes).noneMatch(disabledType -> disabledType.equalsIgnoreCase(harvesterType))) { + result.addContent(node.detach()); + } + } + } else { + throw new ObjectNotFoundEx("No Harvester found with id: " + id); + } + } + + if (onlyInfo) { + removeAllDataExceptInfo(result); + } + + return Xml.getJSON(result); + } + + private void removeAllDataExceptInfo(Element node) { + final List toRemove = Lists.newArrayList(); + @SuppressWarnings("unchecked") + final List children = node.getChildren(); + + for (Element harvesters : children) { + @SuppressWarnings("unchecked") + final List harvesterInfo = harvesters.getChildren(); + for (Element element : harvesterInfo) { + if (!element.getName().equalsIgnoreCase("info") && !element.getName().equalsIgnoreCase("error")) { + toRemove.add(element); + } + } + } + + for (Element element : toRemove) { + element.detach(); + } + } } diff --git a/web-ui/src/main/resources/catalog/js/admin/HarvestController.js b/web-ui/src/main/resources/catalog/js/admin/HarvestController.js index a01f8b56d9f..964be297dcf 100644 --- a/web-ui/src/main/resources/catalog/js/admin/HarvestController.js +++ b/web-ui/src/main/resources/catalog/js/admin/HarvestController.js @@ -91,7 +91,7 @@ $scope.loadHarvesters = function () { $scope.isLoadingHarvester = true; $scope.harvesters = null; - return $http.get("admin.harvester.list?_content_type=json&id=-1").then( + return $http.get("../api/harvesters?id=-1").then( function (response) { var data = response.data; @@ -132,10 +132,7 @@ isPolling = true; $http - .get( - "admin.harvester.list?onlyInfo=true&_content_type=json&id=" + - runningHarvesters.join("&id=") - ) + .get("../api/harvesters?onlyInfo=true&id=" + runningHarvesters.join("&id=")) .then( function (response) { var data = response.data; diff --git a/web-ui/src/main/resources/catalog/js/admin/HarvestSettingsController.js b/web-ui/src/main/resources/catalog/js/admin/HarvestSettingsController.js index dbcd3d5fc9e..0b4a1c61e90 100644 --- a/web-ui/src/main/resources/catalog/js/admin/HarvestSettingsController.js +++ b/web-ui/src/main/resources/catalog/js/admin/HarvestSettingsController.js @@ -107,7 +107,7 @@ function loadHarvester(id) { $scope.isLoadingOneHarvester = true; - return $http.get("admin.harvester.list?_content_type=json&id=" + id).then( + return $http.get("../api/harvesters?id=" + id).then( function (response) { var data = response.data; @@ -300,12 +300,12 @@ }; $scope.cloneHarvester = function (id) { - $http.get("admin.harvester.clone?_content_type=json&id=" + id).then( + $http.put("../api/harvesters/" + id + "/clone").then( function (response) { $scope.$parent.loadHarvesters().then(function () { // Select the clone angular.forEach($scope.$parent.harvesters, function (h) { - if (h["@id"] === response.data[0]) { + if (h["@id"] == response.data) { $scope.selectHarvester(h); } }); @@ -472,32 +472,24 @@ $scope.deleteHarvester = function () { $scope.deleting.push($scope.harvesterSelected["@id"]); - return $http - .get( - "admin.harvester.remove?_content_type=json&id=" + - $scope.harvesterSelected["@id"] - ) - .then( - function (response) { - $scope.harvesterSelected = {}; - $scope.harvesterUpdated = false; - $scope.harvesterNew = false; - $scope.$parent.loadHarvesters(); + return $http.delete("../api/harvesters/" + $scope.harvesterSelected["@id"]).then( + function (response) { + $scope.harvesterSelected = {}; + $scope.harvesterUpdated = false; + $scope.harvesterNew = false; + $scope.$parent.loadHarvesters(); - $scope.deleting.splice($scope.deleting.indexOf(3), 1); - }, - function (response) { - console.log(response.data); - } - ); + $scope.deleting.splice($scope.deleting.indexOf(3), 1); + }, + function (response) { + console.log(response.data); + } + ); }; $scope.deleteHarvesterRecord = function () { return $http - .get( - "admin.harvester.clear?_content_type=json&id=" + - $scope.harvesterSelected["@id"] - ) + .put("../api/harvesters/" + $scope.harvesterSelected["@id"] + "/clear") .then( function (response) { $scope.harvesterSelected = {}; @@ -532,9 +524,7 @@ }; $scope.deleteHarvesterHistory = function () { return $http - .get( - "admin.harvester.history.delete?uuid=" + $scope.harvesterSelected.site.uuid - ) + .delete("../api/harvesters/" + $scope.harvesterSelected.site.uuid + "/history") .then(function (response) { $scope.$parent.loadHarvesters().then(function () { $scope.selectHarvester($scope.harvesterSelected); @@ -543,9 +533,7 @@ }; $scope.runHarvester = function () { return $http - .get( - "admin.harvester.run?_content_type=json&id=" + $scope.harvesterSelected["@id"] - ) + .put("../api/harvesters/" + $scope.harvesterSelected["@id"] + "/run") .then(function (response) { $scope.$parent.loadHarvesters().then(function () { refreshSelectedHarvester(); @@ -558,7 +546,12 @@ var id = $scope.harvesterSelected["@id"]; $scope.stopping = true; return $http - .get("admin.harvester.stop?_content_type=json&id=" + id + "&status=" + status) + .put( + "../api/harvesters/" + + $scope.harvesterSelected["@id"] + + "/stop?status=" + + status + ) .then(function () { $scope.$parent.loadHarvesters().then(refreshSelectedHarvester); $scope.stopping = false; @@ -574,29 +567,27 @@ } var status = $scope.harvesterSelected.options.status; - $http - .get( - "admin.harvester." + - (status === "active" ? "start" : "stop") + - "?_content_type=json&id=" + - $scope.harvesterSelected["@id"] - ) - .then( - function (response) { - deferred.resolve(response.data); - }, - function (response) { - var data = response.data; + var url = + "../api/harvesters/" + + $scope.harvesterSelected["@id"] + + "/" + + (status === "active" ? "start" : "stop"); + $http.put(url).then( + function (response) { + deferred.resolve(response.data); + }, + function (response) { + var data = response.data; - deferred.reject(data); - $rootScope.$broadcast("StatusUpdated", { - title: $translate.instant("harvesterSchedule" + status), - error: data, - timeout: 0, - type: "danger" - }); - } - ); + deferred.reject(data); + $rootScope.$broadcast("StatusUpdated", { + title: $translate.instant("harvesterSchedule" + status), + error: data, + timeout: 0, + type: "danger" + }); + } + ); return deferred.promise; }; diff --git a/web-ui/src/main/resources/catalog/templates/admin/harvest/harvest-settings.html b/web-ui/src/main/resources/catalog/templates/admin/harvest/harvest-settings.html index b758695c5d4..94b542ceca3 100644 --- a/web-ui/src/main/resources/catalog/templates/admin/harvest/harvest-settings.html +++ b/web-ui/src/main/resources/catalog/templates/admin/harvest/harvest-settings.html @@ -390,7 +390,7 @@

diff --git a/web/src/main/webapp/WEB-INF/config/config-service-admin-harvesting.xml b/web/src/main/webapp/WEB-INF/config/config-service-admin-harvesting.xml index 1f2fc59fc5a..06ac2275b1d 100644 --- a/web/src/main/webapp/WEB-INF/config/config-service-admin-harvesting.xml +++ b/web/src/main/webapp/WEB-INF/config/config-service-admin-harvesting.xml @@ -25,7 +25,7 @@ - + @@ -46,21 +46,21 @@ - + - + - + @@ -68,7 +68,7 @@ - + @@ -77,7 +77,7 @@ - + @@ -85,7 +85,7 @@ - + @@ -116,14 +116,14 @@ - + - +