From 2747193e2b9a347b88902375ddf4f6e9eac9326d Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 5 Aug 2022 15:37:10 +1000 Subject: [PATCH 01/22] #1445 added support for DwC-A Event Core updated API with link to archive 3.9-EXTENDED-SNAPSHOT --- build.gradle | 2 +- grails-app/conf/application.groovy | 952 +++++++++++++----- grails-app/conf/application.yml | 4 +- grails-app/conf/data/eventcore/eml.template | 15 + grails-app/conf/data/eventcore/meta.template | 22 + grails-app/conf/logback.groovy | 1 - .../org/ala/ecodata/RecordController.groovy | 24 + .../au/org/ala/ecodata/UrlMappings.groovy | 1 + .../domain/au/org/ala/ecodata/Document.groovy | 14 +- .../au/org/ala/ecodata/ProjectActivity.groovy | 2 + .../au/org/ala/ecodata/ActivityService.groovy | 27 +- .../au/org/ala/ecodata/MapService.groovy | 17 +- .../au/org/ala/ecodata/ProjectService.groovy | 14 +- .../au/org/ala/ecodata/RecordService.groovy | 412 +++++++- .../converter/GenericFieldConverter.groovy | 6 +- .../ecodata/converter/ImageConverter.groovy | 2 + .../ecodata/converter/ListConverter.groovy | 9 +- .../ecodata/converter/RecordConverter.groovy | 145 ++- .../converter/RecordFieldConverter.groovy | 79 +- .../ecodata/converter/SpeciesConverter.groovy | 16 +- 20 files changed, 1477 insertions(+), 287 deletions(-) create mode 100644 grails-app/conf/data/eventcore/eml.template create mode 100644 grails-app/conf/data/eventcore/meta.template diff --git a/build.gradle b/build.gradle index 8ab5d966c..eeb4513fc 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id 'com.craigburke.client-dependencies' version '1.4.0' } -version "3.9-SNAPSHOT" +version "3.9-EXTENDED-SNAPSHOT" group "au.org.ala" description "Ecodata" diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 7d888b2eb..3ea3a2545 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -1,3 +1,7 @@ +import au.org.ala.ecodata.Document +import au.org.ala.ecodata.Status +import org.joda.time.DateTime +import org.joda.time.DateTimeZone def appName = 'ecodata' @@ -47,7 +51,7 @@ environments { grails.mongodb.default.mapping = { version false - '*'(reference:true) + '*'(reference: true) } @@ -143,7 +147,7 @@ if (!biocollect.scienceType) { ] } -if(!biocollect.dataCollectionWhiteList){ +if (!biocollect.dataCollectionWhiteList) { biocollect.dataCollectionWhiteList = [ "Animals", "Biodiversity", @@ -419,12 +423,12 @@ if (!countries) { ] } -if(!spatial.geoJsonEnvelopeConversionThreshold){ +if (!spatial.geoJsonEnvelopeConversionThreshold) { spatial.geoJsonEnvelopeConversionThreshold = 1_000_000 } homepageIdx { - elasticsearch { + elasticsearch { fieldsAndBoosts { name = 50 description = 30 @@ -507,7 +511,7 @@ if (!collectory.baseURL) { if (!headerAndFooter.baseURL) { headerAndFooter.baseURL = "https://www.ala.org.au/commonui-bs3"//"https://www2.ala.org.au/commonui" } - if (!security.apikey.serviceUrl) { +if (!security.apikey.serviceUrl) { security.apikey.serviceUrl = 'https://auth.ala.org.au/apikey/ws/check?apikey=' } if (!biocacheService.baseURL) { @@ -587,7 +591,7 @@ security { casServerName = 'https://auth.ala.org.au' uriFilterPattern = ['/admin/*', '/activityForm/*'] authenticateOnlyIfLoggedInPattern = - uriExclusionFilterPattern = ['/assets/.*','/images/.*','/css/.*','/js/.*','/less/.*', '/activityForm/get.*'] + uriExclusionFilterPattern = ['/assets/.*', '/images/.*', '/css/.*', '/js/.*', '/less/.*', '/activityForm/get.*'] } } @@ -601,9 +605,9 @@ environments { app.elasticsearch.indexOnGormEvents = true grails.serverURL = "http://devt.ala.org.au:8080" app.uploads.url = "/document/download/" - grails.mail.host="localhost" - grails.mail.port=1025 - temp.dir="/data/ecodata/tmp" + grails.mail.host = "localhost" + grails.mail.port = 1025 + temp.dir = "/data/ecodata/tmp" } test { // Override disk store so the travis build doesn't fail. @@ -656,11 +660,11 @@ environments { wiremock.port = 8018 def casBaseUrl = "http://devt.ala.org.au:${wiremock.port}" - security.cas.casServerName="${casBaseUrl}" - security.cas.contextPath="" - security.cas.casServerUrlPrefix="${casBaseUrl}/cas" - security.cas.loginUrl="${security.cas.casServerUrlPrefix}/login" - security.cas.casLoginUrl="${security.cas.casServerUrlPrefix}/login" + security.cas.casServerName = "${casBaseUrl}" + security.cas.contextPath = "" + security.cas.casServerUrlPrefix = "${casBaseUrl}/cas" + security.cas.loginUrl = "${security.cas.casServerUrlPrefix}/login" + security.cas.casLoginUrl = "${security.cas.casServerUrlPrefix}/login" userDetails.url = "${casBaseUrl}/userdetails/" userDetails.admin.url = "${casBaseUrl}/userdetails/ws/admin" @@ -683,128 +687,128 @@ environments { facets.data = [ [ - name: "projectNameFacet", - title: 'Project', + name : "projectNameFacet", + title : 'Project', dataType: 'text', helpText: 'Name of the project' ], [ - name: "organisationNameFacet", - title: 'Organisation', + name : "organisationNameFacet", + title : 'Organisation', dataType: 'text', helpText: 'Organisations either running projects or associated with projects (eg. as partners).' ], [ - name: "projectActivityNameFacet", - title: 'Survey name', + name : "projectActivityNameFacet", + title : 'Survey name', dataType: 'text', helpText: 'Name of survey' ], [ - name: "recordNameFacet", - title: 'Species name', + name : "recordNameFacet", + title : 'Species name', dataType: 'text', helpText: 'Sighting\'s scientific name' ], [ - name: "userId", - title: 'Owner', + name : "userId", + title : 'Owner', dataType: 'text', helpText: 'User who created the record' ], [ - name: "embargoed", + name : "embargoed", dataType: 'text', - title: 'Unpublished records' + title : 'Unpublished records' ], [ - name: "activityLastUpdatedMonthFacet", - title: 'Month', + name : "activityLastUpdatedMonthFacet", + title : 'Month', dataType: 'text', helpText: 'Month the record was last edited' ], [ - name: "activityLastUpdatedYearFacet", - title: 'Year', + name : "activityLastUpdatedYearFacet", + title : 'Year', dataType: 'text', helpText: 'Year the record was last edited' ], [ - name: "surveyMonthFacet", - title: 'Month', + name : "surveyMonthFacet", + title : 'Month', dataType: 'text', helpText: 'Month the sighting was observed' ], [ - name: "surveyYearFacet", - title: 'Year', + name : "surveyYearFacet", + title : 'Year', dataType: 'text', helpText: 'Year the sighting was observed' ], [ - name: "individualCount", - title: 'Presence or Absence', - dataType: 'number', - helpText: 'Is species present or absent', + name : "individualCount", + title : 'Presence or Absence', + dataType : 'number', + helpText : 'Is species present or absent', facetTermType: 'PresenceOrAbsence' ], [ - name: "associatedProgramFacet", - title: 'Program Name', + name : "associatedProgramFacet", + title : 'Program Name', dataType: 'text', helpText: 'The administrative Program under which a project is being run.' ], [ - name: "siteNameFacet", - title: 'Site Name', + name : "siteNameFacet", + title : 'Site Name', dataType: 'text', helpText: 'A site at which data has been collected for one or projects.' ], [ - name: "associatedSubProgramFacet", - title: 'Sub Program', + name : "associatedSubProgramFacet", + title : 'Sub Program', dataType: 'text', helpText: 'Titles of sub-programmes under listed programmes.' ], [ - name: "methodType", - title: 'Method type', + name : "methodType", + title : 'Method type', dataType: 'text', helpText: '' ], [ - name: "spatialAccuracy", - title: 'Spatial accuracy confidence', + name : "spatialAccuracy", + title : 'Spatial accuracy confidence', dataType: 'text', helpText: '' ], [ - name: "speciesIdentification", - title: 'Species identification confidence', + name : "speciesIdentification", + title : 'Species identification confidence', dataType: 'text', helpText: '' ], [ - name: "temporalAccuracy", - title: 'Temporal accuracy confidence', + name : "temporalAccuracy", + title : 'Temporal accuracy confidence', dataType: 'text', helpText: '' ], [ - name: "nonTaxonomicAccuracy", - title: 'Non-taxonomic data accuracy', + name : "nonTaxonomicAccuracy", + title : 'Non-taxonomic data accuracy', dataType: 'text', helpText: '' ], [ - name: "dataQualityAssuranceMethods", - title: 'Data quality assurance method', + name : "dataQualityAssuranceMethods", + title : 'Data quality assurance method', dataType: 'text', helpText: '' ], [ - name: "isDataManagementPolicyDocumented", - title: 'Is data management policy documented?', + name : "isDataManagementPolicyDocumented", + title : 'Is data management policy documented?', dataType: 'text', helpText: '' ] @@ -813,219 +817,219 @@ facets.data = [ facets.project = [ [ - name: "isExternal", - title: 'Project', + name : "isExternal", + title : 'Project', dataType: 'text', helpText: 'Name of the project' ], [ - name: "isBushfire", - title: 'Bushfire Recovery', + name : "isBushfire", + title : 'Bushfire Recovery', dataType: 'text', helpText: 'Project associated to bushfire recovery' ], [ - name: "organisationFacet", - title: 'Organisation', + name : "organisationFacet", + title : 'Organisation', dataType: 'text', helpText: 'Organisations either running projects or associated with projects (eg. as partners).' ], [ - name: "uNRegions", - title: 'UN Regions', + name : "uNRegions", + title : 'UN Regions', dataType: 'text', helpText: 'The continental regions in which projects occur according to the United Nations regional classification scheme.' ], [ - name: "countries", - title: 'Countries', + name : "countries", + title : 'Countries', dataType: 'text', helpText: 'Countries in which people can participate in the project.' ], [ - name: "origin", - title: 'Source System', + name : "origin", + title : 'Source System', dataType: 'text', helpText: 'The project catalogue system in which the project is registered.' ], [ - name: "scienceType", - title: 'Science Type', + name : "scienceType", + title : 'Science Type', dataType: 'text', helpText: 'Categories of science which survey-based projects are addressing.' ], [ - name: "tags", - title: 'Tags', + name : "tags", + title : 'Tags', dataType: 'text', helpText: 'Classifications for citizen science projects to assist decision making for participation' ], [ - name: "difficulty", - title: 'Difficulty', + name : "difficulty", + title : 'Difficulty', dataType: 'text', helpText: 'A general level of difficulty for citizen science participation' ], [ - name: "status", - title: 'Status', + name : "status", + title : 'Status', dataType: 'text', helpText: 'Active projects are still running, whereas \'completed\' projects have ended and are no longer \'active\'' ], [ - name: "typeOfProject", - title: 'Project Types', + name : "typeOfProject", + title : 'Project Types', dataType: 'text', helpText: 'The project type reflects the hub in which projects were created, but will be either \'survey-based\' (Citizen and Ecological science) or \'schedule-based\' (Works and MERIT) formats for recording data.' ], [ - name: "ecoScienceType", - title: 'Science Type', + name : "ecoScienceType", + title : 'Science Type', dataType: 'text', helpText: 'Categories of science which survey-based projects are addressing.' ], [ - name: "associatedProgramFacet", - title: 'Program Name', + name : "associatedProgramFacet", + title : 'Program Name', dataType: 'text', helpText: 'The administrative Program under which a project is being run.' ], [ - name: "siteNameFacet", - title: 'Site Name', + name : "siteNameFacet", + title : 'Site Name', dataType: 'text', helpText: 'A site at which data has been collected for one or projects.' ], [ - name: "associatedSubProgramFacet", - title: 'Sub Program', + name : "associatedSubProgramFacet", + title : 'Sub Program', dataType: 'text', helpText: 'Titles of sub-programmes under listed programmes.' ], [ - name: "plannedStartDate", - title: 'Project Start Date', - dataType: 'date', - helpText: 'Selects projects that start between the specified date range.', + name : "plannedStartDate", + title : 'Project Start Date', + dataType : 'date', + helpText : 'Selects projects that start between the specified date range.', facetTermType: 'Date' ], [ - name: "fundingSourceFacet", - title: 'Funding source', + name : "fundingSourceFacet", + title : 'Funding source', dataType: 'text', helpText: '' ], [ - name: "reportingThemesFacet", - title: 'Reporting theme', + name : "reportingThemesFacet", + title : 'Reporting theme', dataType: 'text', helpText: '' ], [ - name: "typeFacet", - title: 'Activity type', + name : "typeFacet", + title : 'Activity type', dataType: 'text', helpText: '' ], [ - name: "assessment", - title: 'Assessment', + name : "assessment", + title : 'Assessment', dataType: 'text', helpText: '' ], [ - name: "stateFacet", - title: 'State', + name : "stateFacet", + title : 'State', dataType: 'text', helpText: 'Australian State' ], [ - name: "lgaFacet", - title: 'LGA', + name : "lgaFacet", + title : 'LGA', dataType: 'text', helpText: 'Local Government Areas' ], [ - name: "nrmFacet", - title: 'Management Areas', + name : "nrmFacet", + title : 'Management Areas', dataType: 'text', helpText: 'Natural Resource Management Areas' ], [ - name: "mvgFacet", - title: 'Major Vegetation Group', + name : "mvgFacet", + title : 'Major Vegetation Group', dataType: 'text', helpText: 'National Vegetation Information System Major Vegetation Group' ], [ - name: "mainThemeFacet", - title: 'Reporting Theme', + name : "mainThemeFacet", + title : 'Reporting Theme', dataType: 'text', helpText: '' ], [ - name: "ibraFacet", - title: 'Biogeographic Regions', + name : "ibraFacet", + title : 'Biogeographic Regions', dataType: 'text', helpText: '' ], [ - name: "imcra4_pbFacet", - title: 'Marine Regions', + name : "imcra4_pbFacet", + title : 'Marine Regions', helpText: '' ], [ - name: "otherFacet", - title: 'Other Regions', + name : "otherFacet", + title : 'Other Regions', dataType: 'text', helpText: '' ], [ - name: "electFacet", - title: 'Federal Electorates', + name : "electFacet", + title : 'Federal Electorates', dataType: 'text', helpText: '' ], [ - name: "cmzFacet", - title: 'CMZ', + name : "cmzFacet", + title : 'CMZ', dataType: 'text', helpText: 'Conservation Management Zones' ], [ - name: "meriPlanAssetFacet", - title: 'Assets Addressed', + name : "meriPlanAssetFacet", + title : 'Assets Addressed', dataType: 'text', helpText: 'Assets addressed in the MERI plan' ], [ - name: "projLifecycleStatus", - title: 'Project Lifecycle Status', + name : "projLifecycleStatus", + title : 'Project Lifecycle Status', dataType: 'text', helpText: 'Filters projects by project lifecycle status' ], [ - name: "partnerOrganisationTypeFacet", - title: 'Partner Organisations', + name : "partnerOrganisationTypeFacet", + title : 'Partner Organisations', dataType: 'text', helpText: 'Organisation type of partner organisations' ], [ - name: "industryFacet", - title: "Industry", + name : "industryFacet", + title : "Industry", helpText: "The industries relevant to the project" ], [ - name: "bushfireCategoriesFacet", - title: "Bushfire Categories", + name : "bushfireCategoriesFacet", + title : "Bushfire Categories", helpText: "The bushfire categories relevant to the project" ] ] license.default = "https://creativecommons.org/licenses/by-nc/3.0/au/" -projectActivity.notifyOnChange=true -biocollect.baseURL="https://biocollect.ala.org.au" -biocollect.projectActivityDataURL="${biocollect.baseURL}/bioActivity/projectRecords" +projectActivity.notifyOnChange = true +biocollect.baseURL = "https://biocollect.ala.org.au" +biocollect.projectActivityDataURL = "${biocollect.baseURL}/bioActivity/projectRecords" // elasticsearch cluster setting // can transport layer connection be made from apps outside JVM @@ -1050,99 +1054,99 @@ geoServer.elasticPort = "9300" geoServer.clusterName = "elasticsearch" geoServer.readTimeout = 600000 geoServer.layerNames = [ - "_general" : [ - "pa": [ name: "general", attributes: ['sites.geoIndex', 'sites.geometryType']], - "project": [ name: "generalproject", attributes: ['projectArea.geoIndex', 'projectArea.geometryType']], - ], - "_info" : [ - "pa": [ name: "layerinfo", - attributes: [ - 'sites.geoIndex', - 'dateCreated', - 'projectId', - 'thumbnailUrl', - 'activityId', - 'recordNameFacet', - 'projectActivityNameFacet', - 'projectNameFacet', - 'surveyMonthFacet', - 'surveyYearFacet', - 'sites.geometryType' - ] + "_general": [ + "pa" : [name: "general", attributes: ['sites.geoIndex', 'sites.geometryType']], + "project": [name: "generalproject", attributes: ['projectArea.geoIndex', 'projectArea.geometryType']], + ], + "_info" : [ + "pa" : [name : "layerinfo", + attributes: [ + 'sites.geoIndex', + 'dateCreated', + 'projectId', + 'thumbnailUrl', + 'activityId', + 'recordNameFacet', + 'projectActivityNameFacet', + 'projectNameFacet', + 'surveyMonthFacet', + 'surveyYearFacet', + 'sites.geometryType' + ] ], - "project": [ name: "layerinfoproject", - attributes: [ - 'projectArea.geoIndex', - 'projectArea.geometryType', - 'name', - 'aim', - 'projectId', - 'imageUrl', - 'logoAttribution', - 'plannedStartDate', - 'plannedEndDate' - ] + "project": [name : "layerinfoproject", + attributes: [ + 'projectArea.geoIndex', + 'projectArea.geometryType', + 'name', + 'aim', + 'projectId', + 'imageUrl', + 'logoAttribution', + 'plannedStartDate', + 'plannedEndDate' + ] ], ], - "_time": [ - "pa": [ name: "time", attributes: ['sites.geoIndex', 'sites.geometryType']] + "_time" : [ + "pa": [name: "time", attributes: ['sites.geoIndex', 'sites.geometryType']] ], "_indices": [ - "pa": [ name: "colour_by", attributes: ['sites.geometryType']], - "project": [ name: "colour_byproject", attributes: ['projectArea.geometryType']], + "pa" : [name: "colour_by", attributes: ['sites.geometryType']], + "project": [name: "colour_byproject", attributes: ['projectArea.geometryType']], ] ] geoServer.layerConfiguration = [ "pasearch": [ - "name": "layerName", - "nativeName": "layerNativeName", - "title": "BioCollect survey activity", - "keywords": ["activity", "survey", "biocollect"], - "timeEnabled": false, + "name" : "layerName", + "nativeName" : "layerNativeName", + "title" : "BioCollect survey activity", + "keywords" : ["activity", "survey", "biocollect"], + "timeEnabled" : false, "timeAttribute": "dateCreated", - "attributes": [ + "attributes" : [ [ - "name": "sites.geoIndex", - "shortName": "sites.geoIndex", - "useShortName": false, - "type": "com.vividsolutions.jts.geom.Geometry", - "use": true, + "name" : "sites.geoIndex", + "shortName" : "sites.geoIndex", + "useShortName" : false, + "type" : "com.vividsolutions.jts.geom.Geometry", + "use" : true, "defaultGeometry": true, - "geometryType": "GEO_SHAPE", - "srid": "4326", - "stored": false, - "nested": false, - "binding": "com.vividsolutions.jts.geom.Geometry", - "nillable": true, - "minOccurs": 0, - "maxOccurs": 1 + "geometryType" : "GEO_SHAPE", + "srid" : "4326", + "stored" : false, + "nested" : false, + "binding" : "com.vividsolutions.jts.geom.Geometry", + "nillable" : true, + "minOccurs" : 0, + "maxOccurs" : 1 ] ] ], "homepage": [ - "name": "layerName", - "nativeName": "layerNativeName", - "title": "BioCollect survey activity", - "keywords": ["activity", "survey", "biocollect"], - "timeEnabled": false, + "name" : "layerName", + "nativeName" : "layerNativeName", + "title" : "BioCollect survey activity", + "keywords" : ["activity", "survey", "biocollect"], + "timeEnabled" : false, "timeAttribute": "dateCreated", - "attributes": [ + "attributes" : [ [ - "name": "projectArea.geoIndex", - "shortName": "projectArea.geoIndex", - "useShortName": false, - "type": "com.vividsolutions.jts.geom.Geometry", - "use": true, + "name" : "projectArea.geoIndex", + "shortName" : "projectArea.geoIndex", + "useShortName" : false, + "type" : "com.vividsolutions.jts.geom.Geometry", + "use" : true, "defaultGeometry": true, - "geometryType": "GEO_SHAPE", - "srid": "4326", - "stored": false, - "nested": false, - "binding": "com.vividsolutions.jts.geom.Geometry", - "nillable": true, - "minOccurs": 0, - "maxOccurs": 1 + "geometryType" : "GEO_SHAPE", + "srid" : "4326", + "stored" : false, + "nested" : false, + "binding" : "com.vividsolutions.jts.geom.Geometry", + "nillable" : true, + "minOccurs" : 0, + "maxOccurs" : 1 ] ] ] @@ -1167,75 +1171,75 @@ if (!geoserver.facetRangeColour) { geohash.lookupTable = [ [ length: 1, - width: 5009400, + width : 5009400, height: 4992600, - area:25009930440000 + area : 25009930440000 ], [ length: 2, - width: 1252300, + width : 1252300, height: 624100, - area: 781560430000 + area : 781560430000 ], [ length: 3, - width: 156500, + width : 156500, height: 156000, - area: 24414000000 + area : 24414000000 ], [ length: 4, - width: 39100, + width : 39100, height: 19500, - area: 762450000 + area : 762450000 ], [ length: 5, - width: 4900, + width : 4900, height: 4900, - area: 24010000 + area : 24010000 ], [ length: 6, - width: 1200, + width : 1200, height: 609.4, - area: 731280 + area : 731280 ], [ length: 7, - width: 152.9, + width : 152.9, height: 152.4, - area: 23301.96 + area : 23301.96 ], [ length: 8, - width: 38.2, + width : 38.2, height: 19, - area: 725.8 + area : 725.8 ], [ length: 9, - width: 4.8, + width : 4.8, height: 4.8, - area: 23.04 + area : 23.04 ], [ length: 10, - width: 1.2, + width : 1.2, height: 0.0595, - area: 0.0714 + area : 0.0714 ], [ length: 11, - width: 0.0149, + width : 0.0149, height: 0.0149, - area: 0.00022201 + area : 0.00022201 ], [ length: 12, - width: 0.0037, + width : 0.0037, height: 0.0019, - area: 0.00000703 + area : 0.00000703 ] ] @@ -1244,7 +1248,7 @@ geohash.maxNumberOfGrids = 250 // Sets the maximum precision geohash grid. // Using higher precision will be able to narrow the record to precise location. Use lower precision if the aim is to // hide exact location. -geohash.maxLength = 5 +geohash.maxLength = 5 // Dummy / default username and password for elasticsearch, will be ignored if the server is not setup for // basic authentication. @@ -1252,3 +1256,489 @@ elasticsearch { username = 'elastic' password = 'password' } + +if (!darwinCore.projectActivityToDwC) { + darwinCore.projectActivityToDwC = [ + "Event": [ + [ + "name": "eventID", + "ref" : "projectActivityId" + ], + [ + "name": "eventDate", + "code" : { record, params -> + def pActivity = params.pActivity + if (pActivity.startDate && pActivity.endDate) { + return params.recordService.toStringIsoDateTime(pActivity.startDate) + "/" + params.recordService.toStringIsoDateTime(pActivity.endDate) + } + + return "" + } + ], + [ + "name": "eventRemarks", + "ref" : "description" + ], + [ + "name": "samplingProtocol", + "ref" : "methodName" + ], + [ + "name" : "eventType", + "default": "Survey" + ], + [ + "name" : "name" + ], + [ + "name" : "license", + "ref" : "attribution" + ], + [ + "name" : "pActivityFormName" + ], + [ + "name" : "startDate", + "code" : { record, params -> + def pActivity = params.pActivity + if (pActivity.startDate) { + return params.recordService.toStringIsoDateTime(pActivity.startDate) + } + + return "" + } + ], + [ + "name" : "endDate", + "code" : { record, params -> + def pActivity = params.pActivity + if (pActivity.endDate) { + return params.recordService.toStringIsoDateTime(pActivity.endDate) + } + + return "" + } + ], + [ + "name" : "legalCustodianOrganisation" + ], + [ + "name" : "dataAccessExternalURL" + ], + [ + "name" : "dataManagementPolicyDescription" + ], + [ + "name" : "dataManagementPolicyURL" + ], + [ + "name" : "dataManagementPolicyDocument" + ], + [ + "name" : "dataSharingLicense" + ] + ] + ] +} + +if (!darwinCore.termsGroupedByClass) { + darwinCore.termsGroupedByClass = [ + "Dataset" : [ + ], + "Event" : [ + [ + "name" : "eventID", + "namespace": "dwc" + ], + [ + "name" : "parentEventID", + "namespace": "dwc" + ], + [ + "name" : "eventType", + "namespace": "dwc", + "default" : "SiteVisit" + ], + [ + "name" : "eventDate", + "namespace": "dwc", + "code" : { record, params -> + if (record.eventTime && record.eventDate instanceof Date) { + Date date = record.eventDate + DateTime dt = new DateTime(date, DateTimeZone.UTC) + dt = dt.withTimeAtStartOfDay() + return dt.toDateTimeISO().toString() + } + else if (record.eventDate) + return params.recordService.toStringIsoDateTime (record.eventDate) + } + ], + [ + "name" : "eventTime", + "namespace": "dwc" + ], + [ + "name" : "endDayOfYear", + "namespace": "dwc" + ], + [ + "name" : "startDayOfYear", + "namespace": "dwc" + ], + [ + "name" : "verbatimEventDate", + "namespace": "dwc" + ], + [ + "name" : "day", + "namespace": "dwc" + ], + [ + "name" : "month", + "namespace": "dwc" + ], + [ + "name" : "year", + "namespace": "dwc" + ], + [ + "name" : "eventRemarks", + "namespace": "dwc" + ], + [ + "name" : "fieldNotes", + "namespace": "dwc" + ], + [ + "name" : "fieldNumber", + "namespace": "dwc" + ], + [ + "name" : "habitat", + "namespace": "dwc" + ], + [ + "name" : "sampleSizeUnit", + "namespace": "dwc" + ], + [ + "name" : "sampleSizeValue", + "namespace": "dwc" + ], + [ + "name" : "samplingEffort", + "namespace": "dwc" + ], + [ + "name" : "samplingProtocol", + "namespace": "dwc" + ], + [ + "name": "decimalLatitude" + ], + [ + "name": "decimalLongitude" + ], + [ + "name" : "geodeticDatum", + "default": "EPSG:4326" + ], + [ + "name": "coordinateUncertaintyInMeters" + ], + [ + "name": "footprintWKT" + ], + [ + "name" : "geodeticDatum", + "default": "EPSG:4326" + ], + [ + "name": "locationID", + "code": { Map record, Map params -> + params?.site?.siteId + } + ] + ], + "FossilSpecimen" : [ + ], + "GeologicalContext" : [ + ], + "HumanObservation" : [ + ], + "Identification" : [ + ], + "LivingSpecimen" : [ + ], + "Location" : [ + ], + "MachineObservation" : [ + ], + "MaterialCitation" : [ + ], + "MaterialSample" : [ + ], + "MeasurementOrFact" : [ + [ + "name" : "measurementsorfacts", + "substitute": { record, params -> + record.measurementsorfacts ?: [] + }, + order: [ + "eventID", + "occurrenceID", + "measurementValue", + "measurementAccuracy", + "measurementUnit", + "measurementUnitID", + "measurementType", + "measurementTypeID", + "measurementID" + ] + ] + ], + "Media" : [ + [ + "name" : "multimedia", + "substitute": { record, params -> + record?.multimedia?.collect { multimedia -> + def identifier = multimedia?.identifier + + if (multimedia.documentId) { + def document = Document.findByDocumentIdAndStatusNotEqual(multimedia.documentId, Status.DELETED) + if (document) { + identifier = document.getUrl() + multimedia = document + return [ + "eventID" : params?.activity?.activityId, + "occurrenceID": record?.outputSpeciesId , + "type" : multimedia?.type, + "identifier" : identifier, + "format" : multimedia?.contentType, + "creator" : multimedia?.creator, + "licence" : multimedia?.license, + "rightsHolder": multimedia?.rightsHolder + ] + } + } + } + }, + "order": [ + "eventID", + "occurrenceID", + "type", + "identifier", + "format", + "creator", + "licence", + "rightsHolder" + ] + ] + ], + "Occurrence" : [ + [ + "name" : "eventID", + "namespace": "dwc" + ], + [ + "name" : "occurrenceID", + "namespace": "dwc", + "code" : { record, params -> + record?.outputSpeciesId + } + ], + [ + "name": "basisOfRecord" + ], + [ + "name": "scientificName" + ], + [ + "name" : "occurrenceStatus", + "namespace": "dwc", + "code" : { record, params -> + String count = record?.individualCount?.toString()?.trim() ?: "1" + try { + Integer.parseInt(count) + } + catch (NumberFormatException nfe) { + count == null + } + + count == "0" ? "absent" : "present" + } + ], + [ + "name" : "individualCount", + "namespace": "dwc", + "code": { record, params -> + String count = record?.individualCount?.toString()?.trim() ?: "1" + try { + Integer.parseInt(count) + return count + } + catch (NumberFormatException nfe) { + + } + + return "1" + } + ], + [ + "name" : "occurrenceRemarks", + "namespace": "dwc" + ], + [ + "name" : "associatedMedia", + "namespace": "dwc" + ], + [ + "name" : "associatedOccurrences", + "namespace": "dwc" + ], + [ + "name" : "associatedReferences", + "namespace": "dwc" + ], + [ + "name" : "associatedSequences", + "namespace": "dwc" + ], + [ + "name" : "associatedTaxa", + "namespace": "dwc" + ], + [ + "name" : "behavior", + "namespace": "dwc" + ], + [ + "name" : "catalogNumber", + "namespace": "dwc" + ], + [ + "name" : "degreeOfEstablishment", + "namespace": "dwc" + ], + [ + "name" : "disposition", + "namespace": "dwc" + ], + [ + "name" : "establishmentMeans", + "namespace": "dwc" + ], + [ + "name" : "georeferenceVerificationStatus", + "namespace": "dwc" + ], + [ + "name" : "lifeStage", + "namespace": "dwc" + ], + [ + "name" : "organismQuantity", + "namespace": "dwc" + ], + [ + "name" : "organismQuantityType", + "namespace": "dwc" + ], + [ + "name" : "otherCatalogNumbers", + "namespace": "dwc" + ], + [ + "name" : "pathway", + "namespace": "dwc" + ], + [ + "name" : "preparations", + "namespace": "dwc" + ], + [ + "name" : "recordedBy", + "namespace": "dwc" + ], + [ + "name" : "recordedByID", + "namespace": "dwc" + ], + [ + "name" : "recordNumber", + "namespace": "dwc" + ], + [ + "name" : "reproductiveCondition", + "namespace": "dwc" + ], + [ + "name" : "sex", + "namespace": "dwc" + ], + ], + "Organism" : [ + ], + "PreservedSpecimen" : [ + ], + "ResourceRelationship": [ + ], + "Taxon" : [ + ] + ] + + +} + +if (!darwinCore.namespaces) { + darwinCore.namespaces = [ + Archive : "http://rs.tdwg.org/dwc/text/", + Event : "http://rs.tdwg.org/dwc/terms/Event", + Occurrence : "http://rs.tdwg.org/dwc/terms/Occurrence", + MeasurementOrFact : "http://rs.iobis.org/obis/terms/ExtendedMeasurementOrFact", + Media : "http://rs.gbif.org/terms/1.0/Multimedia", + eventID : "http://rs.tdwg.org/dwc/terms/eventID", + parentEventID : "http://rs.tdwg.org/dwc/terms/parentEventID", + eventType : "eventType", + eventTime : "http://rs.tdwg.org/dwc/terms/eventTime", + eventDate : "http://rs.tdwg.org/dwc/terms/eventDate", + samplingProtocol : "http://rs.tdwg.org/dwc/terms/samplingProtocol", + sampleSizeValue : "http://rs.tdwg.org/dwc/terms/sampleSizeValue", + sampleSizeUnit : "http://rs.tdwg.org/dwc/terms/sampleSizeUnit", + samplingEffort : "http://rs.tdwg.org/dwc/terms/samplingEffort", + locationID : "http://rs.tdwg.org/dwc/terms/locationID", + decimalLatitude : "http://rs.tdwg.org/dwc/terms/decimalLatitude", + decimalLongitude : "http://rs.tdwg.org/dwc/terms/decimalLongitude", + geodeticDatum : "http://rs.tdwg.org/dwc/terms/geodeticDatum", + coordinateUncertaintyInMeters: "http://rs.tdwg.org/dwc/terms/coordinateUncertaintyInMeters", + footprintWKT : "http://rs.tdwg.org/dwc/terms/footprintWKT", + geodeticDatum : "http://rs.tdwg.org/dwc/terms/geodeticDatum", + eventRemarks : "http://rs.tdwg.org/dwc/terms/eventRemarks", + occurrenceID : "http://rs.tdwg.org/dwc/terms/occurrenceID", + basisOfRecord : "http://rs.tdwg.org/dwc/terms/basisOfRecord", + recordedBy : "http://rs.tdwg.org/dwc/terms/recordedBy", + individualCount : "http://rs.tdwg.org/dwc/terms/individualCount", + organismQuantity : "http://rs.tdwg.org/dwc/terms/organismQuantity", + organismQuantityType : "http://rs.tdwg.org/dwc/terms/organismQuantityType", + occurrenceStatus : "http://rs.tdwg.org/dwc/terms/occurrenceStatus", + scientificName : "http://rs.tdwg.org/dwc/terms/scientificName", + kingdom : "http://rs.tdwg.org/dwc/terms/kingdom", + family : "http://rs.tdwg.org/dwc/terms/family", + measurementID : "http://rs.tdwg.org/dwc/terms/measurementID", + measurementType : "http://rs.tdwg.org/dwc/terms/measurementType", + measurementTypeID : "http://rs.iobis.org/obis/terms/measurementTypeID", + measurementValue : "http://rs.tdwg.org/dwc/terms/measurementValue", + measurementAccuracy : "http://rs.tdwg.org/dwc/terms/measurementAccuracy", + measurementUnit : "http://rs.tdwg.org/dwc/terms/measurementUnit", + measurementUnitID : "http://rs.iobis.org/obis/terms/measurementUnitID", + measurementDeterminedDate : "http://rs.tdwg.org/dwc/terms/measurementDeterminedDate", + measurementDeterminedBy : "http://rs.tdwg.org/dwc/terms/measurementDeterminedBy", + measurementRemarks : "http://rs.tdwg.org/dwc/terms/measurementRemarks", + type : "http://purl.org/dc/terms/type", + identifier : "http://purl.org/dc/terms/identifier", + format : "http://purl.org/dc/terms/format", + creator : "http://purl.org/dc/terms/creator", + license : "http://purl.org/dc/terms/license", + rightsHolder : "http://purl.org/dc/terms/rightsHolder" + ] +} \ No newline at end of file diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index cb49ca3f5..891ae7280 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -210,4 +210,6 @@ grails: codecs: - au.org.ala.ecodata.customcodec.AccessLevelCodec - +--- +image: + baseURL: "https://images.ala.org.au" \ No newline at end of file diff --git a/grails-app/conf/data/eventcore/eml.template b/grails-app/conf/data/eventcore/eml.template new file mode 100644 index 000000000..3d76a251c --- /dev/null +++ b/grails-app/conf/data/eventcore/eml.template @@ -0,0 +1,15 @@ + + + ${name} + + ${organisationName} + + + Atlas of Living Australia + + ${dateCreated} + + ${description} + + + diff --git a/grails-app/conf/data/eventcore/meta.template b/grails-app/conf/data/eventcore/meta.template new file mode 100644 index 000000000..e13d45ce2 --- /dev/null +++ b/grails-app/conf/data/eventcore/meta.template @@ -0,0 +1,22 @@ + + + + ${core.location} + + + core.fields?.each { + + } + + extensions?.each { + + + ${it.location} + + + it.fields?.each { + + } + + } + diff --git a/grails-app/conf/logback.groovy b/grails-app/conf/logback.groovy index 932ae2b90..cbdcc5cd8 100644 --- a/grails-app/conf/logback.groovy +++ b/grails-app/conf/logback.groovy @@ -1,4 +1,3 @@ -import grails.util.BuildSettings import grails.util.Environment import org.springframework.boot.logging.logback.ColorConverter import org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index 7bb16cf0c..13d3a4f30 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -1,6 +1,9 @@ package au.org.ala.ecodata import org.apache.http.HttpStatus +import org.apache.http.entity.ContentType +import org.grails.web.servlet.mvc.GrailsWebRequest +import org.springframework.web.context.request.RequestAttributes import java.text.SimpleDateFormat @@ -555,6 +558,27 @@ class RecordController { } } + /** + * Get Darwin Core Archive for a project that has ala harvest enabled. + * @param projectId + * @return + */ + @RequireApiKey + def getDarwinCoreArchiveForProject (String projectId) { + if (projectId) { + Project project = Project.findByProjectId(projectId) + if(project?.alaHarvest) { + // Simulate BioCollect as the hostname calling this method. This is done to get the correct URL for + // documents. + GrailsWebRequest.lookup().setAttribute(DocumentHostInterceptor.DOCUMENT_HOST_NAME, grailsApplication.config.getProperty("biocollect.baseURL"), RequestAttributes.SCOPE_REQUEST) + recordService.getDarwinCoreArchiveForProject(response.outputStream, project) + } else + response status: HttpStatus.SC_NOT_FOUND, text: [error: "project not found or ala harvest flag is switched off"] as JSON, contentType: ContentType.APPLICATION_JSON + } else { + response status: HttpStatus.SC_BAD_REQUEST, text: [error: "projectId is required"] as JSON, contentType: ContentType.APPLICATION_JSON + } + } + private def setResponseHeadersForRecord(response, record) { response.addHeader("content-location", grailsApplication.config.grails.serverURL + "/record/" + record.occurrenceID) response.addHeader("location", grailsApplication.config.grails.serverURL + "/record/" + record.occurrenceID) diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index 9dbac8e5c..2cd6b50c8 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -185,6 +185,7 @@ class UrlMappings { "/ws/project/getDataCollectionWhiteList"(controller: "project"){ action = [GET:"getDataCollectionWhiteList"] } "/ws/project/getBiocollectFacets"(controller: "project"){ action = [GET:"getBiocollectFacets"] } "/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets") + "/ws/project/$projectId/archive"(controller: "record", action: "getDarwinCoreArchiveForProject") "/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch") "/ws/document/download"(controller:"document", action:"download") diff --git a/grails-app/domain/au/org/ala/ecodata/Document.groovy b/grails-app/domain/au/org/ala/ecodata/Document.groovy index d3df8cf45..0c35319d1 100644 --- a/grails-app/domain/au/org/ala/ecodata/Document.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Document.groovy @@ -68,6 +68,7 @@ class Document { String identifier /* To be replaced by reportId */ String stage + String imageId boolean thirdPartyConsentDeclarationMade = false String thirdPartyConsentDeclarationText @@ -123,7 +124,11 @@ class Document { return '' } - if(hosted == ALA_IMAGE_SERVER){ + if (imageId) { + return getImageURL() + } + + if (hosted == ALA_IMAGE_SERVER) { return identifier } @@ -145,6 +150,12 @@ class Document { } + String getImageURL () { + if (imageId) { + Holders.getGrailsApplication().config.getProperty("image.baseURL") + "/proxyImage?id=" + imageId + } + } + static constraints = { name nullable: true attribution nullable: true @@ -178,5 +189,6 @@ class Document { identifier nullable: true contentType nullable: true hubId nullable: true + imageId nullable: true } } diff --git a/grails-app/domain/au/org/ala/ecodata/ProjectActivity.groovy b/grails-app/domain/au/org/ala/ecodata/ProjectActivity.groovy index 8929b5bb7..f3efc1a13 100644 --- a/grails-app/domain/au/org/ala/ecodata/ProjectActivity.groovy +++ b/grails-app/domain/au/org/ala/ecodata/ProjectActivity.groovy @@ -45,6 +45,7 @@ class ProjectActivity { MapLayersConfiguration mapLayersConfig String surveySiteOption boolean canEditAdminSelectedSites + boolean published Date dateCreated Date lastUpdated @@ -84,6 +85,7 @@ class ProjectActivity { mapLayersConfig nullable: true surveySiteOption nullable: true, inList: ['sitepick','sitecreate', 'sitepickcreate'] canEditAdminSelectedSites nullable: true + published nullable: true } static mapping = { diff --git a/grails-app/services/au/org/ala/ecodata/ActivityService.groovy b/grails-app/services/au/org/ala/ecodata/ActivityService.groovy index 74750e165..15188b822 100644 --- a/grails-app/services/au/org/ala/ecodata/ActivityService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ActivityService.groovy @@ -1,11 +1,9 @@ package au.org.ala.ecodata + +import au.org.ala.ecodata.metadata.OutputMetadata +import au.org.ala.ecodata.metadata.OutputModelProcessor import com.mongodb.BasicDBObject -import com.mongodb.DBCursor -import com.mongodb.DBObject -import com.mongodb.client.model.Filters -import org.bson.conversions.Bson import org.grails.datastore.mapping.query.api.BuildableCriteria -import au.org.ala.ecodata.metadata.* import javax.persistence.PessimisticLockException @@ -649,4 +647,23 @@ class ActivityService { } } + /** + * An activity is embargoed if either of the below conditions are satisfied + * 1. embargoed flag is set to true on an activity + * 2. project activity is embargoed until a defined date + * @param activity + * @param projectActivity + * @return + */ + boolean isActivityEmbargoed(Activity activity, ProjectActivity projectActivity){ + if (activity.embargoed) { + return activity.embargoed + } + + if (projectActivity?.visibility?.embargoUntil) { + return projectActivity?.visibility?.embargoUntil.after(new Date()) + } + + return false + } } diff --git a/grails-app/services/au/org/ala/ecodata/MapService.groovy b/grails-app/services/au/org/ala/ecodata/MapService.groovy index 9a7faa2d2..8dfe31e5e 100644 --- a/grails-app/services/au/org/ala/ecodata/MapService.groovy +++ b/grails-app/services/au/org/ala/ecodata/MapService.groovy @@ -5,14 +5,12 @@ import com.spatial4j.core.context.SpatialContext import com.spatial4j.core.io.GeohashUtils import com.spatial4j.core.shape.Rectangle import grails.converters.JSON -import groovy.json.JsonSlurper import grails.web.http.HttpHeaders +import groovy.json.JsonSlurper import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse import org.elasticsearch.action.search.SearchRequest import org.elasticsearch.common.Strings -import org.elasticsearch.common.io.stream.OutputStreamStreamOutput -import org.elasticsearch.common.xcontent.XContentHelper import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGrid import org.springframework.beans.factory.annotation.Autowired @@ -622,16 +620,23 @@ class MapService { webService.doDelete(url, headers) } - String bindDataToXMLTemplate (String fileClassPath, Map data) { + String bindDataToXMLTemplate (String fileClassPath, Map data, boolean prettyPrint = false) { def files = resourceResolver.getResources(fileClassPath) def engine = new groovy.text.XmlTemplateEngine() - engine.setIndentation('') + if (!prettyPrint) + engine.setIndentation("") + else + engine.setIndentation(" ") + String content files?.each { Resource file -> content = engine.createTemplate(file.getURL()).make(data).toString() } - content?.replaceAll('\n', ''); + if (prettyPrint) + return content + else + return content?.replaceAll('\n', '') } def buildStyleForTermFacet(String field, List terms, String style, String dataStore) { diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index b7219e798..566fdc98a 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -135,7 +135,19 @@ class ProjectService { order(params.sort, params.order) } - [total: list?.totalCount, list: list?.collect { toMap(it, "basic") }] + def total = list?.totalCount + list = list?.collect { toMap(it, "basic") } + addArchiveLink(list) + [total: total, list: list] + } + + /** + * Adds archive URL to projects + * @param projects + * @return + */ + def addArchiveLink (List projects) { + projects?.each { it.archiveURL = grailsApplication.config.getProperty("grails.serverURL") + "/ws/project/${it.projectId}/archive" } } diff --git a/grails-app/services/au/org/ala/ecodata/RecordService.groovy b/grails-app/services/au/org/ala/ecodata/RecordService.groovy index 59741836f..f78f76542 100644 --- a/grails-app/services/au/org/ala/ecodata/RecordService.groovy +++ b/grails-app/services/au/org/ala/ecodata/RecordService.groovy @@ -1,6 +1,8 @@ package au.org.ala.ecodata + import au.com.bytecode.opencsv.CSVWriter +import au.org.ala.ecodata.converter.RecordConverter import au.org.ala.web.AuthService import grails.converters.JSON import groovy.json.JsonSlurper @@ -18,9 +20,11 @@ import org.joda.time.DateTimeZone import org.joda.time.format.DateTimeFormatter import org.joda.time.format.ISODateTimeFormat +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + import static au.org.ala.ecodata.Status.ACTIVE import static au.org.ala.ecodata.Status.DELETED - /** * Services for handling the creation of records with images. */ @@ -39,9 +43,12 @@ class RecordService { SensitiveSpeciesService sensitiveSpeciesService DocumentService documentService CommonService commonService + MapService mapService final def ignores = ["action", "controller", "associatedMedia"] private static final List EXCLUDED_RECORD_PROPERTIES = ["_id", "activityId", "dateCreated", "json", "outputId", "projectActivityId", "projectId", "status", "dataResourceUid"] + static final List MULTIMEDIA_DATA_TYPES = ['image', 'audio', 'document', 'photoPoints'] + static final String DWC_EVENT = 'Event', DWC_MEDIA = 'Media', DWC_MEASUREMENT = 'MeasurementOrFact', DWC_OCCURRENCE = 'Occurrence' def getProjectActivityService() { grailsApplication.mainContext.projectActivityService @@ -253,7 +260,7 @@ class RecordService { } def getAllRecordsByActivityList(List activityList) { - Record.findAllByActivityIdInList(activityList).collect { toMap(it) } + Record.findAllByActivityIdInListAndStatusNotEqual(activityList, Status.DELETED).collect { toMap(it) } } /** @@ -942,4 +949,405 @@ class RecordService { return grailsApplication.config.license.default; } + + /** + * Creates Darwin Core Archive for a project. Output of this function are + * | - meta.xml + * | - eml.xml + * | - Event.csv + * | - MeasurementOrFact.csv + * | - Media.csv + * | - Occurrence.csv + * @param outputStream + * @param project + */ + void getDarwinCoreArchiveForProject(outputStream, Project project) { + Map result + Map headersByDwcClass = [:].withDefault {[]} + new ZipOutputStream(outputStream).withStream { zip -> + try { + Long start = System.currentTimeMillis(), end + result = generateEventCoreFiles (project) + + result.each { dwcClass, rows -> + if (rows.size()) { + zip.putNextEntry(new ZipEntry("${dwcClass}.csv")) + CSVWriter csvWriter = new CSVWriter(new OutputStreamWriter(zip)) + List headers = getHeadersFromRows(rows) + List defaultOrder = getHeaderOrder(grailsApplication.config.darwinCore.termsGroupedByClass[dwcClass]) + headers = reorderHeaders(headers, defaultOrder) + headersByDwcClass[dwcClass] = headers + csvWriter.writeNext(headers as String[]) + rows.each { row -> + List line = headers.collect { + row[it] instanceof Collection ? row[it].join(" | ") : row[it] ?: "" + } + csvWriter.writeNext(line as String[]) + } + + csvWriter.flush() + log.info("finished writing ${dwcClass}.csv") + } + } + end = System.currentTimeMillis() + log.debug("Time in milliseconds to write event core CSVs- ${end - start}") + start = end + + zip.putNextEntry(new ZipEntry("meta.xml")) + zip << getMetaXML(headersByDwcClass) + zip.closeEntry() + + zip.putNextEntry(new ZipEntry("eml.xml")) + zip << getEmlXML(project) + zip.closeEntry() + end = System.currentTimeMillis() + log.debug("Time in milliseconds to write event core XMLs- ${end - start}") + + log.debug("completed zip") + } catch (Exception e){ + log.error("error creating darwin core archive", e) + } finally { + zip.finish() + zip.flush() + zip.close() + } + } + } + + /** + * Creates a data structure that enables creation of Darwin Core Archive. Project activity is a survey type in Event table. + * [ + * 'Event' : [[a: 'b']], + * 'MeasurementOrFact' : [[a: 'b']], + * 'Media' : [[a: 'b']], + * 'Occurrence' : [[a: 'b']] + * ] + * @param project + * @return + */ + Map generateEventCoreFiles(Project project) { + Map result = [:].withDefault {[]} + Organisation organisation = project?.organisationName ? Organisation.findByName(project?.organisationName) : null + List projectActivitiesForProject = ProjectActivity.findAllByProjectIdAndStatusNotEqualAndPublished(project.projectId, Status.DELETED, true) + projectActivitiesForProject.each { ProjectActivity projectActivity -> + int batchSize = 100 + int offset = 0 + int size = batchSize + + // adds project activity to event table. + result[DWC_EVENT].add(convertProjectActivityToEvent(projectActivity, project)) + while ( batchSize == size ) { + List activities = Activity.findAllByProjectIdAndStatusNotEqualAndProjectActivityId(project.projectId, Status.DELETED, projectActivity.projectActivityId, [max: batchSize, offset: offset]) + activities.each { Activity activity -> + if(activityService.isActivityEmbargoed(activity, projectActivity)) + return + + Site site = activity.siteId ? Site.findBySiteId(activity.siteId) : null + List outputs = Output.findAllByActivityId(activity.activityId) + outputs.eachWithIndex { output, outputIndex -> + Map props = outputService.toMap(output) + Map outputMetadata = metadataService.getOutputDataModelByName(props.name) as Map + try { + List records = RecordConverter.convertRecords(project, organisation, site, projectActivity, activity, output, props.data, outputMetadata, false) + records.eachWithIndex { record, recordIndex -> + Map params = [ + activity: activity, + site: site, + output: output, + data: props.data, + metadata: outputMetadata, + organisation: organisation, + project: project, + recordService: this, + index: "${activity.activityId}-${outputIndex}-${recordIndex}" + ] + // groups dwc attributes to event core tables + Map normalised = normaliseRecords(record, params) + normalised.each { dwcClass, attributes -> + switch (dwcClass) { + case DWC_EVENT: + if (attributes && !result[dwcClass].find { it.eventID == attributes.eventID }) { + attributes.parentEventID = projectActivity.projectActivityId + result[dwcClass].add (attributes) + } + break + case DWC_MEDIA: + case DWC_MEASUREMENT: + if(attributes){ + result[dwcClass].addAll(attributes) + } + break + case DWC_OCCURRENCE: + default: + if(attributes && attributes.scientificName) { + result[dwcClass].add (attributes) + } + break + } + } + } + } catch (Exception ex) { + log.error("error converting record - activity id " + activity.activityId, ex) + } + } + } + offset += batchSize + size = activities.size() + } + } + + if (result[DWC_MEASUREMENT]?.size()) { + result[DWC_MEASUREMENT] = RecordConverter.removeDuplicates(result[DWC_MEASUREMENT]) + } + + if (result[DWC_MEDIA]?.size()) { + result[DWC_MEDIA] = RecordConverter.removeDuplicates(result[DWC_MEDIA]) + } + + result + } + + /** + * Get the order of header to write the CSV. By default, the order is determined by order of entry in config + * darwinCore.termsGroupedByClass[dwcClass]. However, it is possible to override this by an attribute called order. + * Check entry for MeasurementOrFact. + * @param configs + * @return + */ + List getHeaderOrder (List configs) { + Boolean isExplicitOrderAvailable = false + Integer orderIndex = null + List headers = [] + configs.eachWithIndex { config, index -> + headers.add(config.name) + + if (config.order) { + isExplicitOrderAvailable = true + orderIndex = index + } + } + + isExplicitOrderAvailable ? configs.get(orderIndex)?.order : headers + } + + /** + * Reorder content of header based on defaultOrder. + * @param headers + * @param defaultOrder + * @return + */ + List reorderHeaders (List headers, List defaultOrder) { + List reordered = [] + + defaultOrder.each { name -> + if ( headers.contains(name) ) { + reordered.add(name) + headers.remove(name) + } + } + + reordered + headers + } + + /** + * Groups darwin core fields to event core table. Various transformations are done on top of the value. + * @param record + * @param params + * @return + */ + Map normaliseRecords(Map record, Map params) { + Map trimmedGroups = darwinCoreTermsGroupedByClass() + Map attributeLinkedToClasses = invertAttributeToClass(trimmedGroups ) + Map result = [:].withDefault {[:]} + attributeLinkedToClasses.each { attribute, classes -> + classes.each { dwcClass -> + Map config = getAttributeConfig(dwcClass, attribute) + + transformToEventCoreTerm(config, record, params, result, dwcClass) + } + } + + result + } + + /** + * Transform a darwin core value before entering into event core. + * @param config + * @param record + * @param params + * @param result + * @param dwcClass + */ + void transformToEventCoreTerm(Map config, Object record, Map params, Map result, String dwcClass) { + String attribute = config.name + + if (config.code) { + result[dwcClass][attribute] = config.code(record, params) + } else if (config.substitute) { + result[dwcClass] = config.substitute(record, params) + } else if (config.ref) { + result[dwcClass][attribute] = record[config.ref] + } else if (record[attribute] != null) { + if (record[attribute] instanceof List) + result[dwcClass][attribute] = record[attribute]?.join(" | ") + else + result[dwcClass][attribute] = record[attribute] + } else if (config.default) { + result[dwcClass][attribute] = config.default + } + } + + Map getAttributeConfig(String dwcClass, String attribute) { + Map groups = grailsApplication.config.darwinCore.termsGroupedByClass + groups[dwcClass].find { it.name == attribute } + } + + Map darwinCoreTermsGroupedByClass() { + Map groups = grailsApplication.config.darwinCore.termsGroupedByClass + Map trimmedGroups = [:] + groups.each { String key, List values -> + trimmedGroups[key] = values?.collect { + it.name + } + + if (trimmedGroups[key]?.size() == 0 ) { + trimmedGroups.remove(key) + } + } + + trimmedGroups + } + + Map invertAttributeToClass (Map termsGroupedByClass) { + Map inverted = [:].withDefault {[]} + termsGroupedByClass.each {dwcClass, attributes -> + attributes.each { + inverted [it].add (dwcClass) + } + } + + inverted + } + + /** + * Iterates all rows to get union of all keys in the map. + * @param rows + * @return + */ + List getHeadersFromRows (List rows) { + Set headers = new HashSet() + rows.each { Map row -> + headers.addAll(row.keySet()) + } + + headers.asList() + } + + /** + * Generates meta.xml class. It describes the values in various associated CSV files. + * @param headersByDwcClass + * @return + */ + String getMetaXML(Map headersByDwcClass) { + Map dataBinding = getMetaXMLData(headersByDwcClass) + String xml = mapService.bindDataToXMLTemplate("classpath:data/eventcore/meta.template", dataBinding, true) + addXMLDeclaration(xml) + } + + /** + * Generates data that is used to bind to meta.template. + * @param headersByDwcClass + * @return + */ + Map getMetaXMLData(Map headersByDwcClass) { + Map dataBinding = [ + archiveNameSpace: grailsApplication.config.darwinCore.namespaces["Archive"], + emlFileName: "eml.xml", + core: [:], + extensions: [] + ] + + headersByDwcClass.each { String dwcClass, List headers -> + Map result = getFields(headers) + Map defaultValues = [ + encoding: "UTF-8", + fieldsTerminatedBy: CSVWriter.DEFAULT_SEPARATOR, + linesTerminatedBy: "\\n", + fieldsEnclosedBy: CSVWriter.DEFAULT_QUOTE_CHARACTER == '"' ? """ : "", + ignoreHeaderLines: 1 + ] + + switch (dwcClass) { + case DWC_EVENT: + defaultValues.rowType = grailsApplication.config.darwinCore.namespaces[dwcClass] + defaultValues.location = "${dwcClass}.csv" + defaultValues.fields = result.fields + defaultValues.coreIndex = result.coreIndex + dataBinding.core = defaultValues + break + default: + defaultValues.rowType = grailsApplication.config.darwinCore.namespaces[dwcClass] + defaultValues.location = "${dwcClass}.csv" + defaultValues.fields = result.fields + defaultValues.coreIndex = result.coreIndex + dataBinding.extensions.add(defaultValues) + break + } + } + + dataBinding + } + + /** + * Get darwin core namespace for various fields. Used to create meta.xml. + * @param headers + * @return + */ + Map getFields(List headers) { + int coreIndex = 0 + String coreID = "eventID" + List fields = [] + + headers?.eachWithIndex { header, hIndex -> + if (header == coreID) + coreIndex = hIndex + + fields.add([index: hIndex, term: grailsApplication.config.darwinCore.namespaces[header] ?: header]) + } + + [coreIndex: coreIndex, fields: fields] + } + + /** + * Creates eml.xml. Data bound to it is derived from Project domain object. + * @param project + * @return + */ + String getEmlXML(Project project) { + Map proj = projectService.toMap(project) + String xml = mapService.bindDataToXMLTemplate("classpath:data/eventcore/eml.template", proj, true) + addXMLDeclaration(xml) + } + + String addXMLDeclaration (String xml) { + String declaration = "\n" + declaration + xml + } + + /** + * Convert project activity to darwin core records. This is ultimately ends up in Event table. + * @param pActivity + * @param project + * @return + */ + Map convertProjectActivityToEvent (pActivity, project) { + String dwcClass = DWC_EVENT + List configs = grailsApplication.config.darwinCore.projectActivityToDwC[dwcClass] + Map result = [:].withDefault {[:]} + configs.each { config -> + transformToEventCoreTerm(config, pActivity, [project: project, pActivity: pActivity, recordService: this], result, dwcClass) + } + + result[dwcClass] + } } diff --git a/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy index b8fe8633b..609c870b0 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy @@ -9,7 +9,7 @@ class GenericFieldConverter implements RecordFieldConverter { Double latitude = getLatitude(data) Double longitude = getLongitude(data) - // Don't override decimalLongitud or decimalLatitude in case they are null, site info could've already set them + // Don't override decimalLongitude or decimalLatitude in case they are null, site info could've already set them if(latitude) { record.decimalLatitude = latitude } @@ -20,8 +20,8 @@ class GenericFieldConverter implements RecordFieldConverter { Map dwcMappings = extractDwcMapping(metadata) - - record << getDwcAttributes(data, dwcMappings) + Map dwcAttributes = getDwcAttributes(data, dwcMappings, metadata) + record = RecordConverter.overrideAllExceptLists(dwcAttributes, record) if (data.dwcAttribute) { record[data.dwcAttribute] = data.value diff --git a/src/main/groovy/au/org/ala/ecodata/converter/ImageConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/ImageConverter.groovy index d419e67a0..350ec1d5e 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/ImageConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/ImageConverter.groovy @@ -4,6 +4,7 @@ class ImageConverter implements RecordFieldConverter { private static final String DEFAULT_RIGHTS_STATEMENT = "The rights to all uploaded images are held under the specified Creative Commons license, by the contributor of the image and the primary organisation responsible for the project to which they are contributed." private static final String DEFAULT_LICENCE = "CC BY 4.0" + private static final String TYPE = "StillImage" List convert(Map data, Map metadata = [:]) { Map record = [:] @@ -11,6 +12,7 @@ class ImageConverter implements RecordFieldConverter { record.multimedia = data[metadata.name] record.multimedia?.each { if (it instanceof Map) { + it.type = TYPE it.identifier = it.identifier ?: it.url it.creator = it.creator ?: it.attribution it.title = it.title ?: it.filename diff --git a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy index a4d24c835..0528915ee 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy @@ -36,7 +36,9 @@ class ListConverter implements RecordFieldConverter { baseRecordModels?.each { Map dataModel -> RecordFieldConverter converter = RecordConverter.getFieldConverter(dataModel.dataType) List recordFieldSets = converter.convert(row, dataModel) - baseRecord << recordFieldSets[0] + Map recordFieldSet = recordFieldSets[0] + baseRecord = RecordConverter.overrideAllExceptLists(baseRecord, recordFieldSet) + RecordConverter.updateEventIdToMeasurements(baseRecord[PROP_MEASUREMENTS_OR_FACTS], baseRecord.activityId) } // For each species dataType, where present we will generate a new record @@ -48,12 +50,17 @@ class ListConverter implements RecordFieldConverter { // We want to create a record in the DB only if species information is present if(speciesRecord.outputSpeciesId) { speciesRecord.outputItemId = index++ + RecordConverter.updateSpeciesIdToMeasurements(speciesRecord[PROP_MEASUREMENTS_OR_FACTS], speciesRecord.outputSpeciesId) records << speciesRecord } else { log.warn("Record [${speciesRecord}] does not contain full species information. " + "This is most likely a bug.") } } + + if (!speciesModels) { + records << baseRecord + } } records diff --git a/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy index be1764602..b69809c4b 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy @@ -1,14 +1,11 @@ package au.org.ala.ecodata.converter -import au.org.ala.ecodata.Activity -import au.org.ala.ecodata.Output -import au.org.ala.ecodata.Project -import au.org.ala.ecodata.ProjectActivity -import au.org.ala.ecodata.Organisation -import au.org.ala.ecodata.Site +import au.org.ala.ecodata.* import groovy.util.logging.Slf4j import org.apache.commons.lang.StringUtils +import static au.org.ala.ecodata.converter.ListConverter.PROP_MEASUREMENTS_OR_FACTS + /** * Utility for converting the data submitted for an Output into one or more Records with the correct Darwin Core fields * so that it can be exported to the Biocache. @@ -51,8 +48,25 @@ class RecordConverter { static final String DEFAULT_LICENCE_TYPE = "https://creativecommons.org/licenses/by/4.0/" static final String SYSTEMATIC_SURVEY = "systematic" static final List MACHINE_SURVEY_TYPES = ["Bat survey - Echolocation recorder","Fauna survey - Call playback","Fauna survey - Camera trapping"] + static final List LIST_COMBINE_EXCEPTION = ["multimedia"] - static List convertRecords(Project project, Organisation organisation, Site site, ProjectActivity projectActivity, Activity activity, Output output, Map data, Map outputMetadata) { + /** + * Creates darwin core fields from output data. It behaviour is controlled by recordGeneration attribute. + * By default this field is true and backward compatible with record creation logic for BioCollect Activity. It will only + * return value if species is present in data model. Setting recordGeneration is false is used by event core archive + * creation. It returns darwin core fields even if species is not present. + * @param project + * @param organisation + * @param site + * @param projectActivity + * @param activity + * @param output + * @param data + * @param outputMetadata + * @param recordGeneration + * @return + */ + static List convertRecords(Project project, Organisation organisation, Site site, ProjectActivity projectActivity, Activity activity, Output output, Map data, Map outputMetadata, boolean recordGeneration = true) { // Outputs are made up of multiple 'dataModels', where each dataModel could map to one or more Record fields // and/or one or more Records. For example, a dataModel with a type of 'list' will map to one Record per item in // the list. Further, each item in the list is a 'dataModel' on it's own, which will map to one or more fields. @@ -63,8 +77,11 @@ class RecordConverter { projectId : activity.projectId, projectActivityId: activity.projectActivityId, activityId : activity.activityId, - userId : activity.userId + userId : activity.userId, + multimedia : [] ] + Long start = System.currentTimeMillis(), end + // Populate the skeleton with Record attributes which are derived from the Activity. These attributes are shared // by all Records that are generated from this Output. @@ -96,7 +113,8 @@ class RecordConverter { baseRecordModels?.each { Map dataModel -> RecordFieldConverter converter = getFieldConverter(dataModel.dataType) List recordFieldSets = converter.convert(data, dataModel) - baseRecord << recordFieldSets[0] + baseRecord = overrideAllExceptLists(baseRecord, recordFieldSets[0]) + updateEventIdToMeasurements(baseRecord[PROP_MEASUREMENTS_OR_FACTS], baseRecord.activityId) } List records = [] @@ -104,16 +122,25 @@ class RecordConverter { speciesModels?.each { Map dataModel -> RecordFieldConverter converter = getFieldConverter(dataModel.dataType) List recordFieldSets = converter.convert(data, dataModel) - Map speciesRecord = overrideFieldValues(baseRecord, recordFieldSets[0]) - + Map speciesRecord = overrideAllExceptLists(baseRecord, recordFieldSets[0]) // We want to create a record in the DB only if species guid is present i.e. species is valid - if(speciesRecord.guid && speciesRecord.guid != "") { + if(recordGeneration){ + if(speciesRecord.guid && speciesRecord.guid != "") { + records << speciesRecord + } + else { + log.warn("Record [${speciesRecord}] does not contain full species information. " + + "This is most likely a bug.") + } + } + else { + updateSpeciesIdToMeasurements(speciesRecord[PROP_MEASUREMENTS_OR_FACTS], speciesRecord.outputSpeciesId) records << speciesRecord - } else { - log.warn("Record [${speciesRecord}] does not contain full species information. " + - "This is most likely a bug.") } } + end = System.currentTimeMillis() + log.debug("Time in milliseconds to convert root level data model - ${end - start}") + start = end if (multiItemModels) { // For each multiItemModel, get the appropriate field converter for the data type and generate the list of field @@ -124,17 +151,28 @@ class RecordConverter { List recordFieldSets = converter.convert(data, dataModel) recordFieldSets.each { - Map rowRecord = overrideFieldValues(baseRecord, it) - if(rowRecord.guid && rowRecord.guid != "") { + Map rowRecord = overrideAllExceptLists(baseRecord, it) + if(recordGeneration) { + if (rowRecord.guid && rowRecord.guid != "") { + records << rowRecord + + } else { + log.warn("Multi item Record [${rowRecord}] does not contain species information, " + + "was the form intended to work like that?") + } + } + else { + updateEventIdToMeasurements(rowRecord[PROP_MEASUREMENTS_OR_FACTS], baseRecord.activityId) records << rowRecord - } else { - log.warn("Multi item Record [${rowRecord}] does not contain species information, " + - "was the form intended to work like that?") } } } } - + else if(!speciesModels && !recordGeneration) { + records << baseRecord + } + end = System.currentTimeMillis() + log.debug("Time in milliseconds to convert nested data model - ${end - start}") // We are now left with a list of one or more Maps, where each Map contains all the fields for an individual Record. records } @@ -244,4 +282,69 @@ class RecordConverter { dwcFields } + + static void updateSpeciesIdToMeasurements(List measurements, String id) { + measurements?.each { + if(!it.occurrenceID) + it.occurrenceID = id + } + } + + static void updateEventIdToMeasurements(List measurements, String id) { + measurements?.each { + if(!it.eventID) + it.eventID = id + } + } + + static Map overrideAllExceptLists(Map source, Map additional){ + Map result = [:] + overrideValues(source, result) + overrideValues(additional, result) + } + + /** + * Replaces value in result with value in source except when the value is a list. + * If value is list, then it combines the result and source values. However, this is not applicable + * for key found in LIST_COMBINE_EXCEPTION. Therefore, `multimedia` values are replaced. But `measurementorfact` values + * are combined. + * @param source + * @param result + * @return + */ + static Map overrideValues(Map source, Map result) { + source.each { entry -> + if ((entry.value instanceof Collection) && (result[entry.key] instanceof Collection)) { + if (LIST_COMBINE_EXCEPTION.contains(entry.key)) { + result[entry.key] = entry.value + return + } + + if(result[entry.key] == null) { + result[entry.key] = [] + } + + result[entry.key].addAll(entry.value) + } + else { + result[entry.key] = entry.value + } + } + + result + } + + /** + * Removes for duplicate entries. A duplicate entry is when all values in a map are the same. + * [a: 'b', c: 'd'] & [a: 'b', c: 'd'] are duplicates. + * [a: 'b', c: 'd'] & [a: 'b', c: 'e'] are not duplicates. + * @param measurements + * @return + */ + static List removeDuplicates (List measurements) { + measurements?.groupBy { + it.values().toString() }.collect { key, values -> + values?.get(0) + } + } } diff --git a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy index 0301ed5fc..66b95aafa 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy @@ -10,7 +10,14 @@ package au.org.ala.ecodata.converter * target Record attribute is different (the converter is responsible for mapping one to the other). */ trait RecordFieldConverter { - String DWC_ATTRIBUTE_NAME = "dwcAttribute" + static String DWC_ATTRIBUTE_NAME = "dwcAttribute" + static String DWC_MEASUREMENT_VALUE = "measurementValue" + static String DWC_MEASUREMENT_TYPE = "measurementType" + static String DWC_MEASUREMENT_TYPE_ID = "measurementTypeID" + static String DWC_MEASUREMENT_ACCURACY = "measurementAccuracy" + static String DWC_MEASUREMENT_UNIT = "measurementUnit" + static String DWC_MEASUREMENT_UNIT_ID = "measurementUnitID" + static String PROP_MEASUREMENTS_OR_FACTS = "measurementsorfacts" abstract List convert(Map data) @@ -34,18 +41,76 @@ trait RecordFieldConverter { dwcMappings } - Map getDwcAttributes(Map dataModel, Map dwcMappings) { + Map getDwcAttributes(Map data, Map dwcMappings, Map metadata = null) { + Map fields = [:] dwcMappings.each { dwcAttribute, fieldName -> - fields[dwcAttribute] = dataModel[fieldName] + fields[dwcAttribute] = data[fieldName] } - if (dwcMappings.containsKey("species")){ - fields.name = dataModel[dwcMappings["species"]].name - fields.scientificName = dataModel[dwcMappings["species"]].scientificName - fields.guid = dataModel[dwcMappings["species"]].guid + if (dwcMappings.containsKey("species")) { + fields.name = data[dwcMappings["species"]].name + fields.scientificName = data[dwcMappings["species"]].scientificName + fields.guid = data[dwcMappings["species"]].guid + } + + // Add measurements of facts if metadata dwcAttribute value is 'measurementValue'. + // It dwcAttribute measurementValue is taken from output data. + // If measurementType is provided in metadata, its event core value is created by binding metadata value and + // output data. Rest of the attributes are provided by metadata. + String fieldName = dwcMappings[DWC_MEASUREMENT_VALUE] + def value = data[fieldName] + if (dwcMappings.containsKey(DWC_MEASUREMENT_VALUE) && ![null, ""].contains(value)) { + if (!fields[PROP_MEASUREMENTS_OR_FACTS]) + fields[PROP_MEASUREMENTS_OR_FACTS] = [] + + Map measurement = [:] + measurement[DWC_MEASUREMENT_VALUE] = value + measurement[DWC_MEASUREMENT_TYPE] = getMeasurementType(metadata, data) + + if (metadata?.containsKey(DWC_MEASUREMENT_TYPE_ID)) { + measurement[DWC_MEASUREMENT_TYPE_ID] = metadata[DWC_MEASUREMENT_TYPE_ID] + } + + if (metadata?.containsKey(DWC_MEASUREMENT_ACCURACY)) { + measurement[DWC_MEASUREMENT_ACCURACY] = metadata[DWC_MEASUREMENT_ACCURACY] + } + + if (metadata?.containsKey(DWC_MEASUREMENT_UNIT)) { + measurement[DWC_MEASUREMENT_UNIT] = metadata[DWC_MEASUREMENT_UNIT] + } + + if (metadata?.containsKey(DWC_MEASUREMENT_UNIT_ID)) { + measurement[DWC_MEASUREMENT_UNIT_ID] = metadata[DWC_MEASUREMENT_UNIT_ID] + } + + fields[PROP_MEASUREMENTS_OR_FACTS].add(measurement) } fields } + + /** + * Create metadata type value for event core's measurement or fact table. + * 1. If measurementType is provided by metadata i.e. dataModel, then bind output data to it. It uses groovy's SimpleTemplateEngine. + * 2. Otherwise, uses one of description, value or name found in metadata. + * @param metadata + * @param data + * @return + */ + String getMeasurementType(metadata, data) { + def defaultValue = metadata?.description ?: metadata?.value ?: metadata?.name + if (metadata?.measurementType && data) { + try { + def engine = new groovy.text.SimpleTemplateEngine() + return engine.createTemplate(metadata?.measurementType).make(data) + } + catch (Exception ex) { + return defaultValue + } + } else { + return defaultValue + } + } + } \ No newline at end of file diff --git a/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy index 1e73afea0..652e33642 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy @@ -6,23 +6,27 @@ class SpeciesConverter implements RecordFieldConverter { List convert(Map data, Map metadata = [:]) { Map record = [:] + // if species data not found, return a list of empty map + if(data[metadata.name] == null){ + return [record] + } - record.guid = data[metadata.name].guid - record.name = data[metadata.name].name + record.guid = data[metadata.name]?.guid + record.name = data[metadata.name]?.name // if there is a valid guid then pass on scientific name (if it is valid) if (record.guid && record.guid != "") { - if (data[metadata.name].scientificName) { - record.scientificName = data[metadata.name].scientificName + if (data[metadata.name]?.scientificName) { + record.scientificName = data[metadata.name]?.scientificName } } // Force outputSpeciesId generation if not coming in the original data - if(!data[metadata.name].outputSpeciesId) { + if(data[metadata.name] && !data[metadata.name]?.outputSpeciesId) { data[metadata.name].outputSpeciesId = UUID.randomUUID().toString() } - record.outputSpeciesId = data[metadata.name].outputSpeciesId + record.outputSpeciesId = data[metadata.name]?.outputSpeciesId [record] } From 847e48dd61cc9d1f0bbb144ca1057f6d007d874b Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 25 Aug 2022 14:48:12 +1000 Subject: [PATCH 02/22] #1445 added RequireApiKey to listHarvestDataResource action --- .../controllers/au/org/ala/ecodata/RecordController.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index 13d3a4f30..b937e4b78 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -43,7 +43,7 @@ class RecordController { * @param sort = asc | desc * */ - @PreAuthorise + @RequireApiKey def listHarvestDataResource() { def result, error try { From 426d2709dc7e7960b9074f4517bdc45742409107 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 25 Aug 2022 22:04:30 +1000 Subject: [PATCH 03/22] #1445 reverted to @PreAuthorise annotation --- .../controllers/au/org/ala/ecodata/RecordController.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index b937e4b78..1a2b537b4 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -43,7 +43,7 @@ class RecordController { * @param sort = asc | desc * */ - @RequireApiKey + @PreAuthorise def listHarvestDataResource() { def result, error try { @@ -563,7 +563,7 @@ class RecordController { * @param projectId * @return */ - @RequireApiKey + @PreAuthorise def getDarwinCoreArchiveForProject (String projectId) { if (projectId) { Project project = Project.findByProjectId(projectId) From 17b7a400c8a8017fe75069acff73ecc3d1c9fbc7 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 26 Aug 2022 11:05:28 +1000 Subject: [PATCH 04/22] #1445 reorder zip output to prevent read timeout --- .../services/au/org/ala/ecodata/RecordService.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/RecordService.groovy b/grails-app/services/au/org/ala/ecodata/RecordService.groovy index f78f76542..7f4c68e14 100644 --- a/grails-app/services/au/org/ala/ecodata/RecordService.groovy +++ b/grails-app/services/au/org/ala/ecodata/RecordService.groovy @@ -967,6 +967,11 @@ class RecordService { new ZipOutputStream(outputStream).withStream { zip -> try { Long start = System.currentTimeMillis(), end + + zip.putNextEntry(new ZipEntry("eml.xml")) + zip << getEmlXML(project) + zip.closeEntry() + result = generateEventCoreFiles (project) result.each { dwcClass, rows -> @@ -997,9 +1002,6 @@ class RecordService { zip << getMetaXML(headersByDwcClass) zip.closeEntry() - zip.putNextEntry(new ZipEntry("eml.xml")) - zip << getEmlXML(project) - zip.closeEntry() end = System.currentTimeMillis() log.debug("Time in milliseconds to write event core XMLs- ${end - start}") From 40d64c5f9d310e3ab0b47546e88a9a9d02af4a75 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 1 Nov 2023 11:08:02 +1100 Subject: [PATCH 05/22] removed PreAuthorise annotation --- .../controllers/au/org/ala/ecodata/RecordController.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index 3e86701a9..036672318 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -43,7 +43,7 @@ class RecordController { * @param sort = asc | desc * */ - @PreAuthorise +// @PreAuthorise def listHarvestDataResource() { def result, error try { @@ -93,7 +93,7 @@ class RecordController { * @param status = active | deleted | default:active * */ - @PreAuthorise +// @PreAuthorise def listRecordsForDataResourceId (){ def result = [], error, project Date lastUpdated = null @@ -563,7 +563,7 @@ class RecordController { * @param projectId * @return */ - @PreAuthorise +// @PreAuthorise def getDarwinCoreArchiveForProject (String projectId) { if (projectId) { Project project = Project.findByProjectId(projectId) From f74964ba6df137951da128fe567d4998b83a5166 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 1 Nov 2023 11:36:58 +1100 Subject: [PATCH 06/22] added RequireApiKey instead of PreAuthorise --- .../au/org/ala/ecodata/ParatooController.groovy | 12 ++---------- .../au/org/ala/ecodata/RecordController.groovy | 3 +++ 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy index b43ccd693..ed694c423 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy @@ -3,11 +3,10 @@ package au.org.ala.ecodata import au.ala.org.ws.security.SkipApiKeyCheck import au.org.ala.ecodata.paratoo.ParatooCollection import au.org.ala.ecodata.paratoo.ParatooCollectionId -import au.org.ala.ecodata.paratoo.ParatooProject import groovy.util.logging.Slf4j import io.swagger.v3.oas.annotations.OpenAPIDefinition import io.swagger.v3.oas.annotations.Operation -import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.enums.SecuritySchemeType import io.swagger.v3.oas.annotations.info.Contact import io.swagger.v3.oas.annotations.info.Info import io.swagger.v3.oas.annotations.info.License @@ -16,13 +15,7 @@ import io.swagger.v3.oas.annotations.media.Content import io.swagger.v3.oas.annotations.media.Schema import io.swagger.v3.oas.annotations.parameters.RequestBody import io.swagger.v3.oas.annotations.responses.ApiResponse -import io.swagger.v3.oas.annotations.security.SecurityRequirement -import io.swagger.v3.oas.annotations.security.SecurityRequirements -import io.swagger.v3.oas.annotations.security.SecurityScheme -import io.swagger.v3.oas.annotations.security.SecuritySchemes -import io.swagger.v3.oas.annotations.enums.SecuritySchemeType; -import io.swagger.v3.oas.annotations.security.OAuthFlow; -import io.swagger.v3.oas.annotations.security.OAuthFlows +import io.swagger.v3.oas.annotations.security.* import io.swagger.v3.oas.annotations.servers.Server import io.swagger.v3.oas.annotations.servers.ServerVariable import org.apache.http.HttpStatus @@ -32,7 +25,6 @@ import javax.ws.rs.GET import javax.ws.rs.POST import javax.ws.rs.PUT import javax.ws.rs.Path - // Requiring these scopes will guarantee we can get a valid userId out of the process. @au.ala.org.ws.security.RequireApiKey @Slf4j diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index 036672318..f86135c21 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -44,6 +44,7 @@ class RecordController { * */ // @PreAuthorise + @RequireApiKey def listHarvestDataResource() { def result, error try { @@ -94,6 +95,7 @@ class RecordController { * */ // @PreAuthorise + @RequireApiKey def listRecordsForDataResourceId (){ def result = [], error, project Date lastUpdated = null @@ -564,6 +566,7 @@ class RecordController { * @return */ // @PreAuthorise + @RequireApiKey def getDarwinCoreArchiveForProject (String projectId) { if (projectId) { Project project = Project.findByProjectId(projectId) From 08ef7fd08caf55979d1789ee2b5888960d510e6d Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 1 Nov 2023 11:49:13 +1100 Subject: [PATCH 07/22] changed how host name is looked up --- .../au/org/ala/ecodata/RecordController.groovy | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index f86135c21..57145513f 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -1,21 +1,17 @@ package au.org.ala.ecodata -import org.apache.http.HttpStatus -import org.apache.http.entity.ContentType -import org.grails.web.servlet.mvc.GrailsWebRequest -import org.springframework.web.context.request.RequestAttributes - -import java.text.SimpleDateFormat - -import static au.org.ala.ecodata.Status.* -import static javax.servlet.http.HttpServletResponse.* - import grails.converters.JSON import groovy.json.JsonSlurper import org.apache.commons.codec.binary.Base64 +import org.apache.http.HttpStatus +import org.apache.http.entity.ContentType import org.springframework.web.multipart.MultipartFile import org.springframework.web.multipart.MultipartHttpServletRequest +import java.text.SimpleDateFormat + +import static au.org.ala.ecodata.Status.DELETED +import static javax.servlet.http.HttpServletResponse.* /** * Controller for record CRUD operations with support for handling images. */ @@ -573,7 +569,7 @@ class RecordController { if(project?.alaHarvest) { // Simulate BioCollect as the hostname calling this method. This is done to get the correct URL for // documents. - GrailsWebRequest.lookup().setAttribute(DocumentHostInterceptor.DOCUMENT_HOST_NAME, grailsApplication.config.getProperty("biocollect.baseURL"), RequestAttributes.SCOPE_REQUEST) +// GrailsWebRequest.lookup().setAttribute(DocumentHostInterceptor.DOCUMENT_HOST_NAME, grailsApplication.config.getProperty("biocollect.baseURL"), RequestAttributes.SCOPE_REQUEST) recordService.getDarwinCoreArchiveForProject(response.outputStream, project) } else response status: HttpStatus.SC_NOT_FOUND, text: [error: "project not found or ala harvest flag is switched off"] as JSON, contentType: ContentType.APPLICATION_JSON From 113214d16ab66096544f0ec0d31d29a2cfa023ae Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 2 Nov 2023 22:27:00 +1100 Subject: [PATCH 08/22] - AtlasOfLivingAustralia/biocollect#1568 eml content from collectory dr entry - enabled access to APIs with JWT token --- .../au/org/ala/ecodata/RecordController.groovy | 14 +++++++------- .../au/org/ala/ecodata/RecordService.groovy | 14 +++++++++++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index 57145513f..c866f67e6 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -39,8 +39,7 @@ class RecordController { * @param sort = asc | desc * */ -// @PreAuthorise - @RequireApiKey + @au.ala.org.ws.security.RequireApiKey def listHarvestDataResource() { def result, error try { @@ -88,10 +87,12 @@ class RecordController { * @param sort = asc | desc | default:asc * @param lastUpdated = date | dd/MM/yyyy | default:null * @param status = active | deleted | default:active + * @deprecated ALA's records harvester will use getDarwinCoreArchiveForProject once Events system is setup. + * To access it use archiveURL property from {@link RecordController#listHarvestDataResource}. * */ -// @PreAuthorise - @RequireApiKey + @Deprecated + @au.ala.org.ws.security.RequireApiKey def listRecordsForDataResourceId (){ def result = [], error, project Date lastUpdated = null @@ -561,15 +562,14 @@ class RecordController { * @param projectId * @return */ -// @PreAuthorise - @RequireApiKey + @au.ala.org.ws.security.RequireApiKey def getDarwinCoreArchiveForProject (String projectId) { if (projectId) { Project project = Project.findByProjectId(projectId) if(project?.alaHarvest) { // Simulate BioCollect as the hostname calling this method. This is done to get the correct URL for // documents. -// GrailsWebRequest.lookup().setAttribute(DocumentHostInterceptor.DOCUMENT_HOST_NAME, grailsApplication.config.getProperty("biocollect.baseURL"), RequestAttributes.SCOPE_REQUEST) + DocumentHostInterceptor.documentHostUrlPrefix.set(grailsApplication.config.getProperty("biocollect.baseURL")) recordService.getDarwinCoreArchiveForProject(response.outputStream, project) } else response status: HttpStatus.SC_NOT_FOUND, text: [error: "project not found or ala harvest flag is switched off"] as JSON, contentType: ContentType.APPLICATION_JSON diff --git a/grails-app/services/au/org/ala/ecodata/RecordService.groovy b/grails-app/services/au/org/ala/ecodata/RecordService.groovy index e904e68b5..39bf7923a 100644 --- a/grails-app/services/au/org/ala/ecodata/RecordService.groovy +++ b/grails-app/services/au/org/ala/ecodata/RecordService.groovy @@ -48,6 +48,7 @@ class RecordService { CommonService commonService MapService mapService AuthService authService + WebService webService final def ignores = ["action", "controller", "associatedMedia"] private static final List EXCLUDED_RECORD_PROPERTIES = ["_id", "activityId", "dateCreated", "json", "outputId", "projectActivityId", "projectId", "status", "dataResourceUid"] @@ -1504,9 +1505,16 @@ class RecordService { * @return */ String getEmlXML(Project project) { - Map proj = projectService.toMap(project) - String xml = mapService.bindDataToXMLTemplate("classpath:data/eventcore/eml.template", proj, true) - addXMLDeclaration(xml) + String url = grailsApplication.config.getProperty('collectory.baseURL') + "ws/eml/${project.dataResourceId}" + def resp = webService.get(url) + if (resp instanceof String) { + return resp + } + else { + Map proj = projectService.toMap(project) + String xml = mapService.bindDataToXMLTemplate("classpath:data/eventcore/eml.template", proj, true) + addXMLDeclaration(xml) + } } String addXMLDeclaration (String xml) { From 8eb9f7733c3b5cbe1feaf73fd81b9d5e7cd75fe2 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 22 Jan 2025 13:56:26 +1100 Subject: [PATCH 09/22] 5.2-EXTENDED-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index fb1837b2b..215c37add 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.2-SNAPSHOT" +version "5.2-EXTENDED-SNAPSHOT" group "au.org.ala" description "Ecodata" From 6cb56ff80b7d0200e52d93fe02f4c6a4474eaaf8 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 28 Jan 2025 11:50:55 +1100 Subject: [PATCH 10/22] upgraded security to use scopes --- .../controllers/au/org/ala/ecodata/RecordController.groovy | 3 --- 1 file changed, 3 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index fa2d9a246..1309c84fd 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -38,7 +38,6 @@ class RecordController { * @param sort = asc | desc * */ - @au.ala.org.ws.security.RequireApiKey def listHarvestDataResource() { def result, error try { @@ -91,7 +90,6 @@ class RecordController { * */ @Deprecated - @au.ala.org.ws.security.RequireApiKey def listRecordsForDataResourceId (){ def result = [], error, project Date lastUpdated = null @@ -559,7 +557,6 @@ class RecordController { * @param projectId * @return */ - @au.ala.org.ws.security.RequireApiKey def getDarwinCoreArchiveForProject (String projectId) { if (projectId) { Project project = Project.findByProjectId(projectId) From 69cd8c16f375c0c8e53e8bc78d76a3ca31d302bd Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 30 Jan 2025 10:26:14 +1100 Subject: [PATCH 11/22] AtlasOfLivingAustralia/fieldcapture#3432 - added expression evaluator to generate value of an attribute in a record - added test cases for checking event core archive and expression evaluation --- grails-app/conf/application.groovy | 6 +- .../org/ala/ecodata/HarvestController.groovy | 28 +- .../org/ala/ecodata/RecordController.groovy | 148 +---------- .../au/org/ala/ecodata/UrlMappings.groovy | 2 +- .../au/org/ala/ecodata/RecordService.groovy | 10 +- .../ala/ecodata/HarvestControllerSpec.groovy | 242 ++++++++++++++++++ .../ecodata/converter/AudioConverter.groovy | 2 +- .../ecodata/converter/FeatureConverter.groovy | 2 +- .../converter/GenericFieldConverter.groovy | 5 +- .../ecodata/converter/ImageConverter.groovy | 2 +- .../ecodata/converter/ListConverter.groovy | 17 +- .../converter/MasterDetailConverter.groovy | 4 +- .../ecodata/converter/RecordConverter.groovy | 20 +- .../converter/RecordFieldConverter.groovy | 44 +++- .../ecodata/converter/SpeciesConverter.groovy | 4 +- .../converter/GenericConverterSpec.groovy | 113 ++++++++ .../converter/RecordConverterSpec.groovy | 160 ++++++++++++ 17 files changed, 616 insertions(+), 193 deletions(-) create mode 100644 src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 67f65938e..19bbb104d 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -11,7 +11,7 @@ environments { mongodb { host = "localhost" port = "27017" - databaseName = "ecodata" + databaseName = "ecodata-test-server" } } } @@ -1657,7 +1657,7 @@ if (!darwinCore.termsGroupedByClass) { multimedia = document return [ "eventID" : params?.activity?.activityId, - "occurrenceID": record?.outputSpeciesId , + "occurrenceID": record?.occurrenceID , "type" : multimedia?.type, "identifier" : identifier, "format" : multimedia?.contentType, @@ -1690,7 +1690,7 @@ if (!darwinCore.termsGroupedByClass) { "name" : "occurrenceID", "namespace": "dwc", "code" : { record, params -> - record?.outputSpeciesId + record?.occurrenceID ?: record?.outputSpeciesId } ], [ diff --git a/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy b/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy index 6058e3c85..cd9b3a426 100644 --- a/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy @@ -2,6 +2,7 @@ package au.org.ala.ecodata import grails.converters.JSON import org.apache.http.HttpStatus +import org.apache.http.entity.ContentType import java.text.SimpleDateFormat; @@ -70,9 +71,11 @@ class HarvestController { * @param sort = asc | desc | default:asc * @param lastUpdated = date | dd/MM/yyyy | default:null * @param status = active | deleted | default:active - * + * @deprecated ALA's records harvester will use getDarwinCoreArchiveForProject once Events system is setup. + * To access it use archiveURL property from {@link HarvestController#listHarvestDataResource}. */ - def listRecordsForDataResourceId (){ + @Deprecated + def listRecordsForDataResourceId () { def result = [], error, project Date lastUpdated = null try { @@ -134,4 +137,25 @@ class HarvestController { response.setContentType("application/json") render result as JSON } + + /** + * Get Darwin Core Archive for a project that has ala harvest enabled. + * @param projectId + * @return + * At the moment, you need to add their IP address to whitelist. + */ + def getDarwinCoreArchiveForProject (String projectId) { + if (projectId) { + Project project = Project.findByProjectId(projectId) + if(project?.alaHarvest) { + // Simulate BioCollect as the hostname calling this method. This is done to get the correct URL for + // documents. + DocumentHostInterceptor.documentHostUrlPrefix.set(grailsApplication.config.getProperty("biocollect.baseURL")) + recordService.getDarwinCoreArchiveForProject(response.outputStream, project) + } else + response status: HttpStatus.SC_NOT_FOUND, text: [error: "project not found or ala harvest flag is switched off"] as JSON, contentType: ContentType.APPLICATION_JSON + } else { + response status: HttpStatus.SC_BAD_REQUEST, text: [error: "projectId is required"] as JSON, contentType: ContentType.APPLICATION_JSON + } + } } \ No newline at end of file diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index 1309c84fd..35841b9c2 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -3,11 +3,9 @@ package au.org.ala.ecodata import grails.converters.JSON import groovy.json.JsonSlurper import org.apache.commons.codec.binary.Base64 -import org.apache.http.HttpStatus -import org.apache.http.entity.ContentType import org.springframework.web.multipart.MultipartFile import org.springframework.web.multipart.MultipartHttpServletRequest -import java.text.SimpleDateFormat + import static au.org.ala.ecodata.Status.DELETED import static javax.servlet.http.HttpServletResponse.* /** @@ -28,131 +26,6 @@ class RecordController { def index() {} - /** - * List of supported data resource id available for harvesting. - * Note: Data Provider must be BioCollect or MERIT - * - * @param max = number - * @param offset = number - * @param order = lastUpdated - * @param sort = asc | desc - * - */ - def listHarvestDataResource() { - def result, error - try { - if (params.max && !params.max?.isNumber()) { - error = "Invalid parameter max" - } else if (params.offset && !params.offset?.isNumber()) { - error = "Invalid parameter offset" - } else if (params.sort && params.sort != "asc" && params.sort != "desc") { - error = "Invalid parameter sort" - } else if (params.order && params.order != "lastUpdated") { - error = "Invalid parameter order (Expected: lastUpdated)" - } - - if (!error) { - def pagination = [ - max : params.max ?: 10, - offset: params.offset ?: 0, - order : params.order ?: 'asc', - sort : params.sort ?: 'lastUpdated' - ] - - result = projectService.listProjectForAlaHarvesting(pagination) - - } else { - response.status = HttpStatus.SC_BAD_REQUEST - result = [status: 'error', error: error] - } - - } catch (Exception ex) { - response.status = HttpStatus.SC_INTERNAL_SERVER_ERROR - result << [status: 'error', error: "Unexpected error."] - } - - response.setContentType("application/json") - render result as JSON - } - - /** - * List records associated with the given data resource id - * Data Provider must be BioCollect or MERIT - * @param id dataResourceId - * @param max = number - * @param offset = number - * @param order = lastUpdated - * @param sort = asc | desc | default:asc - * @param lastUpdated = date | dd/MM/yyyy | default:null - * @param status = active | deleted | default:active - * @deprecated ALA's records harvester will use getDarwinCoreArchiveForProject once Events system is setup. - * To access it use archiveURL property from {@link RecordController#listHarvestDataResource}. - * - */ - @Deprecated - def listRecordsForDataResourceId (){ - def result = [], error, project - Date lastUpdated = null - try { - if(!params.id) { - error = "Invalid data resource id" - } else if (params.max && !params.max.isNumber()) { - error = "Invalid max parameter vaue" - } else if (params.offset && !params.offset.isNumber()) { - error = "Invalid offset parameter vaue" - } else if (params.order && params.order != "asc" && params.order != "desc") { - error = "Invalid order parameter value (expected: asc, desc)" - } else if (params.sort && params.sort != "lastUpdated") { - error = "Invalid sort parameter value (expected: lastUpdated)" - } else if (params.status && params.status != "active" && params.status != "deleted") { - error = "Invalid status parameter value (expected: active or deleted)" - } else if(params.id){ - project = projectService.getByDataResourceId(params.id, 'active', 'basic') - error = !project ? 'No valid project found for the given data resource id' : !project.alaHarvest ? "Harvest disabled for data resource id - ${params.id}" : '' - } - - if (params.lastUpdated) { - try{ - def df = new SimpleDateFormat("dd/MM/yyyy") - lastUpdated = df.parse(params.lastUpdated) - } catch (Exception ex) { - error = "Invalid lastUpdated format (Expected date format - Example: dd/MM/yyyy" - } - } - - if (!error && project) { - def args = [ - max : params.max ?: 10, - offset : params.offset ?: 0, - order : params.order ?: 'asc', - sort : params.sort ?: 'lastUpdated', - status : params.status ?: 'active', - projectId: project.projectId - ] - - List restrictedProjectActivities = projectActivityService.listRestrictedProjectActivityIds(null, params.id) - log.debug("Retrieving results...") - result = recordService.listByProjectId(args, lastUpdated, restrictedProjectActivities) - result?.list?.each { - it.projectName = project?.name - it.license = recordService.getLicense(it) - } - } else { - response.status = HttpStatus.SC_BAD_REQUEST - log.error(error.toString()) - result = [status: 'error', error: error] - } - - } catch (Exception ex) { - response.status = HttpStatus.SC_INTERNAL_SERVER_ERROR - log.error(ex.toString()) - result << [status: 'error', error: "Unexpected error."] - } - - response.setContentType("application/json") - render result as JSON - } - /** * Exports all active Records with lat/lng coords into a .csv suitable for use by the Biocache to create occurrence records. @@ -552,25 +425,6 @@ class RecordController { } } - /** - * Get Darwin Core Archive for a project that has ala harvest enabled. - * @param projectId - * @return - */ - def getDarwinCoreArchiveForProject (String projectId) { - if (projectId) { - Project project = Project.findByProjectId(projectId) - if(project?.alaHarvest) { - // Simulate BioCollect as the hostname calling this method. This is done to get the correct URL for - // documents. - DocumentHostInterceptor.documentHostUrlPrefix.set(grailsApplication.config.getProperty("biocollect.baseURL")) - recordService.getDarwinCoreArchiveForProject(response.outputStream, project) - } else - response status: HttpStatus.SC_NOT_FOUND, text: [error: "project not found or ala harvest flag is switched off"] as JSON, contentType: ContentType.APPLICATION_JSON - } else { - response status: HttpStatus.SC_BAD_REQUEST, text: [error: "projectId is required"] as JSON, contentType: ContentType.APPLICATION_JSON - } - } private def setResponseHeadersForRecord(response, record) { response.addHeader("content-location", grailsApplication.config.getProperty('grails.serverURL') + "/record/" + record.occurrenceID) diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index f10ff16e0..a776342fe 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -197,7 +197,7 @@ class UrlMappings { "/ws/project/getDataCollectionWhiteList"(controller: "project"){ action = [GET:"getDataCollectionWhiteList"] } "/ws/project/getBiocollectFacets"(controller: "project"){ action = [GET:"getBiocollectFacets"] } "/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets") - "/ws/project/$projectId/archive"(controller: "record", action: "getDarwinCoreArchiveForProject") + "/ws/project/$projectId/archive"(controller: "harvest", action: "getDarwinCoreArchiveForProject") "/ws/project/$projectId/dataSet/$dataSetId/records"(controller: "project", action: "fetchDataSetRecords") "/ws/project/findStateAndElectorateForProject"(controller: "project", action: "findStateAndElectorateForProject") "/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch") diff --git a/grails-app/services/au/org/ala/ecodata/RecordService.groovy b/grails-app/services/au/org/ala/ecodata/RecordService.groovy index d7dc9bf73..df495ab0d 100644 --- a/grails-app/services/au/org/ala/ecodata/RecordService.groovy +++ b/grails-app/services/au/org/ala/ecodata/RecordService.groovy @@ -1147,6 +1147,7 @@ class RecordService { void getDarwinCoreArchiveForProject(outputStream, Project project) { Map result Map headersByDwcClass = [:].withDefault {[]} + Map dwcGroups = grailsApplication.config.getProperty("darwinCore.termsGroupedByClass", Map) new ZipOutputStream(outputStream).withStream { zip -> try { Long start = System.currentTimeMillis(), end @@ -1162,7 +1163,7 @@ class RecordService { zip.putNextEntry(new ZipEntry("${dwcClass}.csv")) CSVWriter csvWriter = new CSVWriter(new OutputStreamWriter(zip)) List headers = getHeadersFromRows(rows) - List defaultOrder = getHeaderOrder(grailsApplication.config.darwinCore.termsGroupedByClass[dwcClass]) + List defaultOrder = getHeaderOrder(dwcGroups[dwcClass]) headers = reorderHeaders(headers, defaultOrder) headersByDwcClass[dwcClass] = headers csvWriter.writeNext(headers as String[]) @@ -1383,12 +1384,12 @@ class RecordService { } Map getAttributeConfig(String dwcClass, String attribute) { - Map groups = grailsApplication.config.darwinCore.termsGroupedByClass + Map groups = grailsApplication.config.getProperty("darwinCore.termsGroupedByClass", Map) groups[dwcClass].find { it.name == attribute } } Map darwinCoreTermsGroupedByClass() { - Map groups = grailsApplication.config.darwinCore.termsGroupedByClass + Map groups = grailsApplication.config.getProperty("darwinCore.termsGroupedByClass", Map) Map trimmedGroups = [:] groups.each { String key, List values -> trimmedGroups[key] = values?.collect { @@ -1534,7 +1535,8 @@ class RecordService { */ Map convertProjectActivityToEvent (pActivity, project) { String dwcClass = DWC_EVENT - List configs = grailsApplication.config.darwinCore.projectActivityToDwC[dwcClass] + Map paDWC = grailsApplication.config.getProperty("darwinCore.projectActivityToDwC", Map) + List configs = paDWC[dwcClass] Map result = [:].withDefault { [:] } configs.each { config -> transformToEventCoreTerm(config, pActivity, [project: project, pActivity: pActivity, recordService: this], result, dwcClass) diff --git a/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy b/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy new file mode 100644 index 000000000..5a574c90a --- /dev/null +++ b/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy @@ -0,0 +1,242 @@ +package au.org.ala.ecodata + +import au.com.bytecode.opencsv.CSVReader +import com.mongodb.BasicDBObject +import grails.gorm.transactions.Rollback +import grails.testing.mixin.integration.Integration +import grails.util.GrailsWebMockUtil +import groovy.xml.XmlSlurper +import groovy.xml.slurpersupport.GPathResult +import org.grails.plugins.testing.GrailsMockHttpServletRequest +import org.grails.plugins.testing.GrailsMockHttpServletResponse +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.context.WebApplicationContext +import spock.lang.Specification + +import java.util.zip.ZipEntry +import java.util.zip.ZipInputStream + +@Integration +@Rollback +class HarvestControllerSpec extends Specification { + @Autowired + HarvestController harvestController + + @Autowired + WebApplicationContext ctx + + def setup() { + cleanup() + GrailsMockHttpServletRequest grailsMockHttpServletRequest = new GrailsMockHttpServletRequest() + GrailsMockHttpServletResponse grailsMockHttpServletResponse = new GrailsMockHttpServletResponse() + GrailsWebMockUtil.bindMockWebRequest(ctx, grailsMockHttpServletRequest, grailsMockHttpServletResponse) + } + + def cleanup() { + Project.collection.remove(new BasicDBObject()) + ProjectActivity.collection.remove(new BasicDBObject()) + Activity.collection.remove(new BasicDBObject()) + Output.collection.remove(new BasicDBObject()) + ActivityForm.collection.remove(new BasicDBObject()) + Organisation.collection.remove(new BasicDBObject()) + } + + // write integration test for harvestController#getDarwinCoreArchiveForProject + void "getDarwinCoreArchiveForProject should be able to generate event core zip file"() { + setup: + // create organization + Date now = new Date() + Date yesterday = now.minus(1) + String nowStr = DateUtil.formatWithMilliseconds(now) + String yesterdayStr = DateUtil.formatWithMilliseconds(yesterday) + def organisation = new Organisation(name: "Test Organisation", organisationId: "org1") + organisation.save(flush: true, failOnError: true) + def project = new Project ( + name: "Test Project", + alaHarvest: true, + projectId: "project1", + organisationId: organisation.organisationId, + organisationName: "Test Organisation", + description: "Test project description", + dataResourceId: "dr1", + dateCreated: yesterday + ) + project.save(flush: true, failOnError: true) + def activityForm = new ActivityForm( + name: 'testForm', + publicationStatus: "published", + status: "active", + type: "Activity", + formVersion: 1, + sections: [ + new FormSection( + name: "section1", + template: [ + dataModel: [ + [ + name : "testField", + dataType : "text", + description : "event remarks", + dwcAttribute: "eventRemarks", + ], + [ + name : "speciesName", + dataType : "species", + description: "species field" + ], + [ + dataType : "number", + name : "distance", + dwcAttribute : "measurementValue", + measurementType : "number", + measurementTypeID : "http://qudt.org/vocab/quantitykind/Number", + measurementAccuracy: "0.001", + measurementUnit : "m", + measurementUnitID : "http://qudt.org/vocab/unit/M", + description : "distance from source" + ], + [ + dataType : "image", + name : "images", + description: "photo of species" + ] + ] + ] + ) + ] + ) + activityForm.save(failOnError: true, flush: true) + def projectActivity = new ProjectActivity( + name: "Test Project Activity", + projectId: project.projectId, + projectActivityId: "pa1", + published: true, + endDate: now, startDate: yesterday, + status: "active", + dataSharingLicense: "CC BY 4.0", + description: "Test event remarks", + methodName: "opportunistic", + pActivityFormName: activityForm.name, + methodType: "opportunistic", + spatialAccuracy: "low", + speciesIdentification: "high", + temporalAccuracy: "moderate", + nonTaxonomicAccuracy: "high", + dataQualityAssuranceMethods: ["dataownercurated"], + dataAccessMethods: ["na"], + ) + projectActivity.save(flush: true, failOnError: true) + // create an activity + def activity = new Activity(name: "Test Activity", + projectId: project.projectId, + projectActivityId: projectActivity.projectActivityId, + activityId: 'activity1', + siteId: 'site1' + ) + activity.save(flush: true, failOnError: true) + def document = new Document( + documentId: "document1", + status: 'active', + imageId: 'image1', + creator : "John Doe", + contentType: 'image/png', + rightsHolder: "John Doe", + license : "CC BY 4.0" + ) + document.save(flush: true, failOnError: true) + // create an output + def output = new Output( + name: "section1", + activityId: activity.activityId, + outputId: 'output1' + ) + output.data = [ + testField : "Test event remarks", + speciesName: [name: "Anura (Frog)", scientificName: 'Anura', commonName: "Frog", outputSpeciesId: "outputSpecies1"], + distance : 1.0, + images : [[ + documentId : "document1", + identifier : "http://example.com/image", + creator : "John Doe", + title : "Image of a frog", + rightHolder: "John Doe", + license : "CC BY 4.0", + ]] + ] + output.save(flush: true, failOnError: true) + // + when: + harvestController.getDarwinCoreArchiveForProject(project.projectId) + + then: + harvestController.response.status == 200 + ByteArrayInputStream zipStream = new ByteArrayInputStream(harvestController.response.contentAsByteArray) + ZipInputStream zipInputStream = new ZipInputStream(zipStream) + ZipEntry entry + while ((entry = zipInputStream.getNextEntry()) != null) { + StringBuffer buffer = new StringBuffer() + BufferedReader reader = new BufferedReader(new InputStreamReader(zipInputStream)) + String line + String content = reader.lines().collect().join("\n") + + switch (entry.name) { + case "eml.xml": + // read content of eml.xml and check if it contains the correct project + // and project activity information + GPathResult xml = new XmlSlurper().parseText(content) + assert xml?.dataset?.title.text().trim() == project.name + assert xml?.dataset?.creator?.organizationName.text().trim() == organisation.name + assert xml?.dataset?.abstract?.para.text().trim() == project.description + break + case "meta.xml": + List spokes = ["Occurrence.csv", "MeasurementOrFact.csv", "Media.csv"] + GPathResult xml = new XmlSlurper().parseText(content) + assert xml?.core?.files.location.text().trim() == "Event.csv" + assert xml?.core?.field?.size() > 1 + assert xml?.core?.id?.size() == 1 + assert spokes.contains(xml?.extension.files.location[0].text().trim()) + assert spokes.contains(xml?.extension.files.location[1].text().trim()) + assert spokes.contains(xml?.extension.files.location[2].text().trim()) + assert xml?.extension[0].coreid.size() == 1 + assert xml?.extension[1].coreid.size() == 1 + assert xml?.extension[2].coreid.size() == 1 + break + case "Event.csv": + CSVReader readerCSV = new CSVReader(new StringReader(content)) + List lines = readerCSV.readAll() + assert lines[0] == ["eventID","parentEventID","eventType","eventDate","eventRemarks","samplingProtocol","geodeticDatum", "locationID", "endDate", "dataSharingLicense", "license", "name", "pActivityFormName", "startDate"] + assert lines[1] == ["pa1", "", "Survey", "${yesterdayStr}/${nowStr}", "Test event remarks", "opportunistic", "","" , nowStr, "CC BY 4.0", "", "Test Project Activity", "testForm", yesterdayStr] + assert lines[2][0] == "activity1" + assert lines[2][1] == "pa1" + assert lines[2][2] == "SiteVisit" + assert lines[2][4] == "Test event remarks" + assert lines.size() == 3 + break + case "Media.csv": + CSVReader readerCSV = new CSVReader(new StringReader(content)) + List lines = readerCSV.readAll() + assert lines.size() == 2 + assert lines[0] == ["eventID","occurrenceID","type","identifier","format","creator","licence","rightsHolder"] + assert lines[1] == ["activity1","outputSpecies1","","","","","",""] + // check Media.csv + break + case "Occurrence.csv": + // check Occurrence.csv + CSVReader readerCSV = new CSVReader(new StringReader(content)) + List lines = readerCSV.readAll() + assert lines[0] == ["eventID","occurrenceID","basisOfRecord","scientificName","occurrenceStatus","individualCount"] + assert lines[1] == ["activity1","outputSpecies1","HumanObservation","Anura","present","1"] + assert lines.size() == 2 + break + case "MeasurementOrFact.csv": + // check MeasurementOrFact.csv + CSVReader readerCSV = new CSVReader(new StringReader(content)) + List lines = readerCSV.readAll() + assert lines[0] == ["eventID","occurrenceID","measurementValue","measurementAccuracy","measurementUnit","measurementUnitID","measurementType","measurementTypeID"] + assert lines[1] == ["activity1","outputSpecies1","1.0","0.001","m","http://qudt.org/vocab/unit/M","distance from source","http://qudt.org/vocab/quantitykind/Number"] + assert lines.size() ==2 + break + } + } + } +} diff --git a/src/main/groovy/au/org/ala/ecodata/converter/AudioConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/AudioConverter.groovy index 9cb3b8c02..0dde9fef7 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/AudioConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/AudioConverter.groovy @@ -5,7 +5,7 @@ class AudioConverter implements RecordFieldConverter { private static final String DEFAULT_RIGHTS_STATEMENT = "The rights to all uploaded audio are held equally, under a Creative Commons Attribution (CC-BY v3.0) license, by the contributor of the audio and the primary organisation responsible for the project to which they are contributed." private static final String DEFAULT_LICENCE = "Creative Commons Attribution" - List convert(Map data, Map metadata = [:]) { + List convert(Map data, Map metadata = [:], Map context = [:]) { Map record = [:] record.multimedia = data[metadata.name] diff --git a/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy index 3a4cae88a..c9f01f325 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy @@ -2,7 +2,7 @@ package au.org.ala.ecodata.converter class FeatureConverter implements RecordFieldConverter { - List convert(Map data, Map metadata = [:]) { + List convert(Map data, Map metadata = [:], Map context = [:]) { Map record = [:] if (data[metadata.name]) { Double latitude = getDecimalLatitude(data[metadata.name]) diff --git a/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy index b6ecdc635..67230045f 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy @@ -2,7 +2,7 @@ package au.org.ala.ecodata.converter class GenericFieldConverter implements RecordFieldConverter { - List convert(Map data, Map metadata = [:]) { + List convert(Map data, Map metadata = [:], Map context = [:]) { Map record = [:] @@ -20,7 +20,8 @@ class GenericFieldConverter implements RecordFieldConverter { Map dwcMappings = extractDwcMapping(metadata) - Map dwcAttributes = getDwcAttributes(data, dwcMappings, metadata) + context.record = record + Map dwcAttributes = getDwcAttributes(data, dwcMappings, metadata, context) record = RecordConverter.overrideAllExceptLists(dwcAttributes, record) if (data?.dwcAttribute) { diff --git a/src/main/groovy/au/org/ala/ecodata/converter/ImageConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/ImageConverter.groovy index 350ec1d5e..e5eed0c98 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/ImageConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/ImageConverter.groovy @@ -6,7 +6,7 @@ class ImageConverter implements RecordFieldConverter { private static final String DEFAULT_LICENCE = "CC BY 4.0" private static final String TYPE = "StillImage" - List convert(Map data, Map metadata = [:]) { + List convert(Map data, Map metadata = [:], Map context = [:]) { Map record = [:] record.multimedia = data[metadata.name] diff --git a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy index dcfe29e14..884d96f53 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy @@ -7,7 +7,7 @@ import groovy.util.logging.Slf4j class ListConverter implements RecordFieldConverter { @Override - List convert(Map data, Map outputMetadata = [:]) { + List convert(Map data, Map outputMetadata = [:], Map context = [:]) { List records = [] int index = 0 @@ -44,28 +44,26 @@ class ListConverter implements RecordFieldConverter { // For each singleItemModel, get the appropriate field converter for the data type, generate the individual // Record fields and add them to the skeleton Record baseRecordModels?.each { Map dataModel -> +// context.record = baseRecord RecordFieldConverter converter = RecordConverter.getFieldConverter(dataModel.dataType) - List recordFieldSets = converter.convert(row, dataModel) + List recordFieldSets = converter.convert(row, dataModel, context) Map recordFieldSet = recordFieldSets[0] baseRecord = RecordConverter.overrideAllExceptLists(baseRecord, recordFieldSet) RecordConverter.updateEventIdToMeasurements(baseRecord[PROP_MEASUREMENTS_OR_FACTS], baseRecord.activityId) - -// TODO: delete? commented code was in dev branch, removed on merge. -// if (recordFieldSets[0]) -// baseRecord << recordFieldSets[0] } // For each species dataType, where present we will generate a new record speciesModels?.each { Map dataModel -> +// context.record = baseRecord RecordFieldConverter converter = RecordConverter.getFieldConverter(dataModel.dataType) - List recordFieldSets = converter.convert(row, dataModel) + List recordFieldSets = converter.convert(row, dataModel, context) if (recordFieldSets) { Map speciesRecord = RecordConverter.overrideFieldValues(baseRecord, recordFieldSets[0]) // We want to create a record in the DB only if species information is present if (speciesRecord.outputSpeciesId) { speciesRecord.outputItemId = index++ - RecordConverter.updateSpeciesIdToMeasurements(speciesRecord[PROP_MEASUREMENTS_OR_FACTS], speciesRecord.outputSpeciesId) + RecordConverter.updateSpeciesIdToMeasurements(speciesRecord[PROP_MEASUREMENTS_OR_FACTS], speciesRecord.occurrenceID) records << speciesRecord } else { log.warn("Record [${speciesRecord}] does not contain full species information. " + @@ -79,8 +77,9 @@ class ListConverter implements RecordFieldConverter { // sets which will be converted into Records. For each field set, add a copy of the skeleton Record so it has // all the common fields multiItemModels?.each { Map dataModel -> +// context.record = baseRecord RecordFieldConverter converter = RecordConverter.getFieldConverter(dataModel.dataType) - List recordFieldSets = converter.convert(row, dataModel) + List recordFieldSets = converter.convert(row, dataModel, context) recordFieldSets.each { Map rowRecord = RecordConverter.overrideFieldValues(baseRecord, it) diff --git a/src/main/groovy/au/org/ala/ecodata/converter/MasterDetailConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/MasterDetailConverter.groovy index fb641529c..14ef4d21a 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/MasterDetailConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/MasterDetailConverter.groovy @@ -3,14 +3,14 @@ package au.org.ala.ecodata.converter class MasterDetailConverter implements RecordFieldConverter { @Override - List convert(Map data, Map outputMetadata = [:]) { + List convert(Map data, Map outputMetadata = [:], Map context = [:]) { // delegate the conversion to a specific converter for the DETAIL portion of the master/detail RecordFieldConverter converter = RecordConverter.getFieldConverter(outputMetadata.detail.dataType) List records = [] data[outputMetadata.name].each { - records.addAll converter.convert(it, outputMetadata.detail as Map) + records.addAll converter.convert(it, outputMetadata.detail as Map, context) } records diff --git a/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy index 800c6b171..5429c37d2 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy @@ -80,6 +80,7 @@ class RecordConverter { userId : activity.userId, multimedia : [] ] + Map context = ['project':project, 'organisation':organisation, 'site':site, 'projectActivity':projectActivity, 'activity':activity, 'output':output, outputMetadata: outputMetadata, rootData: data] Long start = System.currentTimeMillis(), end @@ -112,7 +113,7 @@ class RecordConverter { // Record fields and add them to the skeleton Record baseRecordModels?.each { Map dataModel -> RecordFieldConverter converter = getFieldConverter(dataModel.dataType) - List recordFieldSets = converter.convert(data, dataModel) + List recordFieldSets = converter.convert(data, dataModel, context) baseRecord = overrideAllExceptLists(baseRecord, recordFieldSets[0]) updateEventIdToMeasurements(baseRecord[PROP_MEASUREMENTS_OR_FACTS], baseRecord.activityId) } @@ -121,7 +122,7 @@ class RecordConverter { // For each species dataType, where present we will generate a new record speciesModels?.each { Map dataModel -> RecordFieldConverter converter = getFieldConverter(dataModel.dataType) - List recordFieldSets = converter.convert(data, dataModel) + List recordFieldSets = converter.convert(data, dataModel, context) Map speciesRecord = overrideAllExceptLists(baseRecord, recordFieldSets[0]) // We want to create a record in the DB only if species guid is present i.e. species is valid if(recordGeneration){ @@ -134,7 +135,7 @@ class RecordConverter { } } else { - updateSpeciesIdToMeasurements(speciesRecord[PROP_MEASUREMENTS_OR_FACTS], speciesRecord.outputSpeciesId) + updateSpeciesIdToMeasurements(speciesRecord[PROP_MEASUREMENTS_OR_FACTS], speciesRecord.occurrenceID) records << speciesRecord } } @@ -148,7 +149,7 @@ class RecordConverter { // all the common fields multiItemModels?.each { Map dataModel -> RecordFieldConverter converter = getFieldConverter(dataModel.dataType) - List recordFieldSets = converter.convert(data, dataModel) + List recordFieldSets = converter.convert(data, dataModel, context) recordFieldSets.each { Map rowRecord = overrideAllExceptLists(baseRecord, it) @@ -171,6 +172,13 @@ class RecordConverter { else if(!speciesModels && !recordGeneration) { records << baseRecord } + + // Add measurements or facts only when event core archive is being created. + if (recordGeneration) { + records.each { + it.remove(PROP_MEASUREMENTS_OR_FACTS) + } + } end = System.currentTimeMillis() log.debug("Time in milliseconds to convert nested data model - ${end - start}") // We are now left with a list of one or more Maps, where each Map contains all the fields for an individual Record. @@ -179,9 +187,7 @@ class RecordConverter { /** * Return a new Map with the union of source and additional giving precedence to values from additional - * - * - * If the same key already exists in target it will be overriden + * If the same key already exists in target it will be overridden * @param source the original entries * @param additional Entries with precedence * @return diff --git a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy index 66b95aafa..37918c510 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy @@ -1,5 +1,4 @@ package au.org.ala.ecodata.converter - /** * Converts an Output's data model into one or more Records. * @@ -11,6 +10,8 @@ package au.org.ala.ecodata.converter */ trait RecordFieldConverter { static String DWC_ATTRIBUTE_NAME = "dwcAttribute" + static String DWC_EXPRESSION = "dwcExpression" + static String DWC_DEFAULT = "dwcDefault" static String DWC_MEASUREMENT_VALUE = "measurementValue" static String DWC_MEASUREMENT_TYPE = "measurementType" static String DWC_MEASUREMENT_TYPE_ID = "measurementTypeID" @@ -22,6 +23,7 @@ trait RecordFieldConverter { abstract List convert(Map data) abstract List convert(Map data, Map outputMetadata) + abstract List convert(Map data, Map outputMetadata, Map context) Double toDouble(val) { Double result = null @@ -41,10 +43,16 @@ trait RecordFieldConverter { dwcMappings } - Map getDwcAttributes(Map data, Map dwcMappings, Map metadata = null) { + // write test cases for this method + + Map getDwcAttributes(Map data, Map dwcMappings, Map metadata = null, Map context = [:]) { Map fields = [:] dwcMappings.each { dwcAttribute, fieldName -> + if(metadata?.containsKey(DWC_EXPRESSION)) { + data[fieldName] = evaluateExpression((String)metadata[DWC_EXPRESSION], data, metadata[DWC_DEFAULT], metadata, context) + } + fields[dwcAttribute] = data[fieldName] } @@ -66,7 +74,7 @@ trait RecordFieldConverter { Map measurement = [:] measurement[DWC_MEASUREMENT_VALUE] = value - measurement[DWC_MEASUREMENT_TYPE] = getMeasurementType(metadata, data) + measurement[DWC_MEASUREMENT_TYPE] = getMeasurementType(metadata, data, context) if (metadata?.containsKey(DWC_MEASUREMENT_TYPE_ID)) { measurement[DWC_MEASUREMENT_TYPE_ID] = metadata[DWC_MEASUREMENT_TYPE_ID] @@ -98,19 +106,33 @@ trait RecordFieldConverter { * @param data * @return */ - String getMeasurementType(metadata, data) { + def getMeasurementType(Map metadata, Map data, Map context = [:]) { def defaultValue = metadata?.description ?: metadata?.value ?: metadata?.name if (metadata?.measurementType && data) { - try { - def engine = new groovy.text.SimpleTemplateEngine() - return engine.createTemplate(metadata?.measurementType).make(data) - } - catch (Exception ex) { - return defaultValue - } + return evaluateExpression(metadata.measurementType, data, defaultValue, metadata, context) } else { return defaultValue } } + def evaluateExpression (String expression, Map data, def defaultValue, Map metadata = null, Map context = [:]) { + try { + Map clonedData = data.clone() + context.metadata = metadata + clonedData.context = context + def binding = new Binding(clonedData) + GroovyShell shell = new GroovyShell(binding) + return shell.evaluate(expression) + } + catch (MissingPropertyException exp) { + // This could happen if the expression references a field that is not in the data. + // Or, what is passed is a string and not an expression. This could happen when getMeasurementType calls this method. + // In such cases, return the value of measurementType i.e. expression itself. + return expression + } + catch (Exception ex) { + return defaultValue + } + } + } \ No newline at end of file diff --git a/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy index f28d5bf01..1c0e5a706 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy @@ -5,7 +5,7 @@ class SpeciesConverter implements RecordFieldConverter { "(Unmatched taxon)" ] - List convert(Map data, Map metadata = [:]) { + List convert(Map data, Map metadata = [:], Map context = [:]) { if ((data == null) || (data[metadata.name] == null) ) { return [] } @@ -26,7 +26,7 @@ class SpeciesConverter implements RecordFieldConverter { data[metadata.name].outputSpeciesId = UUID.randomUUID().toString() } - record.outputSpeciesId = data[metadata.name]?.outputSpeciesId + record.occurrenceID = record.outputSpeciesId = data[metadata.name]?.outputSpeciesId [record] } diff --git a/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy index 21d547f0b..8d1973b3a 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy @@ -1,5 +1,6 @@ package au.org.ala.ecodata.converter +import au.org.ala.ecodata.* import groovy.json.JsonSlurper import spock.lang.Specification @@ -75,6 +76,118 @@ class GenericConverterSpec extends Specification { result[0].decimalLongitude == 3.0 } + def "convert should handle expression to evaluate value of dwc attribute"() { + setup: + Map data = [ + "juvenile" : 2, + "adult": 3 + ] + + Map metadata = [ + name: "juvenile", + dwcAttribute: "individualCount", + dwcExpression: "context.metadata.description + ' ' + (juvenile + adult)", + dwcDefault: 0, + description: "Total number of individuals" + ] + GenericFieldConverter converter = new GenericFieldConverter() + Map context = [:] + + when: + List result = converter.convert(data, metadata) + + then: + result.size() == 1 + result[0].individualCount == "Total number of individuals 5" + } + def "convert should handle expressions in groovy, access information from context and add multiple darwin core attributes"() { + setup: + Map data = [ + "juvenile" : 2, + "adult": 3 + ] + + Map otherMetadata = [ + name: "adult", + dwcAttribute: "adultCount", + description: "Total number of adult individuals" + ] + Map metadata = [ + name: "juvenile", + dataType: "number", + dwcAttribute: "individualCount", + dwcExpression: "context.record.projectId = context.project.projectId; context.record.organisationId = context.organisation.organisationId; context.record.individualCount = juvenile + adult;", + dwcDefault: 0, + description: "Total number of individuals" + ] + + GenericFieldConverter converter = new GenericFieldConverter() + Map context = [ + 'project': new Project(projectId: 'project1'), + 'organisation': new Organisation(organisationId: 'org1'), + 'site': new Site(siteId: 'site1'), + 'projectActivity': new ProjectActivity(projectActivityId: 'pa1'), + 'activity':new Activity(activityId: 'activity1', siteId: 'site1'), + 'output': new Output(outputId: 'output1', activityId: 'activity1'), + outputMetadata: [dataModel: [metadata,otherMetadata]], + rootData: data + ] + when: + List result = converter.convert(data, metadata, context) + + then: + result.size() == 1 + result[0].individualCount == 5 + result[0].projectId == "project1" + result[0].organisationId == "org1" + } + + def "convert should return expression if binding not found"() { + setup: + Map data = [ + "juvenile" : 2 + ] + + Map metadata = [ + name: "juvenile", + dwcAttribute: "individualCount", + dwcExpression: "juvenile + adult", + dwcDefault: 0, + description: "Total number of individuals" + ] + GenericFieldConverter converter = new GenericFieldConverter() + + when: + List result = converter.convert(data, metadata) + + then: + result.size() == 1 + result[0].individualCount == "juvenile + adult" + } + + def "convert should return default value for all other exceptions"() { + setup: + Map data = [ + "juvenile" : 2, + adult: null + ] + + Map metadata = [ + name: "juvenile", + dwcAttribute: "individualCount", + dwcExpression: "juvenile + adult", + dwcDefault: 0, + description: "Total number of individuals" + ] + GenericFieldConverter converter = new GenericFieldConverter() + + when: + List result = converter.convert(data, metadata) + + then: + result.size() == 1 + result[0].individualCount == 0 + } } diff --git a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy index 70a2e9cc8..c95f70026 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy @@ -409,4 +409,164 @@ class RecordConverterSpec extends Specification { fieldsets[0].dwc == "firstValue" fieldsets[1].dwc == "secondValue" } + + + def "converter should add measurements or facts to the record field set"() { + setup: + Project project = new Project(projectId: "project1") + Organisation organisation = new Organisation(orgId: "org1") + Site site = new Site(siteId: "site1") + ProjectActivity projectActivity = new ProjectActivity(projectActivityId: "pa1") + Activity activity = new Activity(activityId: "act1") + Output output = new Output(outputId: "output1", activityId: "act1") + Map outputMetadata = [ + record: true, + dataModel: [ + [ + dataType: "text", + name: "field1", + dwcAttribute: "attribute1" + ], + [ + dataType: "number", + name: "distance", + dwcAttribute: "measurementValue", + measurementType: "number", + measurementTypeID: "http://qudt.org/vocab/quantitykind/Number", + measurementAccuracy: "0.001", + measurementUnit: "m", + measurementUnitID: "http://qudt.org/vocab/unit/M", + description: "distance from source" + ] + ] + ] + Map submittedData = [ + "field1": "fieldValue1", + "distance": 10.056 + ] + + when: + List fieldsets = RecordConverter.convertRecords(project, organisation, site, projectActivity, activity, output, submittedData, outputMetadata, false) + + then: + fieldsets.size() == 1 + fieldsets[0].attribute1 == "fieldValue1" + fieldsets[0].measurementsorfacts.size() == 1 + fieldsets[0].measurementsorfacts[0].measurementValue == 10.056 + fieldsets[0].measurementsorfacts[0].measurementType == "number" + fieldsets[0].measurementsorfacts[0].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number" + fieldsets[0].measurementsorfacts[0].measurementAccuracy == "0.001" + fieldsets[0].measurementsorfacts[0].measurementUnit == "m" + fieldsets[0].measurementsorfacts[0].measurementUnitID == "http://qudt.org/vocab/unit/M" + + when: "the measurement is associated with a species and non-species" + outputMetadata = [ + record: true, + dataModel: [ + [ + dataType: "number", + name: "field1", + dwcAttribute: "attribute1" + ], + [ + dataType: "text", + name: "field2", + dwcAttribute: "attribute2", + dwcExpression: "field1 == 0 ? \"absent\" : \"present\"" + ], + [ + dataType: "number", + name: "distance", + dwcAttribute: "measurementValue", + measurementType: "number", + measurementTypeID: "http://qudt.org/vocab/quantitykind/Number", + measurementAccuracy: "0.001", + measurementUnit: "m", + measurementUnitID: "http://qudt.org/vocab/unit/M", + description: "distance from source" + ], + [ + dataType: "list", + name: "speciesList", + columns: [ + [ + dataType: "species", + name: "speciesField", + dwcAttribute: "species" + ], + [ + dataType: "number", + name: "distance", + dwcAttribute: "measurementValue", + dwcExpression: "context.rootData.field1 + distance", + measurementType: "number", + measurementTypeID: "http://qudt.org/vocab/quantitykind/Number", + measurementAccuracy: "0.001", + measurementUnit: "m", + measurementUnitID: "http://qudt.org/vocab/unit/M", + description: "distance from source" + ] + ] + ] + ] + ] + + + submittedData = [ + "field1": 1, + "distance": 10.056, + "speciesList": [ + [ + "speciesField": [commonName: "commonName1", scientificName: "scientificName1", "outputSpeciesId": "speciesFieldId1", "guid": "someguid1"], + "distance": 20.056 + ], + [ + "speciesField": [commonName: "commonName2", scientificName: "scientificName2", "outputSpeciesId": "speciesFieldId2", "guid": "someguid2"], + "distance": 22.056 + ] + ] + ] + + fieldsets = RecordConverter.convertRecords(project, organisation, site, projectActivity, activity, output, submittedData, outputMetadata, false) + + then: + fieldsets.size() == 2 + fieldsets[0].attribute1 == 1 + fieldsets[0].attribute2 == "present" + fieldsets[0].scientificName == "scientificName1" + fieldsets[0].vernacularName == "commonName1" + fieldsets[0].scientificNameID == "someguid1" + fieldsets[0].name == "scientificName1" + fieldsets[0].outputSpeciesId == "speciesFieldId1" + fieldsets[0].occurrenceID == "speciesFieldId1" + fieldsets[0].measurementsorfacts.size() == 3 + fieldsets[0].measurementsorfacts[0].eventID == "act1" + fieldsets[0].measurementsorfacts[0].measurementValue == 10.056 + fieldsets[0].measurementsorfacts[0].measurementType == "number" + fieldsets[0].measurementsorfacts[0].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number" + fieldsets[0].measurementsorfacts[0].measurementAccuracy == "0.001" + fieldsets[0].measurementsorfacts[0].measurementUnit == "m" + fieldsets[0].measurementsorfacts[0].measurementUnitID == "http://qudt.org/vocab/unit/M" + fieldsets[0].measurementsorfacts[1].measurementValue == 21.056 + fieldsets[0].measurementsorfacts[1].measurementType == "number" + fieldsets[0].measurementsorfacts[1].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number" + fieldsets[0].measurementsorfacts[1].measurementAccuracy == "0.001" + fieldsets[0].measurementsorfacts[1].measurementUnit == "m" + fieldsets[0].measurementsorfacts[1].measurementUnitID == "http://qudt.org/vocab/unit/M" + fieldsets[0].measurementsorfacts[2].measurementValue == 23.056 + fieldsets[0].measurementsorfacts[2].measurementType == "number" + fieldsets[0].measurementsorfacts[2].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number" + fieldsets[0].measurementsorfacts[2].measurementAccuracy == "0.001" + fieldsets[0].measurementsorfacts[2].measurementUnit == "m" + fieldsets[0].measurementsorfacts[2].measurementUnitID == "http://qudt.org/vocab/unit/M" + fieldsets[1].attribute1 == 1 + fieldsets[1].attribute2 == "present" + fieldsets[1].scientificName == "scientificName2" + fieldsets[1].vernacularName == "commonName2" + fieldsets[1].scientificNameID == "someguid2" + fieldsets[1].name == "scientificName2" + fieldsets[1].outputSpeciesId == "speciesFieldId2" + fieldsets[1].occurrenceID == "speciesFieldId2" + fieldsets[1].measurementsorfacts.size() == 3 + } } From a7c5941399885fcbecf4edf409d332d46a58b3c2 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 7 Feb 2025 08:08:28 +1100 Subject: [PATCH 12/22] - added more checks - better multimedia data extraction --- grails-app/conf/application.groovy | 25 ++++++++----------- .../ala/ecodata/HarvestControllerSpec.groovy | 16 ++++++++---- .../converter/RecordFieldConverter.groovy | 16 +++++++++--- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 19bbb104d..dc67faf64 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -1,5 +1,5 @@ import au.org.ala.ecodata.Document -import au.org.ala.ecodata.Status +import static au.org.ala.ecodata.Status.DELETED import org.joda.time.DateTime import org.joda.time.DateTimeZone @@ -1647,23 +1647,20 @@ if (!darwinCore.termsGroupedByClass) { [ "name" : "multimedia", "substitute": { record, params -> - record?.multimedia?.collect { multimedia -> - def identifier = multimedia?.identifier - - if (multimedia.documentId) { - def document = Document.findByDocumentIdAndStatusNotEqual(multimedia.documentId, Status.DELETED) + record?.multimedia?.collect { mediaRecord -> + if (mediaRecord.documentId) { + Document document = Document.findByDocumentIdAndStatusNotEqual(mediaRecord.documentId, DELETED) if (document) { - identifier = document.getUrl() - multimedia = document + String identifier = document.getUrl() return [ "eventID" : params?.activity?.activityId, "occurrenceID": record?.occurrenceID , - "type" : multimedia?.type, - "identifier" : identifier, - "format" : multimedia?.contentType, - "creator" : multimedia?.creator, - "licence" : multimedia?.license, - "rightsHolder": multimedia?.rightsHolder + "type" : document?.type ?: mediaRecord.type, + "identifier" : identifier ?: mediaRecord.identifier, + "format" : document?.contentType ?: mediaRecord.contentType, + "creator" : document?.creator ?: document?.attribution ?: mediaRecord.creator, + "licence" : document?.licence ?: document?.license ?: mediaRecord.license, + "rightsHolder": document?.rightsHolder ?: document?.attribution ?: mediaRecord.rightsHolder ] } } diff --git a/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy b/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy index 5a574c90a..90571282a 100644 --- a/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy +++ b/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy @@ -39,6 +39,7 @@ class HarvestControllerSpec extends Specification { Output.collection.remove(new BasicDBObject()) ActivityForm.collection.remove(new BasicDBObject()) Organisation.collection.remove(new BasicDBObject()) + Document.collection.remove(new BasicDBObject()) } // write integration test for harvestController#getDarwinCoreArchiveForProject @@ -136,12 +137,12 @@ class HarvestControllerSpec extends Specification { activity.save(flush: true, failOnError: true) def document = new Document( documentId: "document1", + filename: 'z.png', status: 'active', imageId: 'image1', - creator : "John Doe", contentType: 'image/png', - rightsHolder: "John Doe", - license : "CC BY 4.0" + attribution: "John Doe", + licence : "CC BY 4.0" ) document.save(flush: true, failOnError: true) // create an output @@ -156,6 +157,7 @@ class HarvestControllerSpec extends Specification { distance : 1.0, images : [[ documentId : "document1", + filename : "z.png", identifier : "http://example.com/image", creator : "John Doe", title : "Image of a frog", @@ -173,7 +175,9 @@ class HarvestControllerSpec extends Specification { ByteArrayInputStream zipStream = new ByteArrayInputStream(harvestController.response.contentAsByteArray) ZipInputStream zipInputStream = new ZipInputStream(zipStream) ZipEntry entry + int numberOfEntries = 0 while ((entry = zipInputStream.getNextEntry()) != null) { + numberOfEntries ++ StringBuffer buffer = new StringBuffer() BufferedReader reader = new BufferedReader(new InputStreamReader(zipInputStream)) String line @@ -217,7 +221,7 @@ class HarvestControllerSpec extends Specification { List lines = readerCSV.readAll() assert lines.size() == 2 assert lines[0] == ["eventID","occurrenceID","type","identifier","format","creator","licence","rightsHolder"] - assert lines[1] == ["activity1","outputSpecies1","","","","","",""] + assert lines[1] == ["activity1","outputSpecies1","StillImage","https://images-test.ala.org.au/proxyImage?id=image1","image/png","John Doe","CC BY 4.0","John Doe"] // check Media.csv break case "Occurrence.csv": @@ -233,10 +237,12 @@ class HarvestControllerSpec extends Specification { CSVReader readerCSV = new CSVReader(new StringReader(content)) List lines = readerCSV.readAll() assert lines[0] == ["eventID","occurrenceID","measurementValue","measurementAccuracy","measurementUnit","measurementUnitID","measurementType","measurementTypeID"] - assert lines[1] == ["activity1","outputSpecies1","1.0","0.001","m","http://qudt.org/vocab/unit/M","distance from source","http://qudt.org/vocab/quantitykind/Number"] + assert lines[1] == ["activity1","outputSpecies1","1.0","0.001","m","http://qudt.org/vocab/unit/M","number","http://qudt.org/vocab/quantitykind/Number"] assert lines.size() ==2 break } } + + assert numberOfEntries == 6 } } diff --git a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy index 37918c510..df5bf2ecd 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy @@ -1,4 +1,8 @@ package au.org.ala.ecodata.converter + +import org.apache.commons.logging.Log +import org.apache.commons.logging.LogFactory + /** * Converts an Output's data model into one or more Records. * @@ -9,6 +13,7 @@ package au.org.ala.ecodata.converter * target Record attribute is different (the converter is responsible for mapping one to the other). */ trait RecordFieldConverter { + static Log log = LogFactory.getLog(RecordFieldConverter.class) static String DWC_ATTRIBUTE_NAME = "dwcAttribute" static String DWC_EXPRESSION = "dwcExpression" static String DWC_DEFAULT = "dwcDefault" @@ -115,22 +120,25 @@ trait RecordFieldConverter { } } - def evaluateExpression (String expression, Map data, def defaultValue, Map metadata = null, Map context = [:]) { + def evaluateExpression (String expression, Map data = [:], def defaultValue, Map metadata = null, Map context = [:]) { try { - Map clonedData = data.clone() context.metadata = metadata - clonedData.context = context - def binding = new Binding(clonedData) + data.context = context + def binding = new Binding(data) GroovyShell shell = new GroovyShell(binding) + data.remove('context') return shell.evaluate(expression) } catch (MissingPropertyException exp) { + data.remove('context') // This could happen if the expression references a field that is not in the data. // Or, what is passed is a string and not an expression. This could happen when getMeasurementType calls this method. // In such cases, return the value of measurementType i.e. expression itself. return expression } catch (Exception ex) { + data.remove('context') + log.error(ex.message, ex) return defaultValue } } From 14a5ac9a424e64a410b17ba76b5022e1e6236af0 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 7 Feb 2025 09:37:32 +1100 Subject: [PATCH 13/22] - fixed failing test --- grails-app/conf/application.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index dc67faf64..c03660ff1 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -548,7 +548,7 @@ if (!biocacheService.baseURL) { biocacheService.baseURL = 'https://biocache.ala.org.au/ws' } if (!imagesService.baseURL) { - imagesService.baseURL = 'https://images-dev.ala.org.au' + imagesService.baseURL = 'https://images-test.ala.org.au' } if (!security.cas.bypass) { security.cas.bypass = false From 1766d42958e8d6fcaa4ea84084d4a9d082ce1e1c Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 7 Feb 2025 10:21:21 +1100 Subject: [PATCH 14/22] - fixed failing test --- .../au/org/ala/ecodata/converter/RecordFieldConverter.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy index df5bf2ecd..01ba7cb52 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy @@ -126,8 +126,9 @@ trait RecordFieldConverter { data.context = context def binding = new Binding(data) GroovyShell shell = new GroovyShell(binding) + def value = shell.evaluate(expression) data.remove('context') - return shell.evaluate(expression) + return value } catch (MissingPropertyException exp) { data.remove('context') From 9a50912fc1eac90db7f1cb1dffd84985bb26dd01 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 7 Feb 2025 17:11:23 +1100 Subject: [PATCH 15/22] code review comments --- build.gradle | 2 +- grails-app/conf/application.groovy | 4 +- .../org/ala/ecodata/HarvestController.groovy | 7 +-- .../converter/RecordFieldConverter.groovy | 15 ++++-- .../converter/GenericConverterSpec.groovy | 53 +++---------------- .../converter/RecordConverterSpec.groovy | 4 +- 6 files changed, 26 insertions(+), 59 deletions(-) diff --git a/build.gradle b/build.gradle index 215c37add..fb1837b2b 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "5.2-EXTENDED-SNAPSHOT" +version "5.2-SNAPSHOT" group "au.org.ala" description "Ecodata" diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index c03660ff1..cbccb18c3 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -11,7 +11,7 @@ environments { mongodb { host = "localhost" port = "27017" - databaseName = "ecodata-test-server" + databaseName = "ecodata" } } } @@ -1066,6 +1066,8 @@ biocollect.projectActivityDataURL="${biocollect.baseURL}/bioActivity/projectReco biocollect.projectArea.simplificationThreshold=10000 biocollect.projectArea.simplificationTolerance=0.0001 +fieldcapture.baseURL="https://fieldcapture.ala.org.au" + // elasticsearch cluster setting // can transport layer connection be made from apps outside JVM elasticsearch.local = true diff --git a/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy b/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy index cd9b3a426..56d12faf2 100644 --- a/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy @@ -13,6 +13,7 @@ class HarvestController { ProjectService projectService ProjectActivityService projectActivityService UserService userService + def grailsApplication /** * List of supported data resource id available for harvesting. @@ -148,9 +149,9 @@ class HarvestController { if (projectId) { Project project = Project.findByProjectId(projectId) if(project?.alaHarvest) { - // Simulate BioCollect as the hostname calling this method. This is done to get the correct URL for - // documents. - DocumentHostInterceptor.documentHostUrlPrefix.set(grailsApplication.config.getProperty("biocollect.baseURL")) + // This is done to get the correct URL for documents. + String hostname = project.isMERIT ? grailsApplication.config.getProperty("fieldcapture.baseURL") : grailsApplication.config.getProperty("biocollect.baseURL") + DocumentHostInterceptor.documentHostUrlPrefix.set(hostname) recordService.getDarwinCoreArchiveForProject(response.outputStream, project) } else response status: HttpStatus.SC_NOT_FOUND, text: [error: "project not found or ala harvest flag is switched off"] as JSON, contentType: ContentType.APPLICATION_JSON diff --git a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy index 01ba7cb52..fc3ce6d7c 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy @@ -2,7 +2,10 @@ package au.org.ala.ecodata.converter import org.apache.commons.logging.Log import org.apache.commons.logging.LogFactory - +import org.springframework.expression.Expression +import org.springframework.expression.ExpressionParser +import org.springframework.expression.spel.SpelEvaluationException +import org.springframework.expression.spel.standard.SpelExpressionParser /** * Converts an Output's data model into one or more Records. * @@ -123,14 +126,16 @@ trait RecordFieldConverter { def evaluateExpression (String expression, Map data = [:], def defaultValue, Map metadata = null, Map context = [:]) { try { context.metadata = metadata + String returnType = metadata.returnType ?: "java.lang.String" + Class returnClass = Class.forName(returnType) data.context = context - def binding = new Binding(data) - GroovyShell shell = new GroovyShell(binding) - def value = shell.evaluate(expression) + ExpressionParser expressionParser = new SpelExpressionParser() + Expression expObject = expressionParser.parseExpression(expression) + def value = expObject.getValue(data, returnClass) data.remove('context') return value } - catch (MissingPropertyException exp) { + catch (SpelEvaluationException exp) { data.remove('context') // This could happen if the expression references a field that is not in the data. // Or, what is passed is a string and not an expression. This could happen when getMeasurementType calls this method. diff --git a/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy index 8d1973b3a..7dcd7b94b 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy @@ -1,6 +1,6 @@ package au.org.ala.ecodata.converter -import au.org.ala.ecodata.* + import groovy.json.JsonSlurper import spock.lang.Specification @@ -86,7 +86,7 @@ class GenericConverterSpec extends Specification { Map metadata = [ name: "juvenile", dwcAttribute: "individualCount", - dwcExpression: "context.metadata.description + ' ' + (juvenile + adult)", + dwcExpression: "['context']['metadata']['description'] + ' ' + (['juvenile'] + ['adult'])", dwcDefault: 0, description: "Total number of individuals" ] @@ -101,49 +101,6 @@ class GenericConverterSpec extends Specification { result[0].individualCount == "Total number of individuals 5" } - def "convert should handle expressions in groovy, access information from context and add multiple darwin core attributes"() { - setup: - Map data = [ - "juvenile" : 2, - "adult": 3 - ] - - Map otherMetadata = [ - name: "adult", - dwcAttribute: "adultCount", - description: "Total number of adult individuals" - ] - Map metadata = [ - name: "juvenile", - dataType: "number", - dwcAttribute: "individualCount", - dwcExpression: "context.record.projectId = context.project.projectId; context.record.organisationId = context.organisation.organisationId; context.record.individualCount = juvenile + adult;", - dwcDefault: 0, - description: "Total number of individuals" - ] - - GenericFieldConverter converter = new GenericFieldConverter() - Map context = [ - 'project': new Project(projectId: 'project1'), - 'organisation': new Organisation(organisationId: 'org1'), - 'site': new Site(siteId: 'site1'), - 'projectActivity': new ProjectActivity(projectActivityId: 'pa1'), - 'activity':new Activity(activityId: 'activity1', siteId: 'site1'), - 'output': new Output(outputId: 'output1', activityId: 'activity1'), - outputMetadata: [dataModel: [metadata,otherMetadata]], - rootData: data - ] - - when: - List result = converter.convert(data, metadata, context) - - then: - result.size() == 1 - result[0].individualCount == 5 - result[0].projectId == "project1" - result[0].organisationId == "org1" - } - def "convert should return expression if binding not found"() { setup: Map data = [ @@ -153,7 +110,8 @@ class GenericConverterSpec extends Specification { Map metadata = [ name: "juvenile", dwcAttribute: "individualCount", - dwcExpression: "juvenile + adult", + dwcExpression: "['juvenile'] + ['adult']", + returnType: "java.lang.Integer", dwcDefault: 0, description: "Total number of individuals" ] @@ -177,7 +135,8 @@ class GenericConverterSpec extends Specification { Map metadata = [ name: "juvenile", dwcAttribute: "individualCount", - dwcExpression: "juvenile + adult", + dwcExpression: "['juvenile'] + ['adult']", + returnType: "java.lang.Integer", dwcDefault: 0, description: "Total number of individuals" ] diff --git a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy index c95f70026..cce6b5506 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy @@ -472,7 +472,7 @@ class RecordConverterSpec extends Specification { dataType: "text", name: "field2", dwcAttribute: "attribute2", - dwcExpression: "field1 == 0 ? \"absent\" : \"present\"" + dwcExpression: "['field1'] == 0 ? \"absent\" : \"present\"" ], [ dataType: "number", @@ -498,7 +498,7 @@ class RecordConverterSpec extends Specification { dataType: "number", name: "distance", dwcAttribute: "measurementValue", - dwcExpression: "context.rootData.field1 + distance", + dwcExpression: "['context']['rootData']['field1'] + distance", measurementType: "number", measurementTypeID: "http://qudt.org/vocab/quantitykind/Number", measurementAccuracy: "0.001", From eab8e01821a0db6f2cd41363d3343b1e5a740e12 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 10 Feb 2025 09:40:41 +1100 Subject: [PATCH 16/22] fixed failing test --- .../controllers/au/org/ala/ecodata/HarvestController.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy b/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy index 56d12faf2..29647001b 100644 --- a/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy @@ -13,7 +13,6 @@ class HarvestController { ProjectService projectService ProjectActivityService projectActivityService UserService userService - def grailsApplication /** * List of supported data resource id available for harvesting. From 985bbf41187598211137b50184709782cab51bca Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 10 Feb 2025 12:14:11 +1100 Subject: [PATCH 17/22] fixed failing test --- .../ala/ecodata/HarvestControllerSpec.groovy | 2 +- .../converter/RecordFieldConverter.groovy | 12 ++-------- .../converter/GenericConverterSpec.groovy | 22 ++++++++++++++----- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy b/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy index 90571282a..d0038bf8c 100644 --- a/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy +++ b/src/integration-test/groovy/au/org/ala/ecodata/HarvestControllerSpec.groovy @@ -237,7 +237,7 @@ class HarvestControllerSpec extends Specification { CSVReader readerCSV = new CSVReader(new StringReader(content)) List lines = readerCSV.readAll() assert lines[0] == ["eventID","occurrenceID","measurementValue","measurementAccuracy","measurementUnit","measurementUnitID","measurementType","measurementTypeID"] - assert lines[1] == ["activity1","outputSpecies1","1.0","0.001","m","http://qudt.org/vocab/unit/M","number","http://qudt.org/vocab/quantitykind/Number"] + assert lines[1] == ["activity1","outputSpecies1","1.0","0.001","m","http://qudt.org/vocab/unit/M","distance from source","http://qudt.org/vocab/quantitykind/Number"] assert lines.size() ==2 break } diff --git a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy index fc3ce6d7c..f02a08595 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/RecordFieldConverter.groovy @@ -4,7 +4,6 @@ import org.apache.commons.logging.Log import org.apache.commons.logging.LogFactory import org.springframework.expression.Expression import org.springframework.expression.ExpressionParser -import org.springframework.expression.spel.SpelEvaluationException import org.springframework.expression.spel.standard.SpelExpressionParser /** * Converts an Output's data model into one or more Records. @@ -135,17 +134,10 @@ trait RecordFieldConverter { data.remove('context') return value } - catch (SpelEvaluationException exp) { - data.remove('context') - // This could happen if the expression references a field that is not in the data. - // Or, what is passed is a string and not an expression. This could happen when getMeasurementType calls this method. - // In such cases, return the value of measurementType i.e. expression itself. - return expression - } catch (Exception ex) { - data.remove('context') log.error(ex.message, ex) - return defaultValue + data.remove('context') + return defaultValue == null ? expression : defaultValue } } diff --git a/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy index 7dcd7b94b..d39be5291 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/GenericConverterSpec.groovy @@ -101,7 +101,7 @@ class GenericConverterSpec extends Specification { result[0].individualCount == "Total number of individuals 5" } - def "convert should return expression if binding not found"() { + def "convert should return expression if binding not found and no default value"() { setup: Map data = [ "juvenile" : 2 @@ -112,7 +112,6 @@ class GenericConverterSpec extends Specification { dwcAttribute: "individualCount", dwcExpression: "['juvenile'] + ['adult']", returnType: "java.lang.Integer", - dwcDefault: 0, description: "Total number of individuals" ] GenericFieldConverter converter = new GenericFieldConverter() @@ -122,10 +121,21 @@ class GenericConverterSpec extends Specification { then: result.size() == 1 - result[0].individualCount == "juvenile + adult" + result[0].individualCount == "['juvenile'] + ['adult']" + + when: + data = [ + "juvenile" : 2 + ] + metadata.dwcDefault = 1 + result = converter.convert(data, metadata) + + then: + result.size() == 1 + result[0].individualCount == 1 } - def "convert should return default value for all other exceptions"() { + def "expression should check for null value"() { setup: Map data = [ "juvenile" : 2, @@ -135,7 +145,7 @@ class GenericConverterSpec extends Specification { Map metadata = [ name: "juvenile", dwcAttribute: "individualCount", - dwcExpression: "['juvenile'] + ['adult']", + dwcExpression: "(['juvenile'] != null ? ['juvenile'] : 0) + (['adult'] != null ? ['adult'] : 0)", returnType: "java.lang.Integer", dwcDefault: 0, description: "Total number of individuals" @@ -147,6 +157,6 @@ class GenericConverterSpec extends Specification { then: result.size() == 1 - result[0].individualCount == 0 + result[0].individualCount == 2 } } From fa35fa623c5cc28e39812b2103a4db4d6ceb6c18 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 10 Feb 2025 12:27:08 +1100 Subject: [PATCH 18/22] fixed failing test --- .../au/org/ala/ecodata/converter/RecordConverterSpec.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy index cce6b5506..0d4c5f994 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy @@ -542,19 +542,19 @@ class RecordConverterSpec extends Specification { fieldsets[0].measurementsorfacts.size() == 3 fieldsets[0].measurementsorfacts[0].eventID == "act1" fieldsets[0].measurementsorfacts[0].measurementValue == 10.056 - fieldsets[0].measurementsorfacts[0].measurementType == "number" + fieldsets[0].measurementsorfacts[0].measurementType == "distance from source" fieldsets[0].measurementsorfacts[0].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number" fieldsets[0].measurementsorfacts[0].measurementAccuracy == "0.001" fieldsets[0].measurementsorfacts[0].measurementUnit == "m" fieldsets[0].measurementsorfacts[0].measurementUnitID == "http://qudt.org/vocab/unit/M" fieldsets[0].measurementsorfacts[1].measurementValue == 21.056 - fieldsets[0].measurementsorfacts[1].measurementType == "number" + fieldsets[0].measurementsorfacts[1].measurementType == "distance from source" fieldsets[0].measurementsorfacts[1].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number" fieldsets[0].measurementsorfacts[1].measurementAccuracy == "0.001" fieldsets[0].measurementsorfacts[1].measurementUnit == "m" fieldsets[0].measurementsorfacts[1].measurementUnitID == "http://qudt.org/vocab/unit/M" fieldsets[0].measurementsorfacts[2].measurementValue == 23.056 - fieldsets[0].measurementsorfacts[2].measurementType == "number" + fieldsets[0].measurementsorfacts[2].measurementType == "distance from source" fieldsets[0].measurementsorfacts[2].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number" fieldsets[0].measurementsorfacts[2].measurementAccuracy == "0.001" fieldsets[0].measurementsorfacts[2].measurementUnit == "m" From 25e388f6a2c1724693b4d9dca8a60bfc0894bef3 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 10 Feb 2025 14:32:25 +1100 Subject: [PATCH 19/22] trying again --- .../au/org/ala/ecodata/converter/RecordConverterSpec.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy index 0d4c5f994..3ce634dec 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy @@ -453,7 +453,7 @@ class RecordConverterSpec extends Specification { fieldsets[0].attribute1 == "fieldValue1" fieldsets[0].measurementsorfacts.size() == 1 fieldsets[0].measurementsorfacts[0].measurementValue == 10.056 - fieldsets[0].measurementsorfacts[0].measurementType == "number" + fieldsets[0].measurementsorfacts[0].measurementType == "distance from source" fieldsets[0].measurementsorfacts[0].measurementTypeID == "http://qudt.org/vocab/quantitykind/Number" fieldsets[0].measurementsorfacts[0].measurementAccuracy == "0.001" fieldsets[0].measurementsorfacts[0].measurementUnit == "m" From 09e1101dc2d7582bc663ef835042c06e5250fb71 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 10 Feb 2025 15:21:45 +1100 Subject: [PATCH 20/22] trying again --- .../au/org/ala/ecodata/converter/RecordConverterSpec.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy index 3ce634dec..10bd8a6ff 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy @@ -498,7 +498,7 @@ class RecordConverterSpec extends Specification { dataType: "number", name: "distance", dwcAttribute: "measurementValue", - dwcExpression: "['context']['rootData']['field1'] + distance", + dwcExpression: "['context']['rootData']['field1'] + ['distance']", measurementType: "number", measurementTypeID: "http://qudt.org/vocab/quantitykind/Number", measurementAccuracy: "0.001", From 65dd841b4e0a7f9d408ed715e60b7b6ca5639230 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 10 Feb 2025 15:36:16 +1100 Subject: [PATCH 21/22] another try --- .../au/org/ala/ecodata/converter/RecordConverterSpec.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy index 10bd8a6ff..9cc2984b4 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy @@ -482,6 +482,7 @@ class RecordConverterSpec extends Specification { measurementTypeID: "http://qudt.org/vocab/quantitykind/Number", measurementAccuracy: "0.001", measurementUnit: "m", + returnType: "java.lang.Integer", measurementUnitID: "http://qudt.org/vocab/unit/M", description: "distance from source" ], @@ -503,6 +504,7 @@ class RecordConverterSpec extends Specification { measurementTypeID: "http://qudt.org/vocab/quantitykind/Number", measurementAccuracy: "0.001", measurementUnit: "m", + returnType: "java.lang.Integer", measurementUnitID: "http://qudt.org/vocab/unit/M", description: "distance from source" ] From 25e2e79a36cffe14ced4e68c2dae94f5da668f11 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 10 Feb 2025 15:50:12 +1100 Subject: [PATCH 22/22] another try --- .../au/org/ala/ecodata/converter/RecordConverterSpec.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy index 9cc2984b4..69bcae0d4 100644 --- a/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/converter/RecordConverterSpec.groovy @@ -482,7 +482,7 @@ class RecordConverterSpec extends Specification { measurementTypeID: "http://qudt.org/vocab/quantitykind/Number", measurementAccuracy: "0.001", measurementUnit: "m", - returnType: "java.lang.Integer", + returnType: "java.lang.Double", measurementUnitID: "http://qudt.org/vocab/unit/M", description: "distance from source" ], @@ -504,7 +504,7 @@ class RecordConverterSpec extends Specification { measurementTypeID: "http://qudt.org/vocab/quantitykind/Number", measurementAccuracy: "0.001", measurementUnit: "m", - returnType: "java.lang.Integer", + returnType: "java.lang.Double", measurementUnitID: "http://qudt.org/vocab/unit/M", description: "distance from source" ]