Skip to content

Commit

Permalink
Merge pull request #4292 from nationalarchives/TDRD-87-error-download…
Browse files Browse the repository at this point in the history 
…-tweaks

[TDRD 87] Row validation error report download compatibility tweaks
  • Loading branch information
@annielh
annielh authored Nov 19, 2024
2 parents b99e33b + d91edb3 commit 47b8120
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 29 deletions.
50 changes: 27 additions & 23 deletions app/controllers/DraftMetadataChecksResultsController.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@ import auth.TokenSecurity
import configuration.{ApplicationConfig, KeycloakConfiguration}
import controllers.util.ExcelUtils
import controllers.util.MetadataProperty.filePath
import graphql.codegen.GetConsignmentStatus.getConsignmentStatus.GetConsignment
import org.pac4j.play.scala.SecurityComponents
import play.api.i18n.{I18nSupport, Lang, MessagesApi}
import play.api.mvc.{Action, AnyContent, Request}
import services.FileError.SCHEMA_VALIDATION
import services.FileError.{FileError, ROW_VALIDATION, SCHEMA_VALIDATION, UTF_8}
import services.Statuses._
import services._
import viewsapi.Caching.preventCaching
Expand Down Expand Up @@ -129,28 +128,33 @@ class DraftMetadataChecksResultsController @Inject() (
}
}

def downloadErrorReport(consignmentId: UUID): Action[AnyContent] = standardUserAndTypeAction(consignmentId) { implicit request: Request[AnyContent] =>
for {
reference <- consignmentService.getConsignmentRef(consignmentId, request.token.bearerAccessToken)
errorReport <- draftMetadataService.getErrorReport(consignmentId)
} yield {
val errorList = errorReport.fileError match {
case SCHEMA_VALIDATION =>
errorReport.validationErrors.flatMap { validationErrors =>
val data = validationErrors.data.map(metadata => metadata.name -> metadata.value).toMap
validationErrors.errors.map(error => List(data(filePath), error.property, data(error.property), error.message))
}
case _ => Nil
def downloadErrorReport(consignmentId: UUID, priorityErrorType: FileError = ROW_VALIDATION): Action[AnyContent] = standardUserAndTypeAction(consignmentId) {
implicit request: Request[AnyContent] =>
for {
reference <- consignmentService.getConsignmentRef(consignmentId, request.token.bearerAccessToken)
errorReport <- draftMetadataService.getErrorReport(consignmentId)
} yield {
val errorPriorityOrdering: Ordering[Error] =
Ordering.by[Error, (Boolean, String)](e => (!e.validationProcess.equals(s"$priorityErrorType"), e.validationProcess))
val errorList = errorReport.fileError match {
case SCHEMA_VALIDATION =>
errorReport.validationErrors.flatMap { validationErrors =>
val data = validationErrors.data.map(metadata => metadata.name -> metadata.value).toMap
validationErrors.errors.toList
.sorted(errorPriorityOrdering)
.map(error => List(validationErrors.assetId, error.property, data.getOrElse(error.property, ""), error.message))
}
case _ => Nil
}
val header: List[String] = List(filePath, "Column", "Value", "Error Message")
val excelFile = ExcelUtils.writeExcel(s"Error report for $reference", header :: errorList)
val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH-mm-ss")
val currentDateTime = dateTimeFormatter.format(LocalDateTime.now())
val excelContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
Ok(excelFile)
.as(excelContentType)
.withHeaders("Content-Disposition" -> s"attachment; filename=ErrorReport-${reference}-$currentDateTime.xlsx")
}
val header: List[String] = List(filePath, "Field", "Value", "Error Message")
val excelFile = ExcelUtils.writeExcel(s"Error report for $reference", header :: errorList)
val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH-mm-ss")
val currentDateTime = dateTimeFormatter.format(LocalDateTime.now())
val excelContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
Ok(excelFile)
.as(excelContentType)
.withHeaders("Content-Disposition" -> s"attachment; filename=ErrorReport-${reference}-$currentDateTime.xlsx")
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion app/services/DraftMetadataService.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import scala.concurrent.{ExecutionContext, Future}

object FileError extends Enumeration {
type FileError = Value
val UTF_8, INVALID_CSV, DUPLICATE_HEADER, SCHEMA_REQUIRED, SCHEMA_VALIDATION, VIRUS, UNKNOWN, NONE = Value
val UTF_8, INVALID_CSV, ROW_VALIDATION, DUPLICATE_HEADER, SCHEMA_REQUIRED, SCHEMA_VALIDATION, VIRUS, UNKNOWN, NONE = Value
}

case class Error(validationProcess: String, property: String, errorKey: String, message: String)
Expand Down
45 changes: 40 additions & 5 deletions test/controllers/DraftMetadataChecksResultsControllerSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,9 @@ class DraftMetadataChecksResultsControllerSpec extends FrontEndTestHelper {

setConsignmentTypeResponse(wiremockServer, "standard")
setConsignmentReferenceResponse(wiremockServer)
val error = Error("BASE_SCHEMA", "FOI exemption code", "enum", "BASE_SCHEMA.foi_exmption_code.enum")
val error = Error("BASE_SCHEMA", "FOI exemption code", "enum", "BASE_SCHEMA.foi_exemption_code.enum")
val metadata = List(Metadata("FOI exemption code", "abcd"), Metadata("Filepath", "/aa/bb/faq"))
val errorFileData = ErrorFileData(consignmentId, date = "2024-12-12", FileError.SCHEMA_VALIDATION, List(ValidationErrors("assetId", Set(error), metadata)))
val errorFileData = ErrorFileData(consignmentId, date = "2024-12-12", FileError.SCHEMA_VALIDATION, List(ValidationErrors("/aa/bb/faq", Set(error), metadata)))
val response = instantiateController(errorFileData = Some(errorFileData))
.downloadErrorReport(consignmentId)(FakeRequest(GET, s"/consignment/$consignmentId/draft-metadata/download-report"))

Expand All @@ -261,14 +261,14 @@ class DraftMetadataChecksResultsControllerSpec extends FrontEndTestHelper {
rows.length must equal(2)

rows.head.getCell(0).asString must equal("Filepath")
rows.head.getCell(1).asString must equal("Field")
rows.head.getCell(1).asString must equal("Column")
rows.head.getCell(2).asString must equal("Value")
rows.head.getCell(3).asString must equal("Error Message")

rows(1).getCell(0).asString must equal("/aa/bb/faq")
rows(1).getCell(1).asString must equal("FOI exemption code")
rows(1).getCell(2).asString must equal("abcd")
rows(1).getCell(3).asString must equal("BASE_SCHEMA.foi_exmption_code.enum")
rows(1).getCell(3).asString must equal("BASE_SCHEMA.foi_exemption_code.enum")
}

"download the excel file without error data when the error type is not SCHEMA_VALIDATION" in {
Expand All @@ -291,10 +291,45 @@ class DraftMetadataChecksResultsControllerSpec extends FrontEndTestHelper {
rows.length must equal(1)

rows.head.getCell(0).asString must equal("Filepath")
rows.head.getCell(1).asString must equal("Field")
rows.head.getCell(1).asString must equal("Column")
rows.head.getCell(2).asString must equal("Value")
rows.head.getCell(3).asString must equal("Error Message")
}

"download the excel file with row validation errors at the top where present" in {
setConsignmentTypeResponse(wiremockServer, "standard")
setConsignmentReferenceResponse(wiremockServer)
val schemaError = Error("BASE_SCHEMA", "FOI exemption code", "enum", "BASE_SCHEMA.foi_exmption_code.enum")
val rowValidationError = Error("ROW_VALIDATION", "", "unknown", "This file was listed in your metadata file but does not match to one of your uploaded files")
val metadata = List(Metadata("FOI exemption code", "abcd"), Metadata("Filepath", "/aa/bb/faq"))
val errorFileData =
ErrorFileData(consignmentId, date = "2024-12-12", FileError.SCHEMA_VALIDATION, List(ValidationErrors("/aa/bb/faq", Set(schemaError, rowValidationError), metadata)))
val response = instantiateController(errorFileData = Some(errorFileData))
.downloadErrorReport(consignmentId)(FakeRequest(GET, s"/consignment/$consignmentId/draft-metadata/download-report"))

val responseByteArray: ByteString = contentAsBytes(response)
val bufferedSource = new ByteArrayInputStream(responseByteArray.toArray)
val wb: ReadableWorkbook = new ReadableWorkbook(bufferedSource)
val ws: Sheet = wb.getFirstSheet
val rows: List[Row] = ws.read.asScala.toList

rows.length must equal(3)

rows.head.getCell(0).asString must equal("Filepath")
rows.head.getCell(1).asString must equal("Column")
rows.head.getCell(2).asString must equal("Value")
rows.head.getCell(3).asString must equal("Error Message")

rows(1).getCell(0).asString must equal("/aa/bb/faq")
rows(1).getCell(1).asString must equal("")
rows(1).getCell(2).asString must equal("")
rows(1).getCell(3).asString must equal("This file was listed in your metadata file but does not match to one of your uploaded files")

rows(2).getCell(0).asString must equal("/aa/bb/faq")
rows(2).getCell(1).asString must equal("FOI exemption code")
rows(2).getCell(2).asString must equal("abcd")
rows(2).getCell(3).asString must equal("BASE_SCHEMA.foi_exmption_code.enum")
}
}

private def instantiateController(
Expand Down

0 comments on commit 47b8120

Please sign in to comment.