From ede733559896f628a018a3f563dc9f1bb2d28015 Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Thu, 25 Jan 2018 16:55:44 +0100 Subject: [PATCH 01/10] return also user_id at login --- R/api.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/api.R b/R/api.R index 6016350..593d667 100644 --- a/R/api.R +++ b/R/api.R @@ -309,7 +309,7 @@ openeo.server$api.version <- "0.0.1" token = bin2hex(append(encryption, attr(encryption,"nonce"))) - list(token=token) + list(user_id = user$user_id, token=token) } else { stop("Wrong password") } From f0ac75cdd933051cbd60aac094c4713035b228d5 Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Fri, 26 Jan 2018 16:56:58 +0100 Subject: [PATCH 02/10] initial sqlite refactoring - started sqlite integration - migration script for file based storage into sqlite db - remove User$store since we don't save a user file anymore - adapted .authorization filter --- DESCRIPTION | 4 +- NAMESPACE | 2 + R/Server-class.R | 102 +++++++++++++++++++++++++++++++++++++--- R/User-class.R | 17 ++----- R/api.R | 14 +++++- man/migrateFilesToDB.Rd | 16 +++++++ 6 files changed, 132 insertions(+), 23 deletions(-) create mode 100644 man/migrateFilesToDB.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 5922ec2..1708d02 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -10,7 +10,9 @@ Depends: gdalUtils (>= 2.0.1.7), raster (>= 2.6-7), sodium (>= 1.1), - plumber (>= 0.4.4) + plumber (>= 0.4.4), + DBI (>= 0.7), + RSQLite (>= 2.0) Encoding: UTF-8 LazyData: true RoxygenNote: 6.0.1 diff --git a/NAMESPACE b/NAMESPACE index 899df6a..0ba44c9 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -15,7 +15,9 @@ export(User) export(View) export(isProcess) export(isProduct) +export(migrateFilesToDB) export(openeo.server) +import(DBI) import(plumber) import(raster) importFrom(R6,R6Class) diff --git a/R/Server-class.R b/R/Server-class.R index ad1fa8b..d2c4403 100644 --- a/R/Server-class.R +++ b/R/Server-class.R @@ -19,6 +19,7 @@ #' @importFrom jsonlite fromJSON #' @importFrom jsonlite toJSON #' @importFrom sodium sha256 +#' @import DBI #' @export OpenEOServer <- R6Class( "OpenEOServer", @@ -28,6 +29,8 @@ OpenEOServer <- R6Class( data.path = NULL, workspaces.path = NULL, + sqlite.path = NULL, + api.port = NULL, jobs = NULL, @@ -35,6 +38,8 @@ OpenEOServer <- R6Class( data = NULL, users = NULL, + database = NULL, + initialize = function() { self$jobs = list() self$processes = list() @@ -47,6 +52,9 @@ OpenEOServer <- R6Class( self$api.port = port } + private$initEnvironmentDefault() + + # load descriptions, meta data and file links for provided data sets # private$loadData() @@ -61,6 +69,7 @@ OpenEOServer <- R6Class( root = createAPI() root$registerHook("exit", function(){ + dbDisconnect(self$database) print("Bye bye!") }) @@ -153,7 +162,11 @@ OpenEOServer <- R6Class( user$password = password user$workspace = paste(self$workspaces.path,id,sep="/") - user$store() + user_info = data.frame(user_id = id, user_name=user_name, password = password, login_secret = "") + if (dbGetQuery(self$database, "select count(*) from user where user_id=:id", + param=list(id = id)) == 0) { + dbWriteTable(db,"user",as.data.frame(user_info),append=TRUE) + } return(user) @@ -175,12 +188,29 @@ OpenEOServer <- R6Class( loadDemo = function() { private$initEnvironmentDefault() + self$initializeDatabase() private$loadDemoData() private$loadDemoProcesses() - } - + }, + initializeDatabase = function() { + self$database = dbConnect(RSQLite::SQLite(),self$sqlite.path) + + if (!dbExistsTable(self$database,"user")) { + dbExecute(self$database, "create table user (user_id integer, + user_name text, + password text, + login_secret text)") + } + if (!dbExistsTable(self$database,"job")) { + dbExecute(self$database, "create table job (job_id text, + user_id integer, + status text, + submitted text, + process_graph text)") + } + } ), private = list( loadDemoData = function() { @@ -201,7 +231,7 @@ OpenEOServer <- R6Class( user = User$new(id) user$user_name = parsedJson[["user_name"]] user$password = parsedJson[["password"]] - user$workspace = workspace.path + # user$workspace = workspace.path self$register(user) @@ -211,8 +241,9 @@ OpenEOServer <- R6Class( loadUsers = function() { self$users = list() + user_folders = list.dirs(self$workspaces.path,recursive = FALSE,full.names = FALSE) - for (user_id in list.files(self$workspaces.path)) { + for (user_id in user_folders) { user = private$loadUser(user_id) private$loadExistingJobs(user) } @@ -313,13 +344,72 @@ OpenEOServer <- R6Class( if (is.null(self$secret.key)) { self$secret.key <- sha256(charToRaw("openEO-R")) } + if (is.null(self$sqlite.path)) { + self$sqlite.path <- paste(self$workspaces.path,"openeo.sqlite",sep="/") + } if (is.null(self$api.port)) { self$api.port <- 8000 } } + + ) ) #' @export -openeo.server = OpenEOServer$new() \ No newline at end of file +openeo.server = OpenEOServer$new() + +#' Migration tool +#' +#' This functions migrates the old file based information storage into the sqlite solution. +#' +#' @param db sqlite database connection +#' @param workspace the user workspace, where to find the user folders +#' +#' @export +migrateFilesToDB = function(db, workspace) { + df = data.frame(user_id=list.files(workspace),workspace=list.files(workspace,full.names = TRUE)) + df$jobs_workspace = paste(df$workspace,"jobs",sep="/") + df$files_workspace = paste(df$workspace,"files",sep="/") + df$user_file = paste(df$workspace, "user.json",sep="/") + + for (row_no in 1:nrow(df)) { + row = df[row_no,] + # read user.json + if (file.exists(row$user_file)) { + user_info = fromJSON(row$user_file) + user_info$jobs = NULL + user_info$login_secret = "" + if (dbGetQuery(db,"select count(*) from user where user_id=:id",param=list(id=user_info$user_id))==0) { + dbWriteTable(db,"user",as.data.frame(user_info),append=TRUE) + } + } + + # make job list + if (dir.exists(row$jobs_workspace)) { + jobs = data.frame(job_id = paste(list.files(row$jobs_workspace)), + job_path = paste(list.files(row$jobs_workspace, full.names=TRUE),"process_graph.json",sep="/"), + stringsAsFactors=FALSE) + for(job_row_no in 1:nrow(jobs)) { + job_row = jobs[job_row_no,] + + if (file.exists(job_row$job_path)) { + + if (dbExistsTable(db,"job") && + dbGetQuery(db, "select count(*) from job where job_id = :id",param=list(id=job_row$job_id)) == 0) { + job = fromJSON(paste(job_row$job_path)) + job$job_id = job_row$job_id + process_graph = job$process_graph + job$submitted = job$submitted + job$process_graph = as.character(toJSON(process_graph,auto_unbox = TRUE)) + + + columns = dbListFields(db,"job") + dbWriteTable(db,"job",as.data.frame(job)[,columns],append=TRUE) + } + } + } + } + } +} \ No newline at end of file diff --git a/R/User-class.R b/R/User-class.R index fe2930f..0eadd47 100644 --- a/R/User-class.R +++ b/R/User-class.R @@ -5,7 +5,6 @@ User <- R6Class( "User", public = list( user_id = NULL, - workspace = NULL, user_name = NULL, password = NULL, token = NULL, @@ -35,22 +34,12 @@ User <- R6Class( size = row["size"] )) })) - }, - - store = function() { - if (is.null(self$workspace)) { - stop("Cannot store user, because there was no workspace assigned.") - } - - dir.create(self$workspace, showWarnings = FALSE) - dir.create(paste(self$workspace,private$files.folder,sep="/"), showWarnings = FALSE) - dir.create(paste(self$workspace,private$jobs.folder,sep="/"), showWarnings = FALSE) - - json = toJSON(self$toList(),auto_unbox = TRUE,pretty=TRUE) - write(x=json,file=paste(self$workspace,"user.json",sep="/")) } ), active = list( + workspace = function() { + return(paste(openeo.server$workspaces.path,self$user_id,sep="/")) + }, files = function() { workspace = paste(self$workspace,private$files.folder,sep="/") diff --git a/R/api.R b/R/api.R index 593d667..05d2604 100644 --- a/R/api.R +++ b/R/api.R @@ -370,8 +370,18 @@ openeo.server$api.version <- "0.0.1" nonce = hextoken[((length(hextoken)-nonce.length)+1):length(hextoken)] user_id = rawToChar(data_decrypt(msg,openeo.server$secret.key,nonce)) - if (user_id %in% names(openeo.server$users)) { - req$user = openeo.server$users[[user_id]] + if (dbGetQuery(openeo.server$database, "select count(*) from user where user_id = :id" + ,param = list(id=user_id)) == 1 + ) { + + user_info = dbGetQuery(openeo.server$database, "select * from user where user_id = :id" + ,param = list(id=user_id)) + + user = User$new(user_id) + user$user_name = user_info$user_name + + req$user = user + forward() } else { stop("Incorrect token") diff --git a/man/migrateFilesToDB.Rd b/man/migrateFilesToDB.Rd new file mode 100644 index 0000000..0ca9cc2 --- /dev/null +++ b/man/migrateFilesToDB.Rd @@ -0,0 +1,16 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/Server-class.R +\name{migrateFilesToDB} +\alias{migrateFilesToDB} +\title{Migration tool} +\usage{ +migrateFilesToDB(db, workspace) +} +\arguments{ +\item{db}{sqlite database connection} + +\item{workspace}{the user workspace, where to find the user folders} +} +\description{ +This functions migrates the old file based information storage into the sqlite solution. +} From bb218e0c311ebbf2f4451617ed39211897f3e99e Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Sun, 28 Jan 2018 14:57:08 +0100 Subject: [PATCH 03/10] modified user creation to database and file storage - adapted file upload and login --- R/Server-class.R | 56 ++++++++++++++++++++++++----------------------- R/User-class.R | 5 +++-- R/api.R | 57 +++++++++++++++++++++++++++--------------------- 3 files changed, 64 insertions(+), 54 deletions(-) diff --git a/R/Server-class.R b/R/Server-class.R index d2c4403..0c962c2 100644 --- a/R/Server-class.R +++ b/R/Server-class.R @@ -52,7 +52,7 @@ OpenEOServer <- R6Class( self$api.port = port } - private$initEnvironmentDefault() + self$initEnvironmentDefault() # load descriptions, meta data and file links for provided data sets @@ -61,7 +61,7 @@ OpenEOServer <- R6Class( # register the processes provided by the server provider # private$loadProcesses() - private$loadUsers() + # private$loadUsers() # if there have been previous job postings load those jobs into the system # private$loadExistingJobs() @@ -155,19 +155,23 @@ OpenEOServer <- R6Class( }, createUser = function(user_name, password) { + files.folder = "files" id = private$newUserId() user = User$new(user_id = id) user$user_name = user_name user$password = password - user$workspace = paste(self$workspaces.path,id,sep="/") user_info = data.frame(user_id = id, user_name=user_name, password = password, login_secret = "") if (dbGetQuery(self$database, "select count(*) from user where user_id=:id", param=list(id = id)) == 0) { - dbWriteTable(db,"user",as.data.frame(user_info),append=TRUE) + dbWriteTable(self$database,"user",as.data.frame(user_info),append=TRUE) } + dir.create(user$workspace, showWarnings = FALSE) + dir.create(paste(user$workspace,files.folder,sep="/"), showWarnings = FALSE) + + return(user) }, @@ -187,7 +191,7 @@ OpenEOServer <- R6Class( }, loadDemo = function() { - private$initEnvironmentDefault() + self$initEnvironmentDefault() self$initializeDatabase() private$loadDemoData() @@ -210,6 +214,26 @@ OpenEOServer <- R6Class( submitted text, process_graph text)") } + }, + + initEnvironmentDefault = function() { + + if (is.null(self$data.path)) { + self$data.path <- paste(system.file(package="openEO.R.Backend"),"extdata",sep="/") + } + if (is.null(self$workspaces.path)) { + self$workspaces.path <- getwd() + } + if (is.null(self$secret.key)) { + self$secret.key <- sha256(charToRaw("openEO-R")) + } + if (is.null(self$sqlite.path)) { + self$sqlite.path <- paste(self$workspaces.path,"openeo.sqlite",sep="/") + } + + if (is.null(self$api.port)) { + self$api.port <- 8000 + } } ), private = list( @@ -331,29 +355,7 @@ OpenEOServer <- R6Class( return(floor(id)) } - }, - - initEnvironmentDefault = function() { - - if (is.null(self$data.path)) { - self$data.path <- paste(system.file(package="openEO.R.Backend"),"extdata",sep="/") - } - if (is.null(self$workspaces.path)) { - self$workspaces.path <- getwd() - } - if (is.null(self$secret.key)) { - self$secret.key <- sha256(charToRaw("openEO-R")) - } - if (is.null(self$sqlite.path)) { - self$sqlite.path <- paste(self$workspaces.path,"openeo.sqlite",sep="/") - } - - if (is.null(self$api.port)) { - self$api.port <- 8000 - } } - - ) ) diff --git a/R/User-class.R b/R/User-class.R index 0eadd47..de969cd 100644 --- a/R/User-class.R +++ b/R/User-class.R @@ -51,8 +51,9 @@ User <- R6Class( }, jobs = function() { - user.jobs = paste(self$workspace,private$jobs.folder,sep="/") - return(list.files(user.jobs)) + result = dbGetQuery(openeo.server$database, "select job_id from job where user_id = :id",param(id = self$user_id)) + + return(as.list(result)[[1]]) } ), private = list( diff --git a/R/api.R b/R/api.R index 05d2604..5ba610f 100644 --- a/R/api.R +++ b/R/api.R @@ -219,30 +219,30 @@ openeo.server$api.version <- "0.0.1" # @put /api/users//files/ # @serializer unboxedJSON .uploadFile = function(req,res,userid,path) { - if (! userid %in% names(openeo.server$users)) { - error(res,404,paste("User id with id \"",userid, "\" was not found", sep="")) - } else { - user = openeo.server$users[[userid]] - path = URLdecode(path) + if (paste(userid) == paste(req$user$user_id)) { - storedFilePath = paste(user$workspace,"files",path,sep="/") + path = URLdecode(path) - if (file.exists(storedFilePath)) { - file.remove(storedFilePath) - } + storedFilePath = paste(req$user$workspace,"files",path,sep="/") - dir.split = unlist(strsplit(storedFilePath, "/(?=[^/]+$)", perl=TRUE)) - - req$rook.input$initialize(req$rook.input$.conn,req$rook.input$.length) - - dir.create(dir.split[1],recursive = TRUE,showWarnings = FALSE) - file.create(storedFilePath,showWarnings = FALSE) - - outputFile = file(storedFilePath,"wb") - writeBin(req$rook.input$read(req$rook.input$.length), con=outputFile, useBytes = TRUE) - close(outputFile,type="wb") - ok(res) + if (file.exists(storedFilePath)) { + file.remove(storedFilePath) } + + dir.split = unlist(strsplit(storedFilePath, "/(?=[^/]+$)", perl=TRUE)) + + req$rook.input$initialize(req$rook.input$.conn,req$rook.input$.length) + + dir.create(dir.split[1],recursive = TRUE,showWarnings = FALSE) + file.create(storedFilePath,showWarnings = FALSE) + + outputFile = file(storedFilePath,"wb") + writeBin(req$rook.input$read(req$rook.input$.length), con=outputFile, useBytes = TRUE) + close(outputFile,type="wb") + ok(res) + } else { + error(res,401,"Not authorized to upload data into other users workspaces") + } } #* @delete /api/users//files/ @@ -254,7 +254,7 @@ openeo.server$api.version <- "0.0.1" storedFilePath = paste(user$workspace,"files",path,sep="/") - files = openeo.server$users[[paste(userid)]]$files + files = req$user$files selection = files[files[,"link"]==path,] if (nrow(selection) == 0) { error(res, 404,paste("User has no file under path:",path)) @@ -299,12 +299,19 @@ openeo.server$api.version <- "0.0.1" encoded=substr(auth,7,nchar(auth)) decoded = rawToChar(base64_dec(encoded)) - user_pwd = unlist(strsplit(decoded,":")) - + user_name = unlist(strsplit(decoded,":"))[1] + user_pwd = unlist(strsplit(decoded,":"))[2] + browser() tryCatch( { - user = openeo.server$getUserByName(user_pwd[1]) - if (user$password == user_pwd[2]) { + result = dbGetQuery(openeo.server$database, "select * from user where user_name = :name limit 1",param=list(name=user_name)) + if (nrow(result) == 0) { + stop("Invalid user") + } + + user = as.list(result) + + if (user$password == user_pwd) { encryption = data_encrypt(charToRaw(paste(user$user_id)),openeo.server$secret.key) token = bin2hex(append(encryption, attr(encryption,"nonce"))) From 9f0cae697cd841e8cd4335bcb2df4239e607e6e7 Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Mon, 5 Feb 2018 18:07:46 +0100 Subject: [PATCH 04/10] moved file based job storage to sqlite; fixed redundant argmuent name issue - modified endpoints for interactions with job createNewJob, delete, user.jobs - storing process graphs as hex-binary files - db queries for jobExists, user.jobs, Job$store - removed server lists for user and jobs, which are now dynamically loaded via loadJob (as executable) --- R/Collection-class.R | 2 +- R/Job-class.R | 97 ++++++++++++++++++---- R/Process-class.R | 3 +- R/Server-class.R | 190 ++++++++++++++----------------------------- R/User-class.R | 2 +- R/api.R | 83 ++++++++----------- R/processes.R | 1 - 7 files changed, 184 insertions(+), 194 deletions(-) diff --git a/R/Collection-class.R b/R/Collection-class.R index 538216e..c207ea7 100644 --- a/R/Collection-class.R +++ b/R/Collection-class.R @@ -79,7 +79,7 @@ Collection <- R6Class( .collection.filterbytime = function (granules,from,to) { minpos = -1 maxpos = -1 - + browser() if (from > to) { old <- to to <- from diff --git a/R/Job-class.R b/R/Job-class.R index b259f16..071828a 100644 --- a/R/Job-class.R +++ b/R/Job-class.R @@ -30,38 +30,97 @@ Job <- R6Class( user_id=NULL, consumed_credits=NULL, - filePath = NULL, - - initialize = function(job_id=NULL,filePath=NULL,process_graph=NULL,user_id = NULL) { + initialize = function(job_id=NULL,process_graph=NULL,user_id = NULL) { if (is.null(job_id)||missing(job_id)) { stop("Cannot create new Job. There is no job_id specified") } else { self$job_id = job_id } - - if (is.null(filePath) || missing(filePath)) { - stop("Cannot initialize Job without the job workspace") - } else { - self$filePath = filePath - } - if (!is.null(user_id)) { self$user_id = user_id } + self$consumed_credits = 0 self$process_graph = process_graph + return(self) }, - store = function(json=NA) { - dir.create(self$filePath) - write(x=json,file=paste(self$filePath,"/process_graph.json",sep="")) + store = function(con) { + exists = dbGetQuery(con,"select count(*) from job where job_id = :id",param=list(id=self$job_id)) == 1 + + if (!exists) { + insertIntoQuery = "insert into job (job_id, + user_id, + status, + process_graph, + submitted, + last_update, + consumed_credits) values ( + :job_id, :user_id, :status, :process_graph, :submitted, :last_update, :consumed_credits + );" + dbExecute(con, insertIntoQuery, param = list( + job_id = self$job_id, + user_id = self$user_id, + status = self$status, + process_graph = encodeProcessGraph(toJSON(self$process_graph$detailedInfo(),auto_unbox = TRUE,pretty=TRUE)), + submitted=as.character(self$submitted), + last_update = as.character(self$last_update), + consumed_credits = self$consumed_credits + )) + + } else { + updateQuery = "update job + set + user_id = :id, + status = :status, + process_graph = :process_graph, + submitted = :submitted, + last_update = :last_update, + consumed_credits = :consumed_credits + where job_id = :job_id;" + + dbExecute(con, updateQuery, param = list( + user_id = self$user_id, + status = self$status, + process_graph = encodeProcessGraph(toJSON(self$process_graph$detailedInfo(),auto_unbox = TRUE,pretty=TRUE)), + submitted = as.character(self$submitted), + last_update = as.character(self$last_update), + consumed_credits = self$consumed_credits, + job_id = self$job_id + )) + } + + # dir.create(self$filePath) + # write(x=json,file=paste(self$filePath,"/process_graph.json",sep="")) }, loadProcessGraph = function() { - parsedJson = fromJSON(paste(self$filePath,"/process_graph.json",sep="")) - self$process_graph = self$loadProcess(parsedJson[["process_graph"]]) + if (!isProcess(self$process_graph)) { + + if (is.character(self$process_graph)) { + parsedJson = fromJSON(self$process_graph, simplifyDataFrame = FALSE) + if (!"process_graph" %in% names(parsedJson)) { + parsedJson = list(process_graph=parsedJson) + } + } else if (is.list(self$process_graph)) { + if (!"process_graph" %in% names(self$process_graph)) { + parsedJson = list(process_graph=self$process_graph) + } + } else { + jsonText = dbGetQuery(openeo.server$database, "select process_graph from job where job_id = :id", param=list(id=self$job_id)) + parsedJson = fromJSON(jsonText, simplifyDataFrame = FALSE) + if (!"process_graph" %in% names(parsedJson)) { + parsedJson = list(process_graph=self$process_graph) + } + } + + + self$process_graph = self$loadProcess(parsedJson[["process_graph"]]) + } else { + # do nothing, we already have a process graph + } }, loadProcess = function(parsedJson) { @@ -105,6 +164,14 @@ Job <- R6Class( ) ) +encodeProcessGraph = function(text) { + return(bin2hex(charToRaw(text))) +} + +decodeProcessGraph = function(hex) { + return(rawToChar(hex2bin(hex))) +} + isJob = function(obj) { return("Job" %in% class(obj)) } \ No newline at end of file diff --git a/R/Process-class.R b/R/Process-class.R index 53b0f52..1effa98 100644 --- a/R/Process-class.R +++ b/R/Process-class.R @@ -73,7 +73,8 @@ Process <- R6Class( runner$args = clonedArguments for (key in 1:length(args)) { - value = args[[key]] + #it should always be a single key: value pair per argument + value = args[[key]][[1]] #TODO maybe add a handling for UDF or in the UDF class if (class(value) == "list" && "process_id" %in% names(value)) { diff --git a/R/Server-class.R b/R/Server-class.R index 0c962c2..749cebd 100644 --- a/R/Server-class.R +++ b/R/Server-class.R @@ -33,18 +33,14 @@ OpenEOServer <- R6Class( api.port = NULL, - jobs = NULL, processes = NULL, data = NULL, - users = NULL, database = NULL, initialize = function() { - self$jobs = list() self$processes = list() self$data = list() - self$users = list() }, startup = function (port=NA) { @@ -98,21 +94,7 @@ OpenEOServer <- R6Class( newObj = list(obj) names(newObj) = c(obj$product_id) - } else if (isJob(obj)) { - if (is.null(self$jobs)) { - self$jobs = list() - } - listName = "jobs" - - newObj = list(obj) - names(newObj) = c(obj$job_id) - - } else if (isUser(obj)) { - listName = "users" - - newObj = list(obj) - names(newObj) = c(obj$user_id) - }else { + } else { warning("Cannot register object. It is neither Process, Product nor Job.") return() } @@ -121,40 +103,28 @@ OpenEOServer <- R6Class( }, - deregister = function(obj) { - - if (isJob(obj)) { - self$jobs[[paste(obj$job_id)]] <- NULL - } else if (isUser(obj)) { - self$users[[paste(obj$user_id)]] <- NULL - } - - }, - delete = function(obj) { if (isJob(obj)) { - unlink(obj$filePath, recursive = TRUE,force=TRUE) - self$deregister(obj) + # unlink(obj$filePath, recursive = TRUE,force=TRUE) + dbExecute(openeo.server$database, "delete from job where job_id = :id",param=list(id=obj$job_id)) } else if (isUser(obj)) { unlink(obj$workspace,recursive = TRUE) - self$deregister(obj) + dbExecute(openeo.server$database, "delete from user where user_id = :id",param=list(id=obj$user_id)) } }, - createJob = function(user,job_id = NULL) { + createJob = function(user,job_id = NULL, process_graph = process_graph) { if (is.null(job_id)) { job_id = private$newJobId() } - path=paste(user$workspace,"jobs",job_id,sep="/") - - job = Job$new(job_id = job_id, filePath = path) + job = Job$new(job_id = job_id, process_graph = process_graph,user_id = user$user_id) return(job) }, - createUser = function(user_name, password) { + createUser = function(user_name, password, silent=FALSE) { files.folder = "files" id = private$newUserId() @@ -171,24 +141,12 @@ OpenEOServer <- R6Class( dir.create(user$workspace, showWarnings = FALSE) dir.create(paste(user$workspace,files.folder,sep="/"), showWarnings = FALSE) - - return(user) - - }, - - getUserByName = function(user_name) { - user_names = sapply(self$users, function(user) { - return(user$user_name) - }) - index = which(user_name %in% user_names) - - if (length(index) == 1) { - return(openeo.server$users[[index]]) - } else { - stop(paste("Cannot find user by user_name: ",user_names,sep="")) - return() + if (!silent) { + return(user) } + }, + loadDemo = function() { self$initEnvironmentDefault() @@ -198,8 +156,12 @@ OpenEOServer <- R6Class( private$loadDemoProcesses() }, + getConnection = function() { + return(dbConnect(RSQLite::SQLite(),self$sqlite.path)) + }, + initializeDatabase = function() { - self$database = dbConnect(RSQLite::SQLite(),self$sqlite.path) + self$database = self$getConnection() if (!dbExistsTable(self$database,"user")) { dbExecute(self$database, "create table user (user_id integer, @@ -211,7 +173,9 @@ OpenEOServer <- R6Class( dbExecute(self$database, "create table job (job_id text, user_id integer, status text, - submitted text, + submitted text, + last_update text, + consumed_credits integer, process_graph text)") } }, @@ -234,6 +198,46 @@ OpenEOServer <- R6Class( if (is.null(self$api.port)) { self$api.port <- 8000 } + }, + + loadUser = function(user_id) { + + if (dbGetQuery(openeo.server$database, "select count(*) from user where user_id = :id" + ,param = list(id=user_id)) == 1 + ) { + user_info = dbGetQuery(openeo.server$database, "select * from user where user_id = :id" + ,param = list(id=user_id)) + + user = User$new(user_id) + user$user_name = user_info$user_name + return(user) + } else { + stop("Cannot load user. It doesn't exists or too many user entries.") + } + + }, + loadJob = function(job_id) { + if (self$jobExists(job_id)) { + job_info = dbGetQuery(self$database, "select * from job where job_id = :id" + ,param = list(id=job_id)) + + job = Job$new(job_id) + job$user_id = job_info$user_id + job$status = job_info$status + job$submitted = job_info$submitted + job$last_update = job_info$last_update + job$consumed_credits = job_info$consumed_credits + job$process_graph = decodeProcessGraph(job_info$process_graph) + + job$loadProcessGraph() + + + return(job) + } + }, + jobExists = function(job_id) { + return(dbGetQuery(openeo.server$database, "select count(*) from job where job_id = :id" + ,param = list(id=job_id)) == 1) } ), private = list( @@ -244,35 +248,6 @@ OpenEOServer <- R6Class( loadSentinel2Data() }, - loadUser = function(id) { - ids = list.files(self$workspaces.path) - if(! id %in% ids) { - return() - } - - workspace.path = paste(self$workspaces.path, id,sep="/") - parsedJson = fromJSON(paste(workspace.path,"user.json",sep="/")) - user = User$new(id) - user$user_name = parsedJson[["user_name"]] - user$password = parsedJson[["password"]] - # user$workspace = workspace.path - - self$register(user) - - return(user) - - }, - - loadUsers = function() { - self$users = list() - user_folders = list.dirs(self$workspaces.path,recursive = FALSE,full.names = FALSE) - - for (user_id in user_folders) { - user = private$loadUser(user_id) - private$loadExistingJobs(user) - } - }, - loadDemoProcesses = function() { self$processes = list() @@ -288,48 +263,6 @@ OpenEOServer <- R6Class( }, - loadExistingJobs = function(user) { - if (missing(user) || is.null(user) || !isUser(user)) { - stop("Illegal argument for 'user'. A openeo User object is required") - } - - self$jobs = list() - - for (jobid in user$jobs) { - job.workspace = paste(user$workspace,"jobs",jobid,sep="/") - - parsedJson = fromJSON(file(paste(job.workspace,"process_graph.json",sep="/"))) - - fields = names(parsedJson) - - owner = user$user_id - - job = Job$new(job_id=jobid, filePath = job.workspace) - job$user_id = owner - - if ("submitted" %in% fields) { - job$submitted = parsedJson[["submitted"]] - } - - if ("status" %in% fields) { - job$status = parsedJson[["status"]] - } - - if ("evaluation" %in% fields) { - job$evaluation = parsedJson[["evaluation"]] - } - - if ("process_graph" %in% fields) { - job$loadProcessGraph() - } else { - warning(paste("job '",jobid,"' is corrupt. process_graph is missing. Please delete.",sep=""),immediate. = TRUE) - next() - } - - self$register(job) - } - }, - newJobId = function(n=1, length=15) { # cudos to https://ryouready.wordpress.com/2008/12/18/generate-random-string-name/ randomString <- c(1:n) @@ -339,7 +272,9 @@ OpenEOServer <- R6Class( collapse="") } - if (randomString %in% names(self$jobs)) { + jobIdExists = dbGetQuery(self$database, "select count(*) from job where job_id = :id", param=list(id=randomString)) == 1 + + if (jobIdExists) { # if id exists get a new one (recursive) return(self$newJobId()) } else { @@ -381,7 +316,6 @@ migrateFilesToDB = function(db, workspace) { # read user.json if (file.exists(row$user_file)) { user_info = fromJSON(row$user_file) - user_info$jobs = NULL user_info$login_secret = "" if (dbGetQuery(db,"select count(*) from user where user_id=:id",param=list(id=user_info$user_id))==0) { dbWriteTable(db,"user",as.data.frame(user_info),append=TRUE) diff --git a/R/User-class.R b/R/User-class.R index de969cd..be5913b 100644 --- a/R/User-class.R +++ b/R/User-class.R @@ -51,7 +51,7 @@ User <- R6Class( }, jobs = function() { - result = dbGetQuery(openeo.server$database, "select job_id from job where user_id = :id",param(id = self$user_id)) + result = dbGetQuery(openeo.server$database, "select job_id from job where user_id = :id",param=list(id = self$user_id)) return(as.list(result)[[1]]) } diff --git a/R/api.R b/R/api.R index 5ba610f..c9c8d9c 100644 --- a/R/api.R +++ b/R/api.R @@ -108,12 +108,21 @@ openeo.server$api.version <- "0.0.1" #* @get /api/jobs/ .describeJob = function(req,res,jobid) { - if (!jobid %in% names(openeo.server$jobs)) { - error(res,404,paste("Job with job_id",jobid," was not found")) + if (openeo.server$jobExists(jobid)) { + job = openeo.server$loadJob(jobid) + tryCatch( + { + res$body = toJSON(job$detailedInfo(),na="null",null="null",auto_unbox = TRUE) + res$setHeader("Content-Type","application/json") + }, + error = function(err) { + error(res,"500",err$message) + } + ) } else { - res$body = toJSON(openeo.server$jobs[[jobid]]$detailedInfo(),na="null",null="null",auto_unbox = TRUE) - res$setHeader("Content-Type","application/json") + error(res,404,paste("Job with id:",job_id,"cannot been found")) } + return(res) } @@ -123,25 +132,18 @@ openeo.server$api.version <- "0.0.1" if (is.null(evaluate) || !evaluate %in% c("lazy","batch")) { return(error(res,400, "Missing query parameter \"evaluate\" or it contains a value other then \"lazy\" or \"batch\"")) } - - job = openeo.server$createJob(user = req$user) - - openeo.server$register(job) - - data=list() - + # TODO check if postBody is valid process_graph = fromJSON(req$postBody) - data[["process_graph"]] = process_graph - - data[["status"]] = "submitted" - data[["evaluation"]] = evaluate + job = openeo.server$createJob(user = req$user, process_graph = process_graph) submit_time = Sys.time() - data[["submitted"]] = submit_time + job$status = "submitted" + job$evaluation = evaluate job$submitted = submit_time + job$last_update = submit_time - - job$store(json = toJSON(data,pretty=TRUE,auto_unbox = TRUE)) + job$loadProcessGraph() + job$store(con=openeo.server$database) if (evaluate == "batch") { #TODO load processgraph and execute @@ -155,8 +157,8 @@ openeo.server$api.version <- "0.0.1" #* @delete /api/jobs/ #* @serializer unboxedJSON .deleteJob = function(req,res,job_id) { - if (job_id %in% names(openeo.server$jobs)) { - job = openeo.server$jobs[[job_id]] + if (openeo.server$jobExists(job_id)) { + job = openeo.server$loadJob(job_id) tryCatch( { openeo.server$delete(job) @@ -276,16 +278,14 @@ openeo.server$api.version <- "0.0.1" if (paste(userid) == paste(req$user$user_id)) { user = req$user - possibleUserJobs = user$jobs - foundIndices = which(possibleUserJobs %in% names(openeo.server$jobs)) - userJobsIds = possibleUserJobs[foundIndices] - userJobs = openeo.server$jobs[userJobsIds] - jobRepresentation = lapply(userJobs, function(job){ + possibleUserJobs = user$jobs + jobRepresentation = lapply(possibleUserJobs, function(job_id){ + job = openeo.server$loadJob(job_id) return(job$detailedInfo()) }) - names(jobRepresentation) <- NULL - return(jobRepresentation) + + return(unname(jobRepresentation)) } else { error(res,401,"Not authorized to view jobs of others") } @@ -301,7 +301,6 @@ openeo.server$api.version <- "0.0.1" decoded = rawToChar(base64_dec(encoded)) user_name = unlist(strsplit(decoded,":"))[1] user_pwd = unlist(strsplit(decoded,":"))[2] - browser() tryCatch( { result = dbGetQuery(openeo.server$database, "select * from user where user_name = :name limit 1",param=list(name=user_name)) @@ -338,11 +337,10 @@ openeo.server$api.version <- "0.0.1" #* @serializer contentType list(type="image/GTiff") #* @get /api/download/ .downloadSimple = function(req,res,job_id,format) { - listedJobs = names(openeo.server$jobs) - if (!job_id %in% listedJobs) { + if (!openeo.server$jobExists(job_id)) { error(res, 404, paste("Cannot find job with id:",job_id)) } else { - job = openeo.server$jobs[[job_id]] + job = openeo.server$loadJob(job_id) result = job$run() rasterdata = result$granules[[1]]$data @@ -377,22 +375,13 @@ openeo.server$api.version <- "0.0.1" nonce = hextoken[((length(hextoken)-nonce.length)+1):length(hextoken)] user_id = rawToChar(data_decrypt(msg,openeo.server$secret.key,nonce)) - if (dbGetQuery(openeo.server$database, "select count(*) from user where user_id = :id" - ,param = list(id=user_id)) == 1 - ) { - - user_info = dbGetQuery(openeo.server$database, "select * from user where user_id = :id" - ,param = list(id=user_id)) - - user = User$new(user_id) - user$user_name = user_info$user_name - - req$user = user - - forward() - } else { - stop("Incorrect token") - } + + + user = openeo.server$loadUser(user_id) + req$user = user + + forward() + } else { stop("Incorrect authentication method.") } diff --git a/R/processes.R b/R/processes.R index c299edf..21adbd7 100644 --- a/R/processes.R +++ b/R/processes.R @@ -134,7 +134,6 @@ calculate_ndvi = Process$new( # either a product or an intermediate calculation. In any case the result shall # be a collection on which the calculations shall be performed. getCollectionFromImageryStatement = function (imagery) { - collection = NULL if (isProduct(imagery)) { From 97ca77a2e1dcadcf784a97c3ab20c6e8516a5fd8 Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Tue, 6 Feb 2018 11:16:16 +0100 Subject: [PATCH 05/10] fixed an issue with wrong detailed job information (value names were doubled and nested) --- R/Collection-class.R | 1 - R/ExecutableProcess-class.R | 7 +++++-- R/Job-class.R | 4 +++- R/Process-class.R | 4 +--- R/Server-class.R | 3 +-- R/api.R | 1 + 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/R/Collection-class.R b/R/Collection-class.R index c207ea7..ca44c05 100644 --- a/R/Collection-class.R +++ b/R/Collection-class.R @@ -79,7 +79,6 @@ Collection <- R6Class( .collection.filterbytime = function (granules,from,to) { minpos = -1 maxpos = -1 - browser() if (from > to) { old <- to to <- from diff --git a/R/ExecutableProcess-class.R b/R/ExecutableProcess-class.R index 15a5085..a7b44be 100644 --- a/R/ExecutableProcess-class.R +++ b/R/ExecutableProcess-class.R @@ -54,8 +54,11 @@ ExecutableProcess <- R6Class( detailedInfo = function() { args = list() - args=lapply(self$args, function(argument) {argument$valueInfo()}) - + args=lapply(self$args, function(argument) { + # this needs to be the unlisted version since valueInfo is also used to get a simplified represenation of processes + argument$valueInfo()[[1]] + }) + browser() return(list( process_id = self$process_id, args = args diff --git a/R/Job-class.R b/R/Job-class.R index 071828a..d1acd12 100644 --- a/R/Job-class.R +++ b/R/Job-class.R @@ -109,7 +109,7 @@ Job <- R6Class( parsedJson = list(process_graph=self$process_graph) } } else { - jsonText = dbGetQuery(openeo.server$database, "select process_graph from job where job_id = :id", param=list(id=self$job_id)) + jsonText = dbGetQuery(openeo.server$database, "select process_graph from job where job_id = :id", param=list(id=self$job_id))[1,] parsedJson = fromJSON(jsonText, simplifyDataFrame = FALSE) if (!"process_graph" %in% names(parsedJson)) { parsedJson = list(process_graph=self$process_graph) @@ -152,6 +152,8 @@ Job <- R6Class( last_update = self$last_update, consumed_credits = self$consumed_credits ) + + return(info) }, run = function() { diff --git a/R/Process-class.R b/R/Process-class.R index 1effa98..4855ee9 100644 --- a/R/Process-class.R +++ b/R/Process-class.R @@ -42,7 +42,6 @@ Process <- R6Class( for (arg in self$args) { arglist = append(arglist, arg$shortInfo()) } - res = list( process_id = self$process_id, description = self$description, @@ -56,7 +55,6 @@ Process <- R6Class( if (is.null(job)) { stop("No job defined for this executable process") } - #return a process where the arguments from the parsed json file are set for #this "args". E.g. set a value for args[["from"]]$value and set Process$executable to TRUE @@ -74,7 +72,7 @@ Process <- R6Class( for (key in 1:length(args)) { #it should always be a single key: value pair per argument - value = args[[key]][[1]] + value = args[[key]] #TODO maybe add a handling for UDF or in the UDF class if (class(value) == "list" && "process_id" %in% names(value)) { diff --git a/R/Server-class.R b/R/Server-class.R index 749cebd..c6b11cb 100644 --- a/R/Server-class.R +++ b/R/Server-class.R @@ -220,14 +220,13 @@ OpenEOServer <- R6Class( if (self$jobExists(job_id)) { job_info = dbGetQuery(self$database, "select * from job where job_id = :id" ,param = list(id=job_id)) - job = Job$new(job_id) job$user_id = job_info$user_id job$status = job_info$status job$submitted = job_info$submitted job$last_update = job_info$last_update job$consumed_credits = job_info$consumed_credits - job$process_graph = decodeProcessGraph(job_info$process_graph) + job$process_graph = fromJSON(decodeProcessGraph(job_info$process_graph),simplifyDataFrame = FALSE) job$loadProcessGraph() diff --git a/R/api.R b/R/api.R index c9c8d9c..5be59ec 100644 --- a/R/api.R +++ b/R/api.R @@ -143,6 +143,7 @@ openeo.server$api.version <- "0.0.1" job$last_update = submit_time job$loadProcessGraph() + job$store(con=openeo.server$database) if (evaluate == "batch") { From cdab2ab3c9bb10939d5c1bcb0c81a2cb0bbd38b8 Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Tue, 6 Feb 2018 14:11:36 +0100 Subject: [PATCH 06/10] fixed an issue where either jobs where not stored correctly or which are incorrectly returned --- R/ExecutableProcess-class.R | 3 +-- R/Job-class.R | 4 ++-- R/Process-class.R | 1 - R/api.R | 4 +--- R/processes.R | 14 +++++++++++--- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/R/ExecutableProcess-class.R b/R/ExecutableProcess-class.R index a7b44be..6db86f1 100644 --- a/R/ExecutableProcess-class.R +++ b/R/ExecutableProcess-class.R @@ -56,9 +56,8 @@ ExecutableProcess <- R6Class( args=lapply(self$args, function(argument) { # this needs to be the unlisted version since valueInfo is also used to get a simplified represenation of processes - argument$valueInfo()[[1]] + argument$valueInfo() }) - browser() return(list( process_id = self$process_id, args = args diff --git a/R/Job-class.R b/R/Job-class.R index d1acd12..8b7193b 100644 --- a/R/Job-class.R +++ b/R/Job-class.R @@ -64,7 +64,7 @@ Job <- R6Class( job_id = self$job_id, user_id = self$user_id, status = self$status, - process_graph = encodeProcessGraph(toJSON(self$process_graph$detailedInfo(),auto_unbox = TRUE,pretty=TRUE)), + process_graph = encodeProcessGraph(toJSON(self$process_graph,auto_unbox = TRUE,pretty=TRUE)), submitted=as.character(self$submitted), last_update = as.character(self$last_update), consumed_credits = self$consumed_credits @@ -122,7 +122,7 @@ Job <- R6Class( # do nothing, we already have a process graph } }, - + loadProcess = function(parsedJson) { processId = parsedJson[["process_id"]] #TODO: add cases for udfs diff --git a/R/Process-class.R b/R/Process-class.R index 4855ee9..4484723 100644 --- a/R/Process-class.R +++ b/R/Process-class.R @@ -71,7 +71,6 @@ Process <- R6Class( runner$args = clonedArguments for (key in 1:length(args)) { - #it should always be a single key: value pair per argument value = args[[key]] #TODO maybe add a handling for UDF or in the UDF class diff --git a/R/api.R b/R/api.R index 5be59ec..f9b5ab6 100644 --- a/R/api.R +++ b/R/api.R @@ -133,7 +133,7 @@ openeo.server$api.version <- "0.0.1" return(error(res,400, "Missing query parameter \"evaluate\" or it contains a value other then \"lazy\" or \"batch\"")) } # TODO check if postBody is valid - process_graph = fromJSON(req$postBody) + process_graph = fromJSON(req$postBody,simplifyDataFrame = FALSE) job = openeo.server$createJob(user = req$user, process_graph = process_graph) submit_time = Sys.time() @@ -142,8 +142,6 @@ openeo.server$api.version <- "0.0.1" job$submitted = submit_time job$last_update = submit_time - job$loadProcessGraph() - job$store(con=openeo.server$database) if (evaluate == "batch") { diff --git a/R/processes.R b/R/processes.R index 21adbd7..5f31be9 100644 --- a/R/processes.R +++ b/R/processes.R @@ -22,12 +22,12 @@ filter_daterange = Process$new( ) ), operation = function(imagery, from=NA, to=NA) { + cat("Starting filter_daterange\n") #imagery might be an identifier or a function (Process$execute()) or a json process description or a # udf or a collection we need to specify that collection = NULL collection = getCollectionFromImageryStatement(imagery) - if (is.null(collection)) { stop("no collection element found in function call") } @@ -56,25 +56,30 @@ find_min = Process$new( ) ), operation = function(imagery) { + cat("Starting find_min\n") #get the collection of the imagery collection = getCollectionFromImageryStatement(imagery) #get a list of the data (raster objects) rasters = lapply(collection$granules, function(obj){obj$data}) - + cat("Fetched related granules\n") #create a brick data = stack(rasters) + cat("Stacking data") #calculate minimum = calc(data,fun=min,na.rm=T) + cat("calculating the minimum\n") #create a granule aggregation = Granule$new(time=collection$getMinTime(),data=minimum,extent=extent(minimum),srs=crs(minimum)) + cat("creating single granule for minimum calculation\n") #create a collection collection = Collection$new() collection$addGranule(aggregation) collection$sortGranulesByTime + cat("Creating collection for single granule and setting meta data\n") return(collection) } @@ -99,9 +104,11 @@ calculate_ndvi = Process$new( required = TRUE )), operation=function(imagery,nir,red) { + cat("Starting calculate_ndvi\n") collection = getCollectionFromImageryStatement(imagery) nir.index = collection$getBandIndex(nir) red.index = collection$getBandIndex(red) + cat("Fetched indices for bands\n") # fetch the data elements and simultanously calculate ndvi rasters = lapply(collection$granules, function(obj){ @@ -119,10 +126,12 @@ calculate_ndvi = Process$new( )) return(granule) }) + cat("ndvi calculation applied on all granules\n") result.collection = Collection$new() result.collection$granules = rasters result.collection$sortGranulesByTime() + cat("set metadata for newly calculated collection\n") return(result.collection) } @@ -135,7 +144,6 @@ calculate_ndvi = Process$new( # be a collection on which the calculations shall be performed. getCollectionFromImageryStatement = function (imagery) { collection = NULL - if (isProduct(imagery)) { collection = imagery$getCollection() } else if (isCollection(imagery)) { From f2c3575fb120f6321c5165e0b63ba778c533f700 Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Tue, 6 Feb 2018 16:08:37 +0100 Subject: [PATCH 07/10] added an api augmentation that allows correct process_graph parsing and the simple version --- R/api.R | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/R/api.R b/R/api.R index f9b5ab6..ec8b17f 100644 --- a/R/api.R +++ b/R/api.R @@ -134,6 +134,9 @@ openeo.server$api.version <- "0.0.1" } # TODO check if postBody is valid process_graph = fromJSON(req$postBody,simplifyDataFrame = FALSE) + # TODO check if this is the simple representation or the complex (probably correct version) + # this means search for "args" lists if (names(...$args) == NULL) => unlist(...$args, recursive = FALSE) + process_graph = .createSimpleArgList(process_graph) job = openeo.server$createJob(user = req$user, process_graph = process_graph) submit_time = Sys.time() @@ -153,6 +156,25 @@ openeo.server$api.version <- "0.0.1" )) } +.createSimpleArgList = function(graph) { + if ("args" %in% names(graph)) { + + if (is.null(names(graph$args))) { + args = unlist(graph$args,recursive = FALSE) + + #named list it should be + for (index in names(args)) { + elem = args[[index]] + if ("args" %in% names(elem)) { + args[[index]] = .createSimpleArgList(elem) + } + } + graph$args = args + } + } + return(graph) +} + #* @delete /api/jobs/ #* @serializer unboxedJSON .deleteJob = function(req,res,job_id) { From 4b9e502e6eae6f26e9bd4858aa45170115116749 Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Tue, 6 Feb 2018 16:22:35 +0100 Subject: [PATCH 08/10] removed "exutable" flag on ExecutableProcess (relict) --- R/ExecutableProcess-class.R | 1 - R/Process-class.R | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/R/ExecutableProcess-class.R b/R/ExecutableProcess-class.R index 6db86f1..cd3e373 100644 --- a/R/ExecutableProcess-class.R +++ b/R/ExecutableProcess-class.R @@ -5,7 +5,6 @@ ExecutableProcess <- R6Class( "ExecutableProcess", inherit = Process, public = list( - executable=FALSE, initialize= function(process_id = NA, description = NA, diff --git a/R/Process-class.R b/R/Process-class.R index 4484723..a3fca3c 100644 --- a/R/Process-class.R +++ b/R/Process-class.R @@ -56,7 +56,7 @@ Process <- R6Class( stop("No job defined for this executable process") } #return a process where the arguments from the parsed json file are set for - #this "args". E.g. set a value for args[["from"]]$value and set Process$executable to TRUE + #this "args". E.g. set a value for args[["from"]]$value # json at this point is the named list of the process graph provided by the json stored under jobs args = json$args From 56333d634636bf55e8f6759000fae7aeda851cc5 Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Tue, 6 Feb 2018 16:45:32 +0100 Subject: [PATCH 09/10] added OPTIONS methods for each endpoint regarding CORS --- R/api.R | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/R/api.R b/R/api.R index ec8b17f..c812510 100644 --- a/R/api.R +++ b/R/api.R @@ -415,12 +415,17 @@ openeo.server$api.version <- "0.0.1" .cors_filter = function(res) { res$setHeader("Access-Control-Allow-Origin", "*") - res$setHeader("Access-Control-Allow-Headers", "Authorization, Accept, Content-Type") res$setHeader("Access-Control-Allow-Credentials", TRUE) - res$setHeader("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE,OPTIONS,PATCH") plumber::forward() } +.cors_option_bypass = function(req,res) { + res$setHeader("Access-Control-Allow-Headers", "Authorization, Accept, Content-Type") + res$setHeader("Access-Control-Allow-Methods", "GET,POST,PUT,DELETE,OPTIONS,PATCH") + + return(res) +} + ############################ # # utility functions @@ -465,12 +470,19 @@ createAPI = function() { "/api/version", handler = .version, serializer = serializer_unboxed_json()) + root$handle("OPTIONS", + "/api/version", + handler = .cors_option_bypass) root$handle("GET", "/api/capabilities", handler = .capabilities, serializer = serializer_unboxed_json()) + root$handle("OPTIONS", + "/api/capabilities", + handler = .cors_option_bypass) + root$registerHook("postroute",.cors_filter) data = plumber$new() @@ -479,11 +491,17 @@ createAPI = function() { "/", handler = .listData, serializer = serializer_unboxed_json()) + data$handle("OPTIONS", + "/", + handler = .cors_option_bypass) data$handle("GET", "/", handler = .describeData, serializer = serializer_unboxed_json()) + data$handle("OPTIONS", + "/", + handler = .cors_option_bypass) root$mount("/api/data",data) @@ -493,11 +511,17 @@ createAPI = function() { "/", handler = .listProcesses, serializer = serializer_unboxed_json()) + process$handle("OPTIONS", + "/", + handler = .cors_option_bypass) process$handle("GET", "/", handler = .describeProcess, serializer = serializer_unboxed_json()) + process$handle("OPTIONS", + "/", + handler = .cors_option_bypass) root$mount("/api/processes",process) @@ -507,16 +531,23 @@ createAPI = function() { "/", handler = .describeJob, serializer = serializer_unboxed_json()) + jobs$handle("OPTIONS", + "/", + handler = .cors_option_bypass) jobs$handle("POST", "/", handler = .createNewJob, serializer = serializer_unboxed_json()) + jobs$handle("OPTIONS", + "/", + handler = .cors_option_bypass) jobs$handle("DELETE", "/", handler = .deleteJob, serializer = serializer_unboxed_json()) + jobs$filter("authorization",.authorized) @@ -528,6 +559,9 @@ createAPI = function() { "//files", handler = .listUserFiles, serializer = serializer_unboxed_json()) + users$handle("OPTIONS", + "//files", + handler = .cors_option_bypass) users$handle("GET", "//files/", @@ -544,11 +578,19 @@ createAPI = function() { handler = .deleteUserData, serializer = serializer_unboxed_json()) + users$handle("OPTIONS", + "//files/", + handler = .cors_option_bypass) + users$handle("GET", "//jobs", handler = .listUserJobs, serializer = serializer_unboxed_json()) + users$handle("OPTIONS", + "//jobs", + handler = .cors_option_bypass) + users$filter("authorization",.authorized) root$mount("/api/users",users) @@ -560,15 +602,22 @@ createAPI = function() { handler = .login, serializer = serializer_unboxed_json()) + authentication$handle("OPTIONS", + "/login", + handler = .cors_option_bypass) + root$mount("/api/auth",authentication) download = plumber$new() - download$handle( - "GET", - "/", - handler = .downloadSimple, - serializer = serializer_unboxed_json()) + download$handle("GET", + "/", + handler = .downloadSimple, + serializer = serializer_unboxed_json()) + + download$handle("OPTIONS", + "/", + handler = .cors_option_bypass) download$filter("authorization", .authorized) From 0c8bb4ab915ce63e6bcc764b554d50bec74595e9 Mon Sep 17 00:00:00 2001 From: Florian Lahn Date: Tue, 6 Feb 2018 16:48:34 +0100 Subject: [PATCH 10/10] raised version to 0.1 --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 1708d02..83759ca 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: openEO.R.Backend Title: Reference backend implementation for openEO conformant backend with a local filesystem -Version: 0.0.1 +Version: 0.1 Authors@R: person("Florian", "Lahn", email = "florian.lahn@uni-muenster.de", role = c("aut", "cre")) Description: The package contains a backend solution in compliance with the openEO API. In this demonstration the file backend solution is the local file system containing some raster data collections.