From 73c0c27c7690a98b3d1a7c4bb64242d2190dbeb8 Mon Sep 17 00:00:00 2001 From: gfenoy Date: Thu, 16 Feb 2023 00:08:47 +0100 Subject: [PATCH 1/7] EOEPCA-807 #comment Add support for datetime,min/maxDuration in ADES 2.0 --- .../assets/patch/zoo/response_print.c | 12 +- .../assets/patch/zoo/service.c | 19 +- .../assets/patch/zoo/service.h | 5 +- .../assets/patch/zoo/service_json.c | 409 ++++++++++++++---- .../assets/patch/zoo/ulinet.c | 2 +- .../assets/patch/zoo/zoo_service_loader.c | 400 +++++++++++------ Dockerfile | 4 +- assets/oas.cfg | 24 +- 8 files changed, 641 insertions(+), 234 deletions(-) diff --git a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/response_print.c b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/response_print.c index cd2e7dc..2307f6a 100644 --- a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/response_print.c +++ b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/response_print.c @@ -1,7 +1,7 @@ /* * Author : Gérald FENOY * - * Copyright (c) 2009-2020 GeoLabs SARL + * Copyright (c) 2009-2023 GeoLabs SARL * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -2000,7 +2000,7 @@ void printProcessResponse(maps* m,map* request, int pid,service* serv,const char tmp1 = (char*)malloc((TIME_SIZE+1)*sizeof(char)); - len = strftime ( tmp1, TIME_SIZE, "%Y-%m-%dT%I:%M:%SZ", tm ); + len = strftime ( tmp1, TIME_SIZE, zDateFormat, tm ); xmlNewProp(nc,BAD_CAST "creationTime",BAD_CAST tmp1); @@ -2632,16 +2632,16 @@ void _printExceptionReportResponse(maps* m,map* s){ * (when required) depending on the code. * * @param pmsConf the maps containing the settings of the main.cfg file - * @param psService the service + * @param pmError the map containing the text,code,locator keys (or a map array) */ -void printExceptionReportResponse(maps* pmsConf,map* psService){ +void printExceptionReportResponse(maps* pmsConf,map* pmError){ if(getMapFromMaps(pmsConf,"lenv","hasPrinted")!=NULL) return; map* pmExecutionType=getMapFromMaps(pmsConf,"main","executionType"); if(pmExecutionType!=NULL && strncasecmp(pmExecutionType->value,"xml",3)==0) - _printExceptionReportResponse(pmsConf,psService); + _printExceptionReportResponse(pmsConf,pmError); else - printExceptionReportResponseJ(pmsConf,psService); + printExceptionReportResponseJ(pmsConf,pmError); } /** diff --git a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service.c b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service.c index 914e3bd..82279f6 100644 --- a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service.c +++ b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service.c @@ -840,18 +840,23 @@ void loadMapBinary(map** ppmOut,map* pmIn,int iPos){ sprintf(tmp,"value_%d",iPos); pmTmpVin=getMap(pmIn,tmp); pmTmpVout=getMap(pmOut,tmp); - free(pmTmpVout->value); - pmTmpVout->value=(char*)malloc((atoi(pmSize->value)+1)*sizeof(char)); - memmove(pmTmpVout->value,pmTmpVin->value,atoi(pmSize->value)*sizeof(char)); - pmTmpVout->value[atoi(pmSize->value)]=0; + if(pmTmpVin!=NULL && pmTmpVout!=NULL){ + free(pmTmpVout->value); + pmTmpVout->value=(char*)malloc((atoi(pmSize->value)+1)*sizeof(char)); + memmove(pmTmpVout->value,pmTmpVin->value,atoi(pmSize->value)*sizeof(char)); + pmTmpVout->value[atoi(pmSize->value)]=0; + } }else{ if(pmSize!=NULL){ pmTmpVin=getMap(pmIn,"value"); pmTmpVout=getMap(pmOut,"value"); free(pmTmpVout->value); - pmTmpVout->value=(char*)malloc((atoi(pmSize->value)+1)*sizeof(char)); - memmove(pmTmpVout->value,pmTmpVin->value,atoi(pmSize->value)*sizeof(char)); - pmTmpVout->value[atoi(pmSize->value)]=0; + if(pmTmpVin!=NULL && pmTmpVout!=NULL){ + free(pmTmpVout->value); + pmTmpVout->value=(char*)malloc((atoi(pmSize->value)+1)*sizeof(char)); + memmove(pmTmpVout->value,pmTmpVin->value,atoi(pmSize->value)*sizeof(char)); + pmTmpVout->value[atoi(pmSize->value)]=0; + } } } } diff --git a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service.h b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service.h index 7526191..ac6bec4 100644 --- a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service.h +++ b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service.h @@ -33,7 +33,10 @@ #define ZOO_DLL_EXPORT #endif - // knut: add bool if necessary +// ISO8601 compatible date format (RFC 3339) +#define zDateFormat "%Y-%m-%dT%H:%M:%SZ" + +// knut: add bool if necessary #ifndef __cplusplus #ifndef WIN32 #include diff --git a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service_json.c b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service_json.c index 6359c29..a357212 100644 --- a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service_json.c +++ b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/service_json.c @@ -124,23 +124,7 @@ extern "C" { } } else{ - // In other case we should consider array with potential sizes - int limit=atoi(length->value); - int i=0; - val=json_object_new_array(); - for(i=0;iname,"value",5)==0){ - if(strlen(cursor->value)!=0 && toLoad!=NULL && strncasecmp(toLoad->value,"true",4)==0) - json_object_array_add(val,json_object_new_string_len(cursor->value,atoi(sizeMap->value))); - }else{ - if(strstr(cursor->name,"title")!=NULL) - json_object_array_add(val,json_object_new_string(_(cursor->value))); - else - json_object_array_add(val,json_object_new_string(cursor->value)); - } - } + val=json_object_new_string(cursor->value); } } if(val!=NULL) @@ -299,7 +283,36 @@ extern "C" { */ void printLiteralValueJ(maps* pmConf,map* pmType,map* pmContent,json_object* schema,const char* field){ map* pmTmp=getMap(pmContent,"value"); - if(pmTmp!=NULL) + map* pmLength=getMap(pmContent,"length"); + if(pmLength!=NULL){ + int len=atoi(pmLength->value); + json_object* defaultProp=json_object_new_array(); + for(int i=0;ivalue,"integer",7)==0) + json_object_array_add(defaultProp,json_object_new_int(atoi(pmTmp0->value))); + else{ + if(pmType!=NULL && strncasecmp(pmType->value,"float",5)==0){ + json_object_array_add(defaultProp,json_object_new_double(atof(pmTmp0->value))); + json_object_object_add(schema,"format",json_object_new_string("double")); + } + else{ + if(pmType!=NULL && strncasecmp(pmType->value,"bool",4)==0){ + if(strncasecmp(pmTmp0->value,"true",4)==0) + json_object_array_add(defaultProp,json_object_new_boolean(true)); + else + json_object_array_add(defaultProp,json_object_new_boolean(false)); + } + else + json_object_array_add(defaultProp,json_object_new_string(pmTmp0->value)); + } + } + } + } + json_object_object_add(schema,field,defaultProp); + } + else if(pmTmp!=NULL) if(pmType!=NULL && strncasecmp(pmType->value,"integer",7)==0) json_object_object_add(schema,field,json_object_new_int(atoi(pmTmp->value))); else{ @@ -375,24 +388,6 @@ extern "C" { tmps = strtok_r (NULL, ",", &saveptr); } json_object_object_add(schema,"enum",prop5); - }else{ - tmpMap1=getMap(in->defaults->content,"DataType"); - if(tmpMap1!=NULL && strncasecmp(tmpMap1->value,"string",6)==0){ - map* pmsLength=getMap(in->defaults->content,"length"); - if(pmsLength!=NULL){ - json_object* prop6=json_object_new_array(); - int len=atoi(pmsLength->value); - int i=0; - for(i=0;idefaults->content,"value",i); - if(pmsCurrent!=NULL) - json_object_array_add(prop6,json_object_new_string(pmsCurrent->value)); - } - json_object_object_add(schema,"default",prop6); - } - else - json_object_object_add(schema,"default",json_object_new_string("Any value")); - } } } } @@ -658,19 +653,32 @@ extern "C" { map* cmeta=meta; json_object* carr=json_object_new_array(); int hasElement=-1; - while(cmeta!=NULL){ + int iCnt=0; + int iLen=1; + if(cmeta==NULL) + return; + map* pmLength=getMap(cmeta,"length"); + if(pmLength!=NULL) + iLen=atoi(pmLength->value); + char* pcFields[]={ + (char*)"role", + (char*)"href", + (char*)"title" + }; + for(;iCntname,"role")==0 || - strcasecmp(cmeta->name,"href")==0 || - strcasecmp(cmeta->name,"title")==0 ){ - if(strcasecmp(cmeta->name,"title")==0) - json_object_object_add(jcmeta,cmeta->name,json_object_new_string(_(cmeta->value))); - else - json_object_object_add(jcmeta,cmeta->name,json_object_new_string(cmeta->value)); - hasElement++; + int iCnt1=0; + for(;iCnt1<3;iCnt1++){ + map* pmValue=getMapArray(meta,pcFields[iCnt1],iCnt); + if(pmValue!=NULL){ + if(strcasecmp(pmValue->name,"title")==0) + json_object_object_add(jcmeta,pcFields[iCnt1],json_object_new_string(_(pmValue->value))); + else + json_object_object_add(jcmeta,pcFields[iCnt1],json_object_new_string(pmValue->value)); + hasElement++; + } } json_object_array_add(carr,jcmeta); - cmeta=cmeta->next; } if(hasElement>=0) json_object_object_add(doc,"metadata",carr); @@ -885,12 +893,18 @@ extern "C" { json_object_object_add(res,"version",json_object_new_string("1.0.0")); int limit=4; int i=0; + if(serv->metadata!=NULL){ + printJMetadata(m,serv->metadata,res); + } + if(serv->additional_parameters!=NULL){ + printJAdditionalParameters(m,serv->additional_parameters,res); + } map* sType=getMap(serv->content,"serviceType"); map* pmMutable=getMap(serv->content,"mutable"); if(pmMutable==NULL || strncasecmp(pmMutable->value,"true",4)==0){ i=2; limit=6; - } + } for(;ivalue,"false",5)==0){ - if(m!=NULL){ - map *tmpSid=getMapFromMaps(m,"lenv","sid"); - if(tmpSid!=NULL){ - if( getpid()==atoi(tmpSid->value) ){ + if(getMapFromMaps(m,"headers","Content-Type")==NULL){ + map* pmExceptionContentType=getMapFromMaps(m,"openapi","exception_mime_tye"); + if(pmExceptionContentType!=NULL) + setMapInMaps(m,"headers","Content-Type",pmExceptionContentType->value); + else + setMapInMaps(m,"headers","Content-Type","application/json;encoding=utf-8"); + } + printHeaders(m); + if(getMapFromMaps(m,"headers","status")==NULL){ + pmTmp=getMapFromMaps(m,"lenv","status_code"); + if(pmTmp!=NULL) + exceptionCode=pmTmp->value; + if(exceptionCode==NULL) + exceptionCode=aapccStatusCodes[3][0]; + if(pmNoHeaders==NULL || strncasecmp(pmNoHeaders->value,"false",5)==0){ + if(m!=NULL){ + map *tmpSid=getMapFromMaps(m,"lenv","sid"); + if(tmpSid!=NULL){ + if( getpid()==atoi(tmpSid->value) ){ + printf("Status: %s\r\n\r\n",exceptionCode); + } + } + else{ + printf("Status: %s\r\n\r\n",exceptionCode); + } + }else{ printf("Status: %s\r\n\r\n",exceptionCode); } } - else{ - printf("Status: %s\r\n\r\n",exceptionCode); - } - }else{ - printf("Status: %s\r\n\r\n",exceptionCode); } } - + const char* jsonStr=json_object_to_json_string_ext(res,JSON_C_TO_STRING_NOSLASHESCAPE); if(getMapFromMaps(m,"lenv","jsonStr")==NULL) setMapInMaps(m,"lenv","jsonStr",jsonStr); @@ -1235,6 +1266,25 @@ extern "C" { } } + /** + * Return an exception response with Status: "400 Bad Request" if inputs are + * not in the proper format. + * + * @param pmsConf the maps containing the settings of the main.cfg file + * @param peInput the input causing the issue + */ + void returnBadRequestException(maps* pmsConf,elements* peInput){ + map* pmError=createMap("code","BadRequest"); + char tmpS1[1024]; + sprintf(tmpS1,_("Wrong type used for input %s"),peInput->name); + addToMap(pmError,"message",tmpS1); + setMapInMaps(pmsConf,"lenv","status_code","400 Bad Request"); + printExceptionReportResponseJ(pmsConf,pmError); + freeMap(&pmError); + free(pmError); + return; + } + /** * Parse a single input / output entity * @@ -1533,8 +1583,14 @@ extern "C" { setMapInMaps(conf,"request","response",json_object_get_string(json_io)); }else{ if(getMap(request_inputs,"response")==NULL){ - addToMap(request_inputs,"response","raw"); - setMapInMaps(conf,"request","response","raw"); + // OGC Test Suite requirement (the execute.yaml schema said that it should be raw) + if(getMapFromMaps(conf,"openapi","default_result_as_document")!=NULL){ + addToMap(request_inputs,"response","document"); + setMapInMaps(conf,"request","response","document"); + }else{ + addToMap(request_inputs,"response","raw"); + setMapInMaps(conf,"request","response","raw"); + } } } json_io=NULL; @@ -1572,6 +1628,7 @@ extern "C" { */ json_object* printJobStatus(maps* pmsConf,char* pcJobId){ json_object* pjoRes=NULL; + maps* pmsLenv=getMaps(pmsConf,"lenv"); runGetStatus(pmsConf,pcJobId,(char*)"GetStatus"); map* pmError=getMapFromMaps(pmsConf,"lenv","error"); if(pmError!=NULL && strncasecmp(pmError->value,"true",4)==0){ @@ -1594,7 +1651,7 @@ extern "C" { pjoRes=createStatus(pmsConf,SERVICE_STARTED); } else - pjoRes=createStatus(pmsConf,SERVICE_FAILED); + pjoRes=createStatus(pmsConf,SERVICE_ACCEPTED); } return pjoRes; } @@ -1629,7 +1686,98 @@ extern "C" { return true; #endif } - + + /** + * Verify if the process identified by pccPid is part of the processID filters, + * if any. + * + * @param pmsConf the maps containing the settings of the main.cfg file + * @param pccPid the const char pointer to the processID + * @return true in case the filter is empty or in case the processId pccPid + * is one of the filtered processID, false in other case. + */ + bool isFilteredDMM(maps* pmsConf,const char* pccPid){ +#ifndef RELY_ON_DB + maps* pmsRequest=getMaps(pmsConf,"request"); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + if(pmsRequest!=NULL){ + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + dumpMap(pmsRequest->content); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + map* pmTmpPath=getMapFromMaps(pmsConf,"main","tmpPath"); + map* pmMinDate=getMap(pmsRequest->content,"minDuration"); + map* pmMaxDate=getMap(pmsRequest->content,"maxDuration"); + map* pmTimeInterval=getMap(pmsRequest->content,"datetime"); + if(pmMinDate==NULL && pmMaxDate==NULL && pmTimeInterval==NULL) + return true; + else{ + // Find datetime for files .sid + zStatStruct zssSid_status; + char* pcaSidPath=(char*)malloc((strlen(pccPid)+strlen(pmTmpPath->value)+6)*sizeof(char)); + sprintf(pcaSidPath,"%s/%s.sid",pmTmpPath->value,pccPid); + int iS1=zStat(pcaSidPath, &zssSid_status); + if(pmMinDate!=NULL || pmMaxDate!=NULL){ + // Find datetime for files _lenv.cfg + int iMin=0,iMax=86400; + zStatStruct zssLenv_status; + char* pcaLenvPath=(char*)malloc((strlen(pccPid)+strlen(pmTmpPath->value)+7)*sizeof(char)); + sprintf(pcaLenvPath,"%s/%s.json",pmTmpPath->value,pccPid); + int iS=zStat(pcaLenvPath, &zssLenv_status); + if(iS!=0) + return false; + free(pcaLenvPath); + // Use difftime to get elapsed seconds + double ulElapsedSeconds=0; + if(zssLenv_status.st_mtime!=zssSid_status.st_mtime){ + ulElapsedSeconds=(zssLenv_status.st_mtime-zssSid_status.st_mtime); + } + if(pmMinDate!=NULL) + iMin=atoi(pmMinDate->value); + if(pmMaxDate!=NULL) + iMax=atoi(pmMaxDate->value); + if(iMaxulElapsedSeconds){ + return false; + } + } + if(pmTimeInterval!=NULL){ + // Find datetime or interval + struct tm tmaLimits[2]; + bool baLimits[2]={false,false}; + char* token, *saveptr,*pcaDecodedValue,*pcaDupValue; + int iCnt=0; + pcaDupValue=zStrdup(pmTimeInterval->value); + pcaDecodedValue=url_decode(pcaDupValue); + free(pcaDupValue); + token=strtok_r(pcaDecodedValue,"/",&saveptr); + while(token!=NULL){ + if(iCnt<2){ + if(strncmp(token,"..",2)!=0 && strstr(token,",")==NULL){ + char* pcaDup=zStrdup(token); + strptime(pcaDup,zDateFormat,&tmaLimits[iCnt]); + baLimits[iCnt]=true; + free(pcaDup); + } + } + if(strstr(token,",")==NULL) + iCnt++; + token=strtok_r(NULL,"/",&saveptr); + } + free(pcaDecodedValue); + if((baLimits[0] && zssSid_status.st_mtimemktime(&tmaLimits[1]))){ + return false; + } + } + return true; + } + }else + return true; + return true; +#else + return true; +#endif + } + /** * Print the jobs list * @@ -1705,26 +1853,33 @@ extern "C" { pmsLenv->content = NULL; pmsLenv->child = NULL; pmsLenv->next = NULL; - map* pmPid=NULL; + map *pmaPid=NULL,*pmaUsid=NULL; if(conf_read(pcaLenvPath,pmsLenv) !=2){ map *pmTmp=getMapFromMaps(pmsLenv,"lenv","Identifier"); if(pmTmp!=NULL){ - pmPid=createMap("value",pmTmp->value); + pmaPid=createMap("value",pmTmp->value); } - if(pmPid==NULL) - pmPid=createMap("toRemove","true"); + if(pmaPid==NULL) + pmaPid=createMap("toRemove","true"); else - addToMap(pmPid,"toRemove","true"); + addToMap(pmaPid,"toRemove","true"); + pmTmp=getMapFromMaps(pmsLenv,"lenv","usid"); + if(pmTmp!=NULL){ + pmaUsid=createMap("value",pmTmp->value); + addToMap(pmaPid,"toRemove","true"); + } freeMaps(&pmsLenv); free(pmsLenv); }else{ - pmPid=createMap("toRemove","true"); + pmaPid=NULL; + pmaUsid=NULL; } free(pcaLenvPath); #else - map* pmPid=createMap("toRemove","true"); + map *pmaPid=NULL, *pmaUsid=NULL; #endif - if(isFilteredPid(conf,pmPid->value)){ + if(isFilteredPid(conf,pmaPid->value) && + isFilteredDMM(conf,pmaUsid->value)){ if(cnt>=skip && cntvalue); + setMapInMaps(conf,"lenv","gs_message",pmStatus->value); message=zStrdup(pmStatus->value); hasMessage=1; } @@ -2417,6 +2579,7 @@ extern "C" { } default: { + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); map* pmTmp=getMapFromMaps(conf,"lenv","force"); if(pmTmp==NULL || strncasecmp(pmTmp->value,"false",5)==0){ char* pacTmp=json_getStatusFilePath(conf); @@ -2438,6 +2601,7 @@ extern "C" { needResult=-1; } rstatus="failed"; + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); break; } } @@ -2471,8 +2635,30 @@ extern "C" { pmsLenv->child = NULL; pmsLenv->next = NULL; if (conf_read (pcaLenvPath,pmsLenv) != 2){ + zStatStruct zssLenv_status,zssSid_status; + int iS=zStat(pcaLenvPath, &zssLenv_status); + char* pcaSidPath=(char*)malloc((strlen(sessId->value)+strlen(pmTmpPath->value)+6)*sizeof(char)); + sprintf(pcaSidPath,"%s/%s.sid",pmTmpPath->value,sessId->value); + int iS1=zStat(pcaSidPath, &zssSid_status); map* pmPid=getMapFromMaps(pmsLenv,"lenv","Identifier"); json_object_object_add(res,statusFields[1],json_object_new_string(pmPid->value)); + char *pcaTime=(char*)malloc((TIME_SIZE+1)*sizeof(char)); + size_t st_Time=strftime(pcaTime,TIME_SIZE, zDateFormat, localtime(&zssSid_status.st_mtime) ); + json_object_object_add(res,statusFields[2],json_object_new_string(pcaTime)); + json_object_object_add(res,statusFields[3],json_object_new_string(pcaTime)); + fprintf(stderr,"++++++ %s %d %d %d %d\n",__FILE__,__LINE__,status,status==SERVICE_SUCCEEDED,status==SERVICE_FAILED); + if(status==SERVICE_SUCCEEDED || status==SERVICE_FAILED){ + char* pcaStatusPath=(char*)malloc((strlen(sessId->value)+strlen(pmTmpPath->value)+6)*sizeof(char)); + sprintf(pcaStatusPath,"%s/%s.json",pmTmpPath->value,sessId->value); + zStatStruct zssStatus_status; + int iS2=zStat(pcaStatusPath, &zssStatus_status); + st_Time=strftime(pcaTime,TIME_SIZE, zDateFormat, localtime(&zssStatus_status.st_mtime) ); + json_object_object_add(res,statusFields[4],json_object_new_string(pcaTime)); + json_object_object_add(res,statusFields[5],json_object_new_string(pcaTime)); + }else{ + st_Time=strftime(pcaTime,TIME_SIZE, zDateFormat, localtime(&zssLenv_status.st_mtime) ); + json_object_object_add(res,statusFields[5],json_object_new_string(pcaTime)); + } freeMaps(&pmsLenv); free(pmsLenv); pmsLenv=NULL; @@ -2647,7 +2833,10 @@ extern "C" { else json_object_object_add(cc1,"application/json",cc0); json_object *cc2=json_object_new_object(); - json_object_object_add(cc2,"content",cc1); + if(strcmp(code,"204")!=0) + json_object_object_add(cc2,"content",cc1); + else + json_object_put(cc1); json_object_object_add(cc2,"description",json_object_new_string(msg)); json_object_object_add(res,code,cc2); } @@ -2705,7 +2894,6 @@ extern "C" { } else json_object_object_add(res6,"type",json_object_new_string("string")); - tmpMap=getMap(tmpMaps1->content,"enum"); if(tmpMap!=NULL){ char *saveptr12; @@ -2785,9 +2973,64 @@ extern "C" { } } + + /** + * Add a securityScheme to the OpenAPI components/securitySchemes object + * + * @param pmsConf the maps containing the settings of the main.cfg file + * @param fName the key to add to the res json object + * @param res the json object to store the content + */ + void addSecurityScheme(maps* pmsConf,const char* fName,json_object* res){ + json_object *poJsonObject=json_object_new_object(); + maps* pmsTmp=getMaps(pmsConf,"osecurity"); + // Get type + map* pmTmp=getMap(pmsTmp->content,"type"); + if(pmTmp!=NULL) + json_object_object_add(poJsonObject,"type",json_object_new_string(pmTmp->value)); + // Get scheme if any + pmTmp=getMap(pmsTmp->content,"scheme"); + if(pmTmp!=NULL) + json_object_object_add(poJsonObject,"scheme",json_object_new_string(pmTmp->value)); + // Get format if any + pmTmp=getMap(pmsTmp->content,"format"); + if(pmTmp!=NULL) + json_object_object_add(poJsonObject,"bearerFormat",json_object_new_string(pmTmp->value)); + // Get the in key if any (for type=apiKey) + pmTmp=getMap(pmsTmp->content,"in"); + if(pmTmp!=NULL) + json_object_object_add(poJsonObject,"in",json_object_new_string(pmTmp->value)); + // Get the name of the apiKey + pmTmp=getMap(pmsTmp->content,"oname"); + if(pmTmp!=NULL) + json_object_object_add(poJsonObject,"name",json_object_new_string(pmTmp->value)); + json_object_object_add(res,fName,poJsonObject); + } + + /** + * Produce the JSON object for api securitySchemes + * + * @param conf the maps containing the settings of the main.cfg file + * @param res the JSON object to populate with the parameters + */ + void produceApiSecuritySchemes(maps* conf,json_object* res){ + map* pmTmp=getMapFromMaps(conf,"osecurity","name"); + if(pmTmp!=NULL){ + json_object *res9=json_object_new_object(); + char *saveptr12; + char *tmps12 = strtok_r (pmTmp->value, ",", &saveptr12); + while(tmps12!=NULL){ + addSecurityScheme(conf,tmps12,res9); + tmps12 = strtok_r (NULL, ",", &saveptr12); + } + json_object_object_add(res,"securitySchemes",res9); + } + } + void produceApiComponents(maps*conf,json_object* res){ json_object* res1=json_object_new_object(); produceApiParameters(conf,res1); + produceApiSecuritySchemes(conf,res1); json_object_object_add(res,"components",res1); } @@ -2861,6 +3104,22 @@ extern "C" { map* vMap=getMapArray(tmpMaps->content,"title",i); if(vMap!=NULL) json_object_object_add(methodc,"summary",json_object_new_string(_(vMap->value))); + vMap=getMapArray(tmpMaps->content,"secured",i); + if(vMap!=NULL){ + json_object *poSecurity=json_object_new_array(); + json_object *poScopes=json_object_new_array(); + map* pmSMap=getMapArray(tmpMaps->content,"scopes",i); + char *pcSaveptr; + char *pcTmps = strtok_r (tmpMap->value, ",", &pcSaveptr); + while(pcTmps!=NULL){ + json_object_array_add(poScopes,json_object_new_string(pcTmps)); + pcTmps = strtok_r (NULL, ",", &pcSaveptr); + } + json_object *poSecurityItem=json_object_new_object(); + json_object_object_add(poSecurityItem,vMap->value,poScopes); + json_object_array_add(poSecurity,poSecurityItem); + json_object_object_add(methodc,"security",poSecurity); + } vMap=getMapArray(tmpMaps->content,"abstract",i); if(vMap!=NULL) json_object_object_add(methodc,"description",json_object_new_string(_(vMap->value))); diff --git a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/ulinet.c b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/ulinet.c index c222c13..0397967 100644 --- a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/ulinet.c +++ b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/ulinet.c @@ -501,7 +501,7 @@ HINTERNET InternetOpenUrl(HINTERNET* hInternet,LPCTSTR lpszUrl,LPCTSTR lpszHeade curl_easy_setopt(hInternet->ihandle[hInternet->nb].handle, CURLOPT_VERBOSE, 1); #endif - if(memUse!=NULL && strcasecmp(memUse->value,"load")==0) + if(memUse==NULL || strcasecmp(memUse->value,"load")==0) ldwFlags=INTERNET_FLAG_NO_CACHE_WRITE; switch(ldwFlags) diff --git a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/zoo_service_loader.c b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/zoo_service_loader.c index 1ad3fb5..8832a78 100644 --- a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/zoo_service_loader.c +++ b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/zoo_service_loader.c @@ -220,6 +220,77 @@ translateChar (char *str, char toReplace, char toReplaceBy) } } +/** + * Create PID (SID) file referencing the OS process identifier (service id) + * + * @param pmsConf the conf maps containing the main.cfg settings + * @param pcPath the path to store the sid file + * @param pcUsid the usid value + * @param pcExtension the file extension (sid or pid) + * @param pcValue the value to be stored + */ +int createXidFile(maps* pmsConf,char* pcPath,char* pcUsid,char* pcExtension,char* pcValue){ + char* pcaPath = + (char *) + malloc ((strlen (pcPath) + strlen (pcUsid) + + strlen (pcExtension) + 3) * sizeof (char)); + char* pcaName = (char*) malloc(10*sizeof(char)); + sprintf (pcaPath, "%s/%s.%s", pcPath, pcUsid,pcExtension); + sprintf (pcaName, "file.%s", pcExtension); + setMapInMaps (pmsConf, "lenv", pcaName, pcaPath); + FILE* pfFile = fopen (pcaPath, "w+"); + free(pcaPath); + free(pcaName); + if(pfFile!=NULL){ + fprintf(pfFile,"%s",pcValue); + fflush(pfFile); + }else + return 1; + return 0; +} + +/** + * Create the PID and SID files referencing the OS process identifier and the + * service id, respectively. + * + * @param pmsConf the conf maps containing the main.cfg settings + * @param pcPath the path to store the files + */ +int createSPidFile(maps* pmsConf,char* pcPath){ + char* pcaPid=(char*) malloc(100*sizeof(char)); + map *pmUsid=getMapFromMaps(pmsConf,"lenv","usid"); + map *pmOsid=getMapFromMaps(pmsConf,"lenv","osid"); + sprintf(pcaPid,"%d",zGetpid()); + if(createXidFile(pmsConf,pcPath,pmUsid->value,"sid",pmOsid->value)!=0) + return 1; + if(createXidFile(pmsConf,pcPath,pmUsid->value,"pid",pcaPid)!=0){ + free(pcaPid); + return 1; + } + free(pcaPid); + return 0; +} + +/** + * Create a _lenv.cfg file containing the lenv map + * + * @param pmsConf the conf maps containing the main.cfg settings + * @param pcPath the path to store the sid file + * @param pcUsid the usid value + * @param pcValue the value to be stored + */ +int createLenvFile(maps* pmsConf,char* pcPath,char* pcUsid){ + char *pcaPath = + (char *) + malloc ((strlen (pcPath) + + strlen (pcUsid) + 12) * sizeof (char)); + sprintf (pcaPath, "%s/%s_lenv.cfg", pcPath, pcUsid); + maps* pmsLenv=getMaps(pmsConf,"lenv"); + dumpMapsToFile(pmsLenv,pcaPath,1); + free(pcaPath); + return 0; +} + /** * Dump back the final file fbkp1 to fbkp * @@ -2428,6 +2499,116 @@ loadServiceAndRun (maps ** myMap, service * s1, map * request_inputs, } + +/** + * Set the security flag for the current request. + * Set [lenv] secured_url to true in case the access should be secured, + * false in other cases. + * + * @param pmsConf the maps pointing to the main.cfg file content + * @param pcCgiQueryString the string containing the request + */ +void setSecurityFlags(maps* pmsConf,char* pcCgiQueryString){ + maps* pmsCurrentPath=getMaps(pmsConf,(strlen(pcCgiQueryString)>1?(pcCgiQueryString+1):"root")); + int isSet=-1; + if(pmsCurrentPath==NULL && strstr(pcCgiQueryString,"execution")!=NULL) + pmsCurrentPath=getMaps(pmsConf,"processes/{processId}/execution"); + else if(pmsCurrentPath==NULL && strstr(pcCgiQueryString,"processes/")!=NULL) + pmsCurrentPath=getMaps(pmsConf,"processes/{processId}"); + else if(pmsCurrentPath==NULL && strstr(pcCgiQueryString,"results")!=NULL) + pmsCurrentPath=getMaps(pmsConf,"jobs/{jobID}/results"); + else if(pmsCurrentPath==NULL && strstr(pcCgiQueryString,"jobs/")!=NULL) + pmsCurrentPath=getMaps(pmsConf,"jobs/{jobID}"); + if(pmsCurrentPath!=NULL){ + map* pmLength=getMap(pmsCurrentPath->content,"length"); + int iLength=1; + int iCnt=0; + if(pmLength!=NULL) + iLength=atoi(pmLength->value); + for(;iCntcontent,"method",iCnt); + if(pmCurrentElement!=NULL && strcasecmp(pmCurrentElement->value,cgiRequestMethod)==0){ + map* pmSecured=getMapArray(pmsCurrentPath->content,"secured",iCnt); + if(pmSecured!=NULL) + setMapInMaps(pmsConf,"lenv","secured_url","true"); + else + setMapInMaps(pmsConf,"lenv","secured_url","false"); + return; + } + } + } + if(isSet==-1) + setMapInMaps(pmsConf,"lenv","secured_url","false"); +} + + +/** + * Invoke the execution of the security module in case security is activated + * + * @param pmsConf the maps pointing to the main.cfg file content + * @param pcType the string defining the process to execute ('in' or 'out') + * @return 0 in case of success, 1 in case or error + */ +int ensureFiltered(maps* pmsConf,const char* pcType){ + int eres=0; + service* psaService=NULL; + maps* pmsOsecurity=getMaps(pmsConf,"osecurity"); + map* pmPath=NULL; + map* pmName=NULL; + map* pmRun=NULL; + int iLen=1; + char* pcaName=(char*)malloc((strlen(pcType)+8)*sizeof(char)); + sprintf(pcaName,"filter_%s",pcType); + maps* pmsSection=getMaps(pmsConf,pcaName); + if(pmsSection==NULL){ + free(pcaName); + return 0; + } + map* pmLength=getMap(pmsSection->content,"length"); + if(pmLength!=NULL) + iLen=atoi(pmLength->value); + for(int iCnt=0;iCntcontent,"path",iCnt))!=NULL) && + ((pmName=getMapArray(pmsSection->content,"service",iCnt))!=NULL)){ + if(fetchService(NULL,pmsConf,&psaService,NULL,pmPath->value,pmName->value,printExceptionReportResponseJ)!=0){ + fprintf(stderr,"ERROR fetching the service %s %d \n",__FILE__,__LINE__); + free(pcaName); + return 1; + } + maps* pmsInputs=NULL; + maps* pmsOutputs=NULL; + loadServiceAndRun (&pmsConf, psaService, NULL, + &pmsInputs, + &pmsOutputs, &eres); + freeService(&psaService); + free(psaService); + psaService=NULL; + if(eres!=SERVICE_SUCCEEDED){ + setMapInMaps(pmsConf,"lenv",pcaName,"true"); + free(pcaName); + return 1; + } + } + } + setMapInMaps(pmsConf,"lenv",pcaName,"true"); + free(pcaName); + return 0; +} + +/** + * Invoke the ensureFiltered then printExceptionReportResponseJ functions + * + * @param pmsConf the maps containing the settings of the main.cfg file + * @param pmError the map containing the text,code,locator keys (or a map array) + * @see printExceptionReportResponseJ,ensureFiltered + */ +void localPrintExceptionJ(maps* pmsConf,map* pmError){ + ensureFiltered(pmsConf,"out"); + printExceptionReportResponseJ(pmsConf,pmError); +} + #ifdef WIN32 /** * createProcess function: create a new process after setting some env variables @@ -2922,7 +3103,7 @@ runRequest (map ** inputs) value=url_decode(token1); token1=strtok_r(NULL,"=",&saveptr1); } - addToMapA(request_inputs,name, value != NULL ? value : ""); + addToMapA(request_inputs, name, value != NULL ? value : ""); free(name); free(value); name=NULL; @@ -2937,7 +3118,10 @@ runRequest (map ** inputs) if(pcaCgiQueryString==NULL) pcaCgiQueryString=zStrdup(cgiQueryString); - //dumpMap(request_inputs); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + dumpMap(request_inputs); + fprintf(stderr,"%s %d \n",__FILE__,__LINE__); + r_inputs = getMapOrFill (&request_inputs, "metapath", ""); char conf_file1[10240]; maps* m1 = (maps *) malloc (MAPS_SIZE); @@ -2956,6 +3140,29 @@ runRequest (map ** inputs) freeMaps(&m1); free(m1); + initAllEnvironment(m,request_inputs,ntmp,"jrequest"); + setSecurityFlags(m,pcaCgiQueryString); + // In case security is activated, then execute the security module + if(ensureFiltered(m,"in")!=0){ + maps* pmsTmp=getMaps(m,"lenv"); + localPrintExceptionJ(m,pmsTmp->content); + freeMaps(&m); + free(m); + free (REQUEST); + map* pmTest=getMap(request_inputs,"shouldFree"); + if(pmTest!=NULL){ + freeMap (inputs); + free (*inputs); + *inputs=NULL; + freeMap(&r_inputs); + free (r_inputs); + r_inputs=NULL; + } + free(pcaCgiQueryString); + return 1; + } + setRootUrlMap(m); + /** EOEPCA SPEC **/ // Define specific route for Undeploy/DeployProcess setMapInMaps(m,"lenv","request_method",cgiRequestMethod); @@ -2989,7 +3196,7 @@ runRequest (map ** inputs) map* error=createMap("code","NoSuchProcess"); addToMap(error,"message",_("The process is not yet deployed.")); setMapInMaps(m,"headers","Content-Type","application/json;charset=UTF-8"); - printExceptionReportResponseJ(m,error); + localPrintExceptionJ(m,error); //json_object_put(res); // TODO: cleanup memory return 1; @@ -3021,14 +3228,14 @@ runRequest (map ** inputs) printGetCapabilitiesForProcessJ, (void*) res4, (void*) res3, ntmp, request_inputs, - printExceptionReportResponseJ); + localPrintExceptionJ); zDup2 (saved_stdout, fileno (stdout)); zClose(saved_stdout); json_object *res5=json_object_array_get_idx(res4,0); if(res5==NULL){ map* pmError=createMap("code","NoSuchProcess"); addToMap(pmError,"message",_("The process failed to be updated.")); - printExceptionReportResponseJ(m,pmError); + localPrintExceptionJ(m,pmError); // TODO: cleanup memory json_object_put(res4); json_object_put(res3); @@ -3081,7 +3288,7 @@ runRequest (map ** inputs) setMapInMaps(m,"lenv","status_code","405"); map* error=createMap("code","InvalidMethod"); addToMap(error,"message",_("The request method used to access the current path is not supported.")); - printExceptionReportResponseJ(m,error); + localPrintExceptionJ(m,error); json_object_put(res); // TODO: cleanup memory freeMaps(&m); @@ -3101,7 +3308,7 @@ runRequest (map ** inputs) map* pamError=createMap("code","InvalidMethod"); const char* pccErr=_("This API does not support the method."); addToMap(pamError,"message",pccErr); - printExceptionReportResponseJ(m,pamError); + localPrintExceptionJ(m,pamError); // TODO: cleanup memory return 1; } @@ -3272,7 +3479,7 @@ runRequest (map ** inputs) else{ map* pmIdentifier=getMapFromMaps(m1,"lenv","oIdentifier"); if(pmIdentifier!=NULL){ - if(fetchService(zooRegistry,m,&s1,request_inputs,ntmp,pmIdentifier->value,printExceptionReportResponseJ)!=0){ + if(fetchService(zooRegistry,m,&s1,request_inputs,ntmp,pmIdentifier->value,localPrintExceptionJ)!=0){ register_signals(donothing); freeService (&s1); free(s1); @@ -3299,13 +3506,13 @@ runRequest (map ** inputs) } } free(pcaPath); - initAllEnvironment(m,request_inputs,ntmp,"jrequest"); + //initAllEnvironment(m,request_inputs,ntmp,"jrequest"); setMapInMaps(m,"lenv","callback_request_method","DELETE"); /** EOEPCA SPEC END **/ runDismiss(m,jobId); map* pmError=getMapFromMaps(m,"lenv","error"); if(pmError!=NULL && strncasecmp(pmError->value,"true",4)==0){ - printExceptionReportResponseJ(m,getMapFromMaps(m,"lenv","code")); + localPrintExceptionJ(m,getMapFromMaps(m,"lenv","code")); register_signals(donothing); freeService (&s1); free(s1); @@ -3604,7 +3811,7 @@ runRequest (map ** inputs) runDismiss(m,jobId); map* pmError=getMapFromMaps(m,"lenv","error"); if(pmError!=NULL && strncasecmp(pmError->value,"true",4)==0){ - printExceptionReportResponseJ(m,getMapFromMaps(m,"lenv","code")); + localPrintExceptionJ(m,getMapFromMaps(m,"lenv","code")); freeMaps(&m); free(m); json_object_put(res); @@ -3632,7 +3839,7 @@ runRequest (map ** inputs) if(sid==NULL){ map* error=createMap("code","NoSuchJob"); addToMap(error,"message",_("The JobID from the request does not match any of the Jobs running on this server")); - printExceptionReportResponseJ(m,error); + localPrintExceptionJ(m,error); free(jobId); freeMap(&error); free(error); @@ -3645,7 +3852,7 @@ runRequest (map ** inputs) if(isRunning(m,jobId)>0){ map* error=createMap("code","ResultNotReady"); addToMap(error,"message",_("The job is still running.")); - printExceptionReportResponseJ(m,error); + localPrintExceptionJ(m,error); free(jobId); freeMap(&error); free(error); @@ -3668,7 +3875,7 @@ runRequest (map ** inputs) json_object_object_get_ex(pjoTmp,"description",&pjoMessage)!=FALSE){ map* error=createMap("code",json_object_get_string(pjoCode)); addToMap(error,"message",json_object_get_string(pjoMessage)); - printExceptionReportResponseJ(m,error); + localPrintExceptionJ(m,error); free(jobId); freeMap(&error); free(error); @@ -3714,7 +3921,7 @@ runRequest (map ** inputs) } map* error=createMap("code","NoApplicableCode"); addToMap(error,"message",_("The service failed to execute.")); - printExceptionReportResponseJ(m,error); + localPrintExceptionJ(m,error); free(jobId); freeMap(&error); free(error); @@ -3728,7 +3935,7 @@ runRequest (map ** inputs) }else{ map* error=createMap("code","NoSuchJob"); addToMap(error,"message",_("The JobID seem to be running on this server but not for this process id")); - printExceptionReportResponseJ(m,error); + localPrintExceptionJ(m,error); free(jobId); freeMap(&error); free(error); @@ -3757,7 +3964,7 @@ runRequest (map ** inputs) map* error=createMap("code","BadRequest"); addToMap(error,"message",_("The resource is not available")); //setMapInMaps(conf,"lenv","status_code","404 Bad Request"); - printExceptionReportResponseJ(m,error); + localPrintExceptionJ(m,error); freeMaps (&m); free (m); free (REQUEST); @@ -3780,7 +3987,7 @@ runRequest (map ** inputs) map* error=createMap("code","BadRequest"); addToMap(error,"message",_("The request body is empty")); setMapInMaps(m,"lenv","status_code","400 Bad Request"); - printExceptionReportResponseJ(m,error); + localPrintExceptionJ(m,error); freeMaps (&m); free (m); free (REQUEST); @@ -3812,7 +4019,7 @@ runRequest (map ** inputs) char* pacMessage=(char*)malloc((strlen(pcTmpErr)+strlen(pccErr)+1)*sizeof(char)); sprintf(pacMessage,pccErr,pcTmpErr); addToMap(pamError,"message",pacMessage); - printExceptionReportResponseJ(m,pamError); + localPrintExceptionJ(m,pamError); fprintf(stderr, "Error: %s\n", json_tokener_error_desc(jerr)); json_tokener_free(tok); return 1; @@ -3824,7 +4031,7 @@ runRequest (map ** inputs) char* pacMessage=(char*)malloc((strlen(pcTmpErr)+strlen(pccErr)+1)*sizeof(char)); sprintf(pacMessage,pccErr,pcTmpErr); addToMap(pamError,"message",pacMessage); - printExceptionReportResponseJ(m,pamError); + localPrintExceptionJ(m,pamError); fprintf(stderr, "Error: %s\n", json_tokener_error_desc(jerr)); json_tokener_free(tok); return 1; @@ -3854,7 +4061,7 @@ runRequest (map ** inputs) } } - if(fetchService(zooRegistry,m,&s1,request_inputs,ntmp,cIdentifier,printExceptionReportResponseJ)!=0){ + if(fetchService(zooRegistry,m,&s1,request_inputs,ntmp,cIdentifier,localPrintExceptionJ)!=0){ // TODO: cleanup memory register_signals(donothing); freeService (&s1); @@ -3943,10 +4150,14 @@ runRequest (map ** inputs) //invokeBasicCallback(m,SERVICE_ACCEPTED); setMapInMaps(m,"headers","Status","201 Created"); map* pmTmp=getMapFromMaps(m,"lenv","usid"); + fprintf(stderr,"%s %d \n", __FILE__,__LINE__); + dumpMap(pmTmp); + fprintf(stderr,"%s %d \n", __FILE__,__LINE__); if(pmTmp!=NULL){ if(res!=NULL) json_object_put(res); res=printJobStatus(m,pmTmp->value); + fprintf(stderr,"%s %d \n", __FILE__,__LINE__); } //printHeaders(m); //printf("Status: 201 Created \r\n\r\n"); @@ -3984,19 +4195,25 @@ runRequest (map ** inputs) #endif eres = SERVICE_ACCEPTED; createStatusFile(m,eres); + map* pmUsid = getMapFromMaps (m, "lenv", "usid"); + map* pmOsid = getMapFromMaps (m, "lenv", "osid"); + map* pmTmpPath = getMapFromMaps (m, "main", "tmpPath"); + + createSPidFile(m,pmTmpPath->value); + createLenvFile(m,pmTmpPath->value,pmUsid->value); + maps* pmsLenv=getMaps(m,"lenv"); if(preference!=NULL) setMapInMaps(m,"headers","Preference-Applied",preference->value); - //invokeBasicCallback(m,SERVICE_ACCEPTED); setMapInMaps(m,"headers","Status","201 Created"); - printHeaders(m); - map* pmTmp=getMapFromMaps(m,"lenv","uusid"); + map* pmTmp=getMapFromMaps(m,"lenv","usid"); if(pmTmp!=NULL){ if(res!=NULL) json_object_put(res); + setMapInMaps(m,"lenv","status","Accepted"); + setMapInMaps(m,"lenv","message",_("Initializing")); + res=printJobStatus(m,pmTmp->value); } - //printf("Status: 201 Created \r\n\r\n"); - return 1; } else if (pid == 0) { @@ -4025,30 +4242,8 @@ runRequest (map ** inputs) bmap->content=createMap("usid",usid->value); addToMap(bmap->content,"sid",tmpm->value); addIntToMap(bmap->content,"pid",zGetpid()); - - // Create PID file referencing the OS process identifier - fbkpid = - (char *) - malloc ((strlen (r_inputs->value) + - strlen (usid->value) + 7) * sizeof (char)); - sprintf (fbkpid, "%s/%s.pid", r_inputs->value, usid->value); - setMapInMaps (m, "lenv", "file.pid", fbkpid); - f0 = freopen (fbkpid, "w+",stdout); - printf("%d",zGetpid()); - fflush(stdout); - - // Create SID file referencing the semaphore name - fbkp = - (char *) - malloc ((strlen (r_inputs->value) + strlen (oid->value) + - strlen (usid->value) + 7) * sizeof (char)); - sprintf (fbkp, "%s/%s.sid", r_inputs->value, usid->value); - setMapInMaps (m, "lenv", "file.sid", fbkp); - FILE* f2 = freopen (fbkp, "w+",stdout); - printf("%s",tmpm->value); - fflush(f2); - free(fbkp); + createSPidFile(m,r_inputs->value); fbkp = (char *) @@ -4100,14 +4295,7 @@ runRequest (map ** inputs) if(serviceTypeMap!=NULL) setMapInMaps (m, "lenv", "serviceType", serviceTypeMap->value); - char *flenv = - (char *) - malloc ((strlen (r_inputs->value) + - strlen (usid->value) + 12) * sizeof (char)); - sprintf (flenv, "%s/%s_lenv.cfg", r_inputs->value, usid->value); - maps* lenvMaps=getMaps(m,"lenv"); - dumpMapsToFile(lenvMaps,flenv,1); - free(flenv); + createLenvFile(m,r_inputs->value,usid->value); map* testMap=getMapFromMaps(m,"main","memory"); loadHttpRequests(m,request_input_real_format); @@ -4137,7 +4325,7 @@ runRequest (map ** inputs) if(pmMutable==NULL || strncasecmp(pmMutable->value,"true",4)==0){ map* pmError=createMap("code","None"); addToMap(pmError,"message",_("The synchronous mode is not allowed for mutable services.")); - printExceptionReportResponseJ(m,pmError); + localPrintExceptionJ(m,pmError); freeMap(&pmError); free(pmError); freeMaps(&m); @@ -4186,7 +4374,7 @@ runRequest (map ** inputs) free(pcaFileName); map* pmError=createMap("code","NoSuchProcess"); addToMap(pmError,"message",_("The process failed to be updated.")); - printExceptionReportResponseJ(m,pmError); + localPrintExceptionJ(m,pmError); pmDeployed=NULL; } if(pmDeployed!=NULL){ @@ -4213,7 +4401,7 @@ runRequest (map ** inputs) printGetCapabilitiesForProcessJ, (void*) res4, (void*) res3, ntmp, request_inputs, - printExceptionReportResponseJ); + localPrintExceptionJ); zDup2 (saved_stdout, fileno (stdout)); zClose(saved_stdout); json_object *res5=json_object_array_get_idx(res4,0); @@ -4279,7 +4467,7 @@ runRequest (map ** inputs) printGetCapabilitiesForProcessJ, NULL, (void*) res3, ntmp, request_inputs, - printExceptionReportResponseJ); + localPrintExceptionJ); if(t==1){ /*map* error=createMap("code","BadRequest"); addToMap(error,"message",_("Failed to acces the requested service")); @@ -4292,6 +4480,7 @@ runRequest (map ** inputs) free(pcaCgiQueryString); return 1; } + json_object_put(res); res=json_object_get(res3); free(orig); json_object_put(res3); @@ -4309,7 +4498,7 @@ runRequest (map ** inputs) cnt++; } - fetchService(zooRegistry,m,&s1,request_inputs,ntmp,cIdentifier,printExceptionReportResponseJ); + fetchService(zooRegistry,m,&s1,request_inputs,ntmp,cIdentifier,localPrintExceptionJ); } @@ -4318,11 +4507,13 @@ runRequest (map ** inputs) } } + ensureFiltered(m,"out"); map* pmHasPrinted=getMapFromMaps(m,"lenv","hasPrinted"); if(res!=NULL && (pmHasPrinted==NULL || strncasecmp(pmHasPrinted->value,"false",5)==0)){ if(getMapFromMaps(m,"lenv","no-headers")==NULL){ printHeaders(m); - printf("Status: 200 OK \r\n\r\n"); + if(getMapFromMaps(m,"headers","Status")==NULL) + printf("Status: 200 OK \r\n\r\n"); } const char* jsonStr=json_object_to_json_string_ext(res,JSON_C_TO_STRING_NOSLASHESCAPE); printf(jsonStr); @@ -5005,29 +5196,7 @@ runRequest (map ** inputs) addToMap(bmap->content,"sid",tmpm->value); addIntToMap(bmap->content,"pid",zGetpid()); - // Create PID file referencing the OS process identifier - fbkpid = - (char *) - malloc ((strlen (r_inputs->value) + - strlen (usid->value) + 7) * sizeof (char)); - sprintf (fbkpid, "%s/%s.pid", r_inputs->value, usid->value); - setMapInMaps (m, "lenv", "file.pid", fbkpid); - - f0 = freopen (fbkpid, "w+",stdout); - printf("%d",zGetpid()); - fflush(stdout); - - // Create SID file referencing the semaphore name - fbkp = - (char *) - malloc ((strlen (r_inputs->value) + strlen (r_inputs1->value) + - strlen (usid->value) + 7) * sizeof (char)); - sprintf (fbkp, "%s/%s.sid", r_inputs->value, usid->value); - setMapInMaps (m, "lenv", "file.sid", fbkp); - FILE* f2 = freopen (fbkp, "w+",stdout); - printf("%s",tmpm->value); - fflush(f2); - free(fbkp); + createSPidFile(m,r_inputs->value); fbkp = (char *) @@ -5095,14 +5264,7 @@ runRequest (map ** inputs) if(serviceTypeMap!=NULL) setMapInMaps (m, "lenv", "serviceType", serviceTypeMap->value); - char *flenv = - (char *) - malloc ((strlen (r_inputs->value) + - strlen (usid->value) + 12) * sizeof (char)); - sprintf (flenv, "%s/%s_lenv.cfg", r_inputs->value, usid->value); - maps* lenvMaps=getMaps(m,"lenv"); - dumpMapsToFile(lenvMaps,flenv,1); - free(flenv); + createLenvFile(m,r_inputs->value,usid->value); #ifdef USE_CALLBACK invokeCallback(m,request_input_real_format,NULL,1,0); @@ -5125,7 +5287,9 @@ runRequest (map ** inputs) invokeCallback(m,NULL,NULL,7,0); #endif #endif - zUnlink (fbkpid); + map *pmPid=getMapFromMaps(m,"lenv","file.pid"); + if(pmPid!=NULL) + zUnlink (pmPid->value); unhandleStatus (m); #ifdef RELY_ON_DB #ifdef META_DB @@ -5212,7 +5376,9 @@ runRequest (map ** inputs) if(dumpBackFinalFile(m,fbkp,fbkp1)<0) return -1; - zUnlink (fbkpid); + map *pmPid=getMapFromMaps(m,"lenv","file.pid"); + if(pmPid!=NULL) + zUnlink (pmPid->value); switch(eres){ default: case SERVICE_FAILED: @@ -5539,30 +5705,7 @@ runAsyncRequest (maps** iconf, map ** lenv, map ** irequest_inputs,json_object * addToMap(bmap->content,"sid",tmpm->value); addIntToMap(bmap->content,"pid",zGetpid()); - // Create PID file referencing the OS process identifier - fbkpid = - (char *) - malloc ((strlen (r_inputs->value) + - - strlen (usid->value) + 7) * sizeof (char)); - sprintf (fbkpid, "%s/%s.pid", r_inputs->value, usid->value); - setMapInMaps (lconf, "lenv", "file.pid", fbkpid); - - f0 = freopen (fbkpid, "w+",stdout); - printf("%d",zGetpid()); - fflush(stdout); - - // Create SID file referencing the semaphore name - fbkp = - (char *) - malloc ((strlen (r_inputs->value) + strlen (r_inputs1->value) + - strlen (usid->value) + 7) * sizeof (char)); - sprintf (fbkp, "%s/%s.sid", r_inputs->value, usid->value); - setMapInMaps (lconf, "lenv", "file.sid", fbkp); - FILE* f2 = freopen (fbkp, "w+",stdout); - printf("%s",tmpm->value); - fflush(f2); - free(fbkp); + createSPidFile(m,r_inputs->value); int iNbChars=7; if(pmExecutionType!=NULL && strncasecmp(pmExecutionType->value,"xml",3)==0) @@ -5648,14 +5791,7 @@ runAsyncRequest (maps** iconf, map ** lenv, map ** irequest_inputs,json_object * if(serviceTypeMap!=NULL) setMapInMaps (lconf, "lenv", "serviceType", serviceTypeMap->value); - char *flenv = - (char *) - malloc ((strlen (r_inputs->value) + - strlen (usid->value) + 12) * sizeof (char)); - sprintf (flenv, "%s/%s_lenv.cfg", r_inputs->value, usid->value); - maps* lenvMaps=getMaps(lconf,"lenv"); - dumpMapsToFile(lenvMaps,flenv,1); - free(flenv); + createLenvFile(m,r_inputs->value,usid->value); #ifdef USE_CALLBACK invokeCallback(lconf,request_input_real_format,NULL,1,0); #endif @@ -5677,7 +5813,9 @@ runAsyncRequest (maps** iconf, map ** lenv, map ** irequest_inputs,json_object * invokeCallback(lconf,NULL,NULL,7,0); #endif #endif - zUnlink (fbkpid); + map *pmPid=getMapFromMaps(m,"lenv","file.pid"); + if(pmPid!=NULL) + zUnlink (pmPid->value); unhandleStatus (conf); #ifdef RELY_ON_DB #ifdef META_DB @@ -5752,7 +5890,9 @@ runAsyncRequest (maps** iconf, map ** lenv, map ** irequest_inputs,json_object * if(dumpBackFinalFile(lconf,fbkp,fbkp1)<0) return -1; - zUnlink (fbkpid); + map *pmPid=getMapFromMaps(m,"lenv","file.pid"); + if(pmPid!=NULL) + zUnlink (pmPid->value); switch(eres){ default: case SERVICE_FAILED: diff --git a/Dockerfile b/Dockerfile index b4458ab..12dabf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -117,11 +117,11 @@ RUN wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | t rm -rf /var/lib/{apt,dpkg,cache,log} && \ cp ./micromamba /usr/bin && \ micromamba create -n workflow_executor_env && \ - micromamba install workflow-executor=1.0.41 -c eoepca -c conda-forge -n workflow_executor_env && \ + micromamba install workflow-executor=1.0.40 -c eoepca -c conda-forge -n workflow_executor_env && \ rm -fr /srv/conda/pkgs && \ rm -fr /tmp/* -RUN git clone https://github.com/swagger-api/swagger-ui.git && \ +RUN git clone --depth=1 https://github.com/swagger-api/swagger-ui.git && \ mv swagger-ui /var/www/html/swagger-ui && \ sed "s=https://petstore.swagger.io/v2/swagger.json=http://localhost:8080/ogc-api/api=g" -i /var/www/html/swagger-ui/dist/* && \ mv /var/www/html/swagger-ui/dist /var/www/html/swagger-ui/oapip && \ diff --git a/assets/oas.cfg b/assets/oas.cfg index 5a6c9a7..c03e540 100644 --- a/assets/oas.cfg +++ b/assets/oas.cfg @@ -342,9 +342,9 @@ in=query required=false [conformsTo] -rootUrl=http://www.opengis.net/spec/ogcapi-processes/1.0/req/ +rootUrl=http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ extentionsLength=1 -extentionUrl=http://www.opengis.net/spec/ogcapi-processes-2/1.0/req/ +extentionUrl=http://www.opengis.net/spec/ogcapi-processes-2/1.0/conf/ link=core link_1=oas30 link_2=json @@ -366,28 +366,28 @@ default_schema=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/res [responses] length=5 code=404 -schema=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/NotFound.yaml -eschema=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/NotFound.yaml +eschema=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/NotFound.yaml +schema=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/NotFound.yaml type=application/json title=NotFound code_1=500 -schema_1=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/ServerError.yaml -eschema_1=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/ServerError.yaml +zschema_1=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/ServerError.yaml +schema_1=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/ServerError.yaml type_1=application/json title_1=ServerError code_2=400 -schema_2=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/InvalidParameter.yaml -eschema_2=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/InvalidParameter.yaml +eschema_2=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/InvalidParameter.yaml +schema_2=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/InvalidParameter.yaml type_2=appliction/json title_2=InvalidParameter code_3=405 -schema_3=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/NotAllowed.yaml -eschema_3=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/NotAllowed.yaml +eschema_3=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/NotAllowed.yaml +schema_3=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/NotAllowed.yaml type_3=appliction/json title_3=NotAllowed code_4=406 -schema_3=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/NotSupported.yaml -eschema_4=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/NotSupported.yaml +eschema_3=https://raw.githubusercontent.com/GeoLabs/ogcapi-processes/master/core/openapi/responses/NotSupported.yaml +schema_4=http://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/responses/NotSupported.yaml type_4=appliction/json title_4=NotSupported From acb95e61be176e72d9d4c70633656b6012778ca1 Mon Sep 17 00:00:00 2001 From: bbrauzzi Date: Wed, 5 Jul 2023 14:34:34 +0200 Subject: [PATCH 2/7] gitignore update --- .gitignore | 1 + charts/ades/values.minikube.yaml | 238 +++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 charts/ades/values.minikube.yaml diff --git a/.gitignore b/.gitignore index f3f2dc9..9eb22ef 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,7 @@ charts/ades/values.ades2.eoepcacreodiasdev.yaml charts/ades/values.eoepca-dev-ades1.terradue.yaml charts/ades/values.wasdi.yaml charts/ades/values.eoepca-dev-eoepca.terradue.yaml +charts/ades/values.minikube.yaml # workflow executor test not yet ready 3ty/workflow_executor/workflow_executor/workflow_executor/test diff --git a/charts/ades/values.minikube.yaml b/charts/ades/values.minikube.yaml new file mode 100644 index 0000000..809a8f2 --- /dev/null +++ b/charts/ades/values.minikube.yaml @@ -0,0 +1,238 @@ +# Default values for ades. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: eoepca/proc-ades + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "2.0.17" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} +# fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true +# runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: ades.eoepca.com + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: +# We usually recommend not to specify default resources and to leave this as a conscious +# choice for the user. This also increases chances charts run on environments with little +# resources, such as Minikube. If you do want to specify resources, uncomment the following +# lines, adjust them as necessary, and remove the curly braces after 'resources:'. +# limits: +# cpu: 2 +# memory: 4Gi +# requests: +# cpu: 1 +# memory: 2Gi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +clusterAdminRoleName: cluster-admin + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +useKubeProxy: true + +workflowExecutor: + # Necessary if useKubeProxy set to false + kubeconfig: "" + + # Here specify fixed inputs to all workflows execution in all stages (main, stage-in/out) + # They will be prefixed with 'ADES_'. e.g. 'APP: ades' will be 'ADES_APP: ades' + inputs: + APP: ades + + #STAGEOUT_AWS_SERVICEURL: http://my-minio-fs:9000 + STAGEOUT_AWS_SERVICEURL: https://minio.develop.eoepca.org + STAGEOUT_AWS_ACCESS_KEY_ID: user-6UDdrhHD1BiwSw + STAGEOUT_AWS_SECRET_ACCESS_KEY: ntdXuKg4coQhl-zq7oyPyw + STAGEOUT_AWS_REGION: RegionOne + STAGEOUT_OUTPUT: s3://develop-user-eric + + #STAGEIN_AWS_SERVICEURL: https://mys3repositoryendpoint.com + #STAGEIN_AWS_ACCESS_KEY_ID: myAccesKeyId + #STAGEIN_AWS_SECRET_ACCESS_KEY: mySecretAccessKey + + + + stageout: + cwl: | + cwlVersion: v1.0 + id: stageout + baseCommand: ['/bin/bash', 'stageout.sh'] + doc: "Staging results with aws cli s3" + class: CommandLineTool + hints: + DockerRequirement: + dockerPull: eoepca/aws-cli:1.0.0 + "cwltool:Secrets": + secrets: + - ADES_STAGEOUT_AWS_SERVICEURL + - ADES_STAGEOUT_AWS_REGION + - ADES_STAGEOUT_AWS_ACCESS_KEY_ID + - ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY + inputs: + ADES_STAGEOUT_AWS_PROFILE: + type: string? + ADES_STAGEOUT_AWS_SERVICEURL: + type: string? + ADES_STAGEOUT_AWS_ACCESS_KEY_ID: + type: string? + ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY: + type: string? + aws_profiles_location: + type: File? + ADES_STAGEOUT_OUTPUT: + type: string? + ADES_STAGEOUT_AWS_REGION: + type: string? + process: + type: string? + outputs: + s3_catalog_output: + outputBinding: + outputEval: ${ return inputs.ADES_STAGEOUT_OUTPUT + "/" + inputs.process + "/catalog.json"; } + type: string + requirements: + InitialWorkDirRequirement: + listing: + - entryname: stageout.sh + entry: |- + #!/bin/bash + export AWS_DEFAULT_REGION=$(inputs.ADES_STAGEOUT_AWS_REGION) + export AWS_ACCESS_KEY_ID=$(inputs.ADES_STAGEOUT_AWS_ACCESS_KEY_ID) + export AWS_SECRET_ACCESS_KEY=$(inputs.ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY) + ${ + if( !Array.isArray(inputs.wf_outputs) ) + { + aws --endpoint-url=$(inputs.ADES_STAGEOUT_AWS_SERVICEURL) s3 sync $(inputs.wf_outputs.path) $(inputs.ADES_STAGEOUT_OUTPUT)/$(inputs.process) + } + var args=[]; + for (var i = 0; i < inputs.wf_outputs.length; i++) + { + aws --endpoint-url=$(inputs.ADES_STAGEOUT_AWS_SERVICEURL) s3 sync $(inputs.wf_outputs[i].path) $(inputs.ADES_STAGEOUT_OUTPUT)/$(inputs.process) + } + return args; + } + + InlineJavascriptRequirement: {} + EnvVarRequirement: + envDef: + PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + ResourceRequirement: {} + + rulez: + cwl: "" + + # kubernetes storage class to be used for provisioning volumes. Must be a persistent volume claim compliant (longhorn) + processingStorageClass: standard + # Size of the Kubernetes Tmp Volumes + processingVolumeTmpSize: "5Gi" + # Size of the Kubernetes Output Volumes + processingVolumeOutputSize: "10Gi" + # Max ram to use for a job + processingMaxRam: "16Gi" + # Max number of CPU cores to use concurrently for a job + processingMaxCores: "8" + # if false the Ades will clean the volume after the workflow has successfully finished running + processingKeepWorkspace: false + # if false the Ades will clean the volume after the workflow has finished with an error + processingKeepWorkspaceIfFailed: false + # processing pods node selector: + #processingNodeSelector: {} + # includes terradue.docker.com docker credentials + imagePullSecrets: [] + # Calrissian image tag + calrissianImage: "terradue/calrissian:0.12.0" + # some configuration values for submitted pod + pod: + env: {} + # HTTP_PROXY: http://1.2.3.4:8534 + + useResourceManager: false + resourceManagerEndpoint: "https://resourcemanager-api.com" + resourceManagerWorkspacePrefix: "rm-user" + + # adds a label to the job namespace + jobNamespaceLabels: + app: "ades-app" + + # Number of retries before considering a Job as failed + backofflimit: 3 + +wps: + pepBaseUrl: "https://pep.eoepca.terradue.com" + pepClientStatePath: "/opt/zooservices_user/pepclient" + usePep: "false" + maincfgtpl: "files/main.cfg.tpl" + + +persistence: + enabled: true + # existingUserDataClaim: + # existingProcServicesClaim: + storageClass: standard + userDataAccessMode: ReadWriteOnce + userDataSize: 10Gi + procServicesAccessMode: ReadWriteOnce + procServicesSize: 5Gi + +# installs longhorn storageClass using ades-longhorn chart +ades-longhorn: + enabled: false + persistence: + defaultClassReplicaCount: 1 + #defaultSettings: + # systemManagedComponentsNodeSelector: "longhorn:yes" From f90e0e71ae4ab108547c03bdb0c5f759bceb5aef Mon Sep 17 00:00:00 2001 From: bbrauzzi Date: Wed, 5 Jul 2023 14:36:35 +0200 Subject: [PATCH 3/7] version bump 2.0.18 --- build.yml | 2 +- charts/ades/values.minikube.yaml | 238 ------------------------------- 2 files changed, 1 insertion(+), 239 deletions(-) delete mode 100644 charts/ades/values.minikube.yaml diff --git a/build.yml b/build.yml index 100617f..f334254 100644 --- a/build.yml +++ b/build.yml @@ -1,2 +1,2 @@ docker_image_name: eoepca/proc-ades -docker_image_version: 2.0.17 \ No newline at end of file +docker_image_version: 2.0.18 \ No newline at end of file diff --git a/charts/ades/values.minikube.yaml b/charts/ades/values.minikube.yaml deleted file mode 100644 index 809a8f2..0000000 --- a/charts/ades/values.minikube.yaml +++ /dev/null @@ -1,238 +0,0 @@ -# Default values for ades. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: eoepca/proc-ades - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "2.0.17" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} -# fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true -# runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: ades.eoepca.com - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: -# We usually recommend not to specify default resources and to leave this as a conscious -# choice for the user. This also increases chances charts run on environments with little -# resources, such as Minikube. If you do want to specify resources, uncomment the following -# lines, adjust them as necessary, and remove the curly braces after 'resources:'. -# limits: -# cpu: 2 -# memory: 4Gi -# requests: -# cpu: 1 -# memory: 2Gi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 3 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -clusterAdminRoleName: cluster-admin - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -useKubeProxy: true - -workflowExecutor: - # Necessary if useKubeProxy set to false - kubeconfig: "" - - # Here specify fixed inputs to all workflows execution in all stages (main, stage-in/out) - # They will be prefixed with 'ADES_'. e.g. 'APP: ades' will be 'ADES_APP: ades' - inputs: - APP: ades - - #STAGEOUT_AWS_SERVICEURL: http://my-minio-fs:9000 - STAGEOUT_AWS_SERVICEURL: https://minio.develop.eoepca.org - STAGEOUT_AWS_ACCESS_KEY_ID: user-6UDdrhHD1BiwSw - STAGEOUT_AWS_SECRET_ACCESS_KEY: ntdXuKg4coQhl-zq7oyPyw - STAGEOUT_AWS_REGION: RegionOne - STAGEOUT_OUTPUT: s3://develop-user-eric - - #STAGEIN_AWS_SERVICEURL: https://mys3repositoryendpoint.com - #STAGEIN_AWS_ACCESS_KEY_ID: myAccesKeyId - #STAGEIN_AWS_SECRET_ACCESS_KEY: mySecretAccessKey - - - - stageout: - cwl: | - cwlVersion: v1.0 - id: stageout - baseCommand: ['/bin/bash', 'stageout.sh'] - doc: "Staging results with aws cli s3" - class: CommandLineTool - hints: - DockerRequirement: - dockerPull: eoepca/aws-cli:1.0.0 - "cwltool:Secrets": - secrets: - - ADES_STAGEOUT_AWS_SERVICEURL - - ADES_STAGEOUT_AWS_REGION - - ADES_STAGEOUT_AWS_ACCESS_KEY_ID - - ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY - inputs: - ADES_STAGEOUT_AWS_PROFILE: - type: string? - ADES_STAGEOUT_AWS_SERVICEURL: - type: string? - ADES_STAGEOUT_AWS_ACCESS_KEY_ID: - type: string? - ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY: - type: string? - aws_profiles_location: - type: File? - ADES_STAGEOUT_OUTPUT: - type: string? - ADES_STAGEOUT_AWS_REGION: - type: string? - process: - type: string? - outputs: - s3_catalog_output: - outputBinding: - outputEval: ${ return inputs.ADES_STAGEOUT_OUTPUT + "/" + inputs.process + "/catalog.json"; } - type: string - requirements: - InitialWorkDirRequirement: - listing: - - entryname: stageout.sh - entry: |- - #!/bin/bash - export AWS_DEFAULT_REGION=$(inputs.ADES_STAGEOUT_AWS_REGION) - export AWS_ACCESS_KEY_ID=$(inputs.ADES_STAGEOUT_AWS_ACCESS_KEY_ID) - export AWS_SECRET_ACCESS_KEY=$(inputs.ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY) - ${ - if( !Array.isArray(inputs.wf_outputs) ) - { - aws --endpoint-url=$(inputs.ADES_STAGEOUT_AWS_SERVICEURL) s3 sync $(inputs.wf_outputs.path) $(inputs.ADES_STAGEOUT_OUTPUT)/$(inputs.process) - } - var args=[]; - for (var i = 0; i < inputs.wf_outputs.length; i++) - { - aws --endpoint-url=$(inputs.ADES_STAGEOUT_AWS_SERVICEURL) s3 sync $(inputs.wf_outputs[i].path) $(inputs.ADES_STAGEOUT_OUTPUT)/$(inputs.process) - } - return args; - } - - InlineJavascriptRequirement: {} - EnvVarRequirement: - envDef: - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - ResourceRequirement: {} - - rulez: - cwl: "" - - # kubernetes storage class to be used for provisioning volumes. Must be a persistent volume claim compliant (longhorn) - processingStorageClass: standard - # Size of the Kubernetes Tmp Volumes - processingVolumeTmpSize: "5Gi" - # Size of the Kubernetes Output Volumes - processingVolumeOutputSize: "10Gi" - # Max ram to use for a job - processingMaxRam: "16Gi" - # Max number of CPU cores to use concurrently for a job - processingMaxCores: "8" - # if false the Ades will clean the volume after the workflow has successfully finished running - processingKeepWorkspace: false - # if false the Ades will clean the volume after the workflow has finished with an error - processingKeepWorkspaceIfFailed: false - # processing pods node selector: - #processingNodeSelector: {} - # includes terradue.docker.com docker credentials - imagePullSecrets: [] - # Calrissian image tag - calrissianImage: "terradue/calrissian:0.12.0" - # some configuration values for submitted pod - pod: - env: {} - # HTTP_PROXY: http://1.2.3.4:8534 - - useResourceManager: false - resourceManagerEndpoint: "https://resourcemanager-api.com" - resourceManagerWorkspacePrefix: "rm-user" - - # adds a label to the job namespace - jobNamespaceLabels: - app: "ades-app" - - # Number of retries before considering a Job as failed - backofflimit: 3 - -wps: - pepBaseUrl: "https://pep.eoepca.terradue.com" - pepClientStatePath: "/opt/zooservices_user/pepclient" - usePep: "false" - maincfgtpl: "files/main.cfg.tpl" - - -persistence: - enabled: true - # existingUserDataClaim: - # existingProcServicesClaim: - storageClass: standard - userDataAccessMode: ReadWriteOnce - userDataSize: 10Gi - procServicesAccessMode: ReadWriteOnce - procServicesSize: 5Gi - -# installs longhorn storageClass using ades-longhorn chart -ades-longhorn: - enabled: false - persistence: - defaultClassReplicaCount: 1 - #defaultSettings: - # systemManagedComponentsNodeSelector: "longhorn:yes" From f30d545e1a93d4c09cf1df3c6797d64fa1749baa Mon Sep 17 00:00:00 2001 From: gfenoy Date: Fri, 7 Jul 2023 15:38:53 +0200 Subject: [PATCH 4/7] Fix issue when no limit has been defined --- .../assets/patch/zoo/zoo_service_loader.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/zoo_service_loader.c b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/zoo_service_loader.c index 8832a78..d262398 100644 --- a/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/zoo_service_loader.c +++ b/3ty/proc-comm-zoo-1.2-alpha/assets/patch/zoo/zoo_service_loader.c @@ -386,6 +386,7 @@ int addServicesNamespaceToMap(maps* conf){ char *namespaceFolder = (char *) malloc(1024); memset(namespaceFolder, '\0', 1024); map *servicesNamespaceParentFolder = getMapFromMaps(conf, "servicesNamespace", "path"); + if(servicesNamespaceParentFolder!=NULL){ sprintf(namespaceFolder, "%s/%s", servicesNamespaceParentFolder->value, namespaceName); DIR *dir = opendir(namespaceFolder); if (dir) { @@ -412,6 +413,7 @@ int addServicesNamespaceToMap(maps* conf){ freeMap(&error); free(error); } +} } } } From 563c98567f5d716df987236b449ce8e0840d8d72 Mon Sep 17 00:00:00 2001 From: bbrauzzi Date: Mon, 17 Jul 2023 17:28:44 +0200 Subject: [PATCH 5/7] getAuthorizationBearer method bug fix --- src/deployundeploy/zoo/deploy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deployundeploy/zoo/deploy.cpp b/src/deployundeploy/zoo/deploy.cpp index 2ae5b34..fa0974f 100644 --- a/src/deployundeploy/zoo/deploy.cpp +++ b/src/deployundeploy/zoo/deploy.cpp @@ -238,7 +238,7 @@ std::string getAuthorizationBearer(maps *&conf){ char* baseS=strchr(userServicePathMap->value,' '); if (baseS){ fprintf(stderr,"getAuthorizationBearer >%s<\n",++baseS); - return std::string(++baseS); + return std::string(baseS); } } } From 7e41fd5b8af63abb1c7d4f215d3c96850de21f1e Mon Sep 17 00:00:00 2001 From: bbrauzzi Date: Mon, 17 Jul 2023 17:29:42 +0200 Subject: [PATCH 6/7] workflow-executor update 1.0.49 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3246995..ab64203 100644 --- a/Dockerfile +++ b/Dockerfile @@ -117,7 +117,7 @@ RUN wget -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | t rm -rf /var/lib/{apt,dpkg,cache,log} && \ cp ./micromamba /usr/bin && \ micromamba create -n workflow_executor_env && \ - micromamba install workflow-executor=1.0.48 -c eoepca -c conda-forge -n workflow_executor_env && \ + micromamba install workflow-executor=1.0.49 -c eoepca -c conda-forge -n workflow_executor_env && \ rm -fr /srv/conda/pkgs && \ rm -fr /tmp/* From 2db8707e4653c119b1f382ebf252c50492c2c55c Mon Sep 17 00:00:00 2001 From: bbrauzzi Date: Mon, 17 Jul 2023 17:37:10 +0200 Subject: [PATCH 7/7] version bump 2.0.19 --- build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.yml b/build.yml index f334254..728ffa3 100644 --- a/build.yml +++ b/build.yml @@ -1,2 +1,2 @@ docker_image_name: eoepca/proc-ades -docker_image_version: 2.0.18 \ No newline at end of file +docker_image_version: 2.0.19 \ No newline at end of file