Skip to content

Commit

Permalink
Add CSV export of stimuli data
Browse files Browse the repository at this point in the history
  • Loading branch information
franksn90 committed May 2, 2024
1 parent 3ba3a89 commit b36e131
Show file tree
Hide file tree
Showing 6 changed files with 113 additions and 45 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import cambio.monitoring.mosim.StimuliSearchOrchestrator
import cambio.monitoring.mosim.analysis.DefaultMetricsAnalyzer
import cambio.monitoring.mosim.config.SearchConfiguration
import cambio.monitoring.mosim.evaluation.DefaultEvaluator
import cambio.monitoring.mosim.export.ConsoleExporter
import cambio.monitoring.mosim.export.CSVFileExporter
import cambio.monitoring.mosim.import.CSVDataImporter
import cambio.monitoring.mosim.import.DefaultDataSplitter
import cambio.monitoring.mosim.import.DefaultStimuliParser
Expand All @@ -13,12 +13,11 @@ import cambio.monitoring.mosim.search.DefaultSearchExecutor
import cambio.monitoring.mosim.search.DefaultSearchInitializer
import com.google.common.collect.Multimap
import org.springframework.stereotype.Service
import java.nio.file.Path

@Service
class SearchRunningService {

fun runSearch(inputFiles: Multimap<String, String>, outPutDir: Path /* TODO: Export */) {
fun runSearch(inputFiles: Multimap<String, String>, id: String) {
val monitoringDataPathCollection = inputFiles["monitoring-data"]
val mtlPathCollection = inputFiles["mtl"]
if (monitoringDataPathCollection.isEmpty()) {
Expand All @@ -29,7 +28,7 @@ class SearchRunningService {
val mtlPath = mtlPathCollection.iterator().next()
val monitoringDataPath = monitoringDataPathCollection.iterator().next()

val config = SearchConfiguration()
val config = SearchConfiguration(id = id)

val orchestrator = StimuliSearchOrchestrator(
DefaultMetricsAnalyzer(),
Expand All @@ -38,7 +37,7 @@ class SearchRunningService {
DefaultSearchExecutor(),
DefaultStimuliParser(),
DefaultEvaluator(config),
ConsoleExporter()
CSVFileExporter(monitoringDataPath, config)
)

orchestrator.search(CSVDataImporter(monitoringDataPath), FileStimuliImporter(mtlPath, config))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import java.nio.file.Path
@RestController
class SimulationRunningController @Autowired constructor(private var simulationRunningService: SearchRunningService) {

var logger = LoggerFactory.getLogger(SimulationRunningController::class.java)
private var logger = LoggerFactory.getLogger(SimulationRunningController::class.java)

@PostMapping("/search/upload")
@Throws(IOException::class)
Expand Down Expand Up @@ -49,21 +49,18 @@ class SimulationRunningController @Autowired constructor(private var simulationR
}

val tmpFolder = prepareTmpFolder()
val outputFolder = prepareOutputFolder(id)
prepareOutputFolder(id)
val savedFiles = prepareFiles(mtls, monitoringData, tmpFolder)

simulationRunningService.runSearch(savedFiles, outputFolder)
simulationRunningService.runSearch(savedFiles, id)
if (!TempFileUtils.existsSimulationId(id)) {
return ResponseEntity(
String.format(
"An Error happened when running the search with the ID: " + "%s.", id
), HttpStatus.INTERNAL_SERVER_ERROR
)
}

export(outputFolder)
cleanUp(tmpFolder)

ResponseEntity<String?>(
"Files have been successfully uploaded, and the search is running.", HttpStatus.OK
)
Expand All @@ -75,7 +72,7 @@ class SimulationRunningController @Autowired constructor(private var simulationR
}

private fun prepareOutputFolder(id: String): Path {
return TempFileUtils.createOutputDir(TempFileUtils.RAW_OUTPUT_DIR, id)
return TempFileUtils.createOutputDir(TempFileUtils.OUTPUT_DIR, id)
}

private fun prepareTmpFolder(): Path {
Expand All @@ -96,9 +93,4 @@ class SimulationRunningController @Autowired constructor(private var simulationR
FileUtils.deleteDirectory(folder.toFile())
}

private fun export(outputFolder: Path) {
val rawResultsDirPath = (outputFolder.toString() + TempFileUtils.SEPARATOR) + "raw"
// TODO: implement
}

}
33 changes: 8 additions & 25 deletions src/main/kotlin/cambio/monitoring/mosim/api/util/TempFileUtils.kt
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,13 @@ import java.io.IOException
import java.nio.file.FileSystems
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.util.*
import java.util.function.Consumer
import java.util.stream.Collectors

class TempFileUtils {
companion object {

val RAW_OUTPUT_DIR = "raw_search_results"
val OUTPUT_DIR = "search_results"
val SEPARATOR = FileSystems.getDefault().separator
const val OUTPUT_DIR = "search_results"
private val SEPARATOR: String = FileSystems.getDefault().separator

@Throws(
IOException::
Expand Down Expand Up @@ -58,7 +54,7 @@ class TempFileUtils {
temDir: Path
): Multimap<String, String> {
if (files != null) {
Arrays.asList(*files).forEach(Consumer { file: MultipartFile? ->
listOf(*files).forEach(Consumer { file: MultipartFile? ->
try {
val tmpFile = saveFile(file!!, temDir)
val filePath = tmpFile.toString()
Expand All @@ -83,29 +79,16 @@ class TempFileUtils {
if (!Files.exists(outPutDirPath)) {
Files.createDirectory(outPutDirPath)
}
val simulationOutputDirPath = outputDirName + SEPARATOR + simulationId
return Files.createDirectory(Path.of(simulationOutputDirPath))
val simulationOutputDirPath = getOutputDir(outputDirName, simulationId)
return Files.createDirectory(simulationOutputDirPath)
}


@Throws(IOException::class)
fun getFilesFromResultsDir(dirPath: Path): Set<String> {
val dir = dirPath.toString()
Files.list(Paths.get(dir)).use { stream ->
return stream
.filter { file: Path? ->
!Files.isDirectory(
file
)
}
.map { obj: Path -> obj.fileName }
.map { obj: Path -> obj.toString() }
.collect(Collectors.toSet())
}
fun getOutputDir(outputDirName: String, simulationId: String): Path {
return Path.of(outputDirName + SEPARATOR + simulationId)
}

fun existsSimulationId(simulationId: String): Boolean {
val simulationOutputDirPath = RAW_OUTPUT_DIR + SEPARATOR + simulationId
val simulationOutputDirPath = OUTPUT_DIR + SEPARATOR + simulationId
return Files.exists(Path.of(simulationOutputDirPath))
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package cambio.monitoring.mosim.config

import kotlin.random.Random

/**
* The configuration of the overall search.
*/
data class SearchConfiguration(
val searchWindowSize: Double = 5.0,
val searchInterval: Double = 1.0,
val isDebugOn: Boolean = true
val isDebugOn: Boolean = true,
val id: String = Random.nextInt().toString()
)
91 changes: 91 additions & 0 deletions src/main/kotlin/cambio/monitoring/mosim/export/CSVFileExporter.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package cambio.monitoring.mosim.export

import cambio.monitoring.mosim.api.util.TempFileUtils
import cambio.monitoring.mosim.config.SearchConfiguration
import cambio.tltea.parser.core.temporal.TimeInstance
import java.io.BufferedReader
import java.io.BufferedWriter
import java.io.File
import java.io.FileReader

class CSVFileExporter(private val monitoringCSVLoc: String, private val config: SearchConfiguration) : Exporter {
private val columnSeparator: String = ","

private data class MonitoringData(
val headers: String,
val times: MutableList<Double> = mutableListOf(),
val data: MutableList<String> = mutableListOf()
)

override fun export(occurrences: List<Pair<TimeInstance, TimeInstance>>) {
val monitoringData = readMonitoringData()
for (occurrence in occurrences.withIndex()) {
val startValue = occurrence.value.first.time
val endValue = occurrence.value.second.time
val (startIndex, endIndex) = findIndices(startValue, endValue, monitoringData)
val writer = prepareOutputFile(occurrence.index)
writeMonitoringData(startIndex, endIndex, monitoringData, writer)
}
}

private fun readMonitoringData(): MonitoringData {
val br = BufferedReader(FileReader(monitoringCSVLoc))
var line = br.readLine()
val monitoringData = MonitoringData(line)
line = br.readLine()
while (line != null) {
val valueInLine = line.split(columnSeparator)
val time = valueInLine[0].toDouble()
monitoringData.times.add(time)
monitoringData.data.add(line)
line = br.readLine()
}
return monitoringData
}

private fun findIndices(startValue: Double, endValue: Double, monitoringData: MonitoringData): Pair<Int, Int> {
var startIndex = -1
var endIndex = 0
for (time in monitoringData.times.withIndex()) {
if (!isIndexSet(startIndex) && time.value >= startValue) {
startIndex = time.index
}
if (isIndexSet(startIndex)) {
if (time.value <= endValue) {
endIndex = time.index
} else {
break
}
}
}
return Pair(startIndex, endIndex)
}

private fun isIndexSet(index: Int): Boolean {
return index >= 0
}

private fun prepareOutputFile(occurrenceId: Int): BufferedWriter {
val outputDir = TempFileUtils.getOutputDir(TempFileUtils.OUTPUT_DIR, config.id)
val outputFile = File(outputDir.toFile(), "$occurrenceId.csv")
outputFile.createNewFile()
return outputFile.bufferedWriter()
}

private fun writeMonitoringData(
startIndex: Int,
endIndex: Int,
monitoringData: MonitoringData,
writer: BufferedWriter
) {
writer.write(monitoringData.headers)
writer.newLine()
val endIndexInclusive = (endIndex + 1).coerceAtMost(monitoringData.data.size)
for (line in monitoringData.data.subList(startIndex, endIndexInclusive)) {
writer.write(line)
writer.newLine()
}
writer.close()
}

}
4 changes: 2 additions & 2 deletions src/test/kotlin/Test.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import cambio.monitoring.mosim.StimuliSearchOrchestrator
import cambio.monitoring.mosim.analysis.DefaultMetricsAnalyzer
import cambio.monitoring.mosim.config.SearchConfiguration
import cambio.monitoring.mosim.evaluation.DefaultEvaluator
import cambio.monitoring.mosim.export.ConsoleExporter
import cambio.monitoring.mosim.export.CSVFileExporter
import cambio.monitoring.mosim.import.CSVDataImporter
import cambio.monitoring.mosim.import.DefaultDataSplitter
import cambio.monitoring.mosim.import.DefaultStimuliParser
Expand All @@ -29,7 +29,7 @@ class Test {
DefaultSearchExecutor(),
DefaultStimuliParser(),
DefaultEvaluator(config),
ConsoleExporter()
CSVFileExporter(monitoringCSVLoc, config)
)

orchestrator.search(CSVDataImporter(monitoringCSVLoc), FileStimuliImporter(mtlLoc, config))
Expand Down

0 comments on commit b36e131

Please sign in to comment.