diff --git a/.eslintrc.json b/.eslintrc.json index 615db55..a6404f3 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -3,7 +3,7 @@ "rules": { "no-console": ["error", { "allow": ["warn", "error"] }], "no-unused-vars": [ - "warn", + "error", { "ignoreRestSiblings": true, "varsIgnorePattern": "_+", "argsIgnorePattern": "^_" } ], "semi": ["error", "never"] diff --git a/package-lock.json b/package-lock.json index 4b49f8a..c1d5b2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "@tabler/icons-react": "^3.19.0", "@tanstack/react-query": "^5.59.0", "@types/jsonwebtoken": "^9.0.7", + "@types/papaparse": "^5.3.15", "@types/uuid": "^10.0.0", "@vanilla-extract/css": "^1.16", "@vanilla-extract/next-plugin": "^2.4", @@ -44,6 +45,7 @@ "mantine-form-zod-resolver": "^1.1.0", "mantine-react-table": "^2.0.0-beta.6", "next-swagger-doc": "^0.4", + "papaparse": "^5.4.1", "pg": "^8.13.0", "react-icons": "^5.3.0", "remeda": "^2.14.0", @@ -6420,6 +6422,15 @@ "undici-types": "~6.19.2" } }, + "node_modules/@types/papaparse": { + "version": "5.3.15", + "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.15.tgz", + "integrity": "sha512-JHe6vF6x/8Z85nCX4yFdDslN11d+1pr12E526X8WAfhadOeaOTx5AuIkvDKIBopfvlzpzkdMx4YyvSKCM9oqtw==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/pg": { "version": "8.11.10", "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.11.10.tgz", @@ -12511,6 +12522,12 @@ "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", "license": "BlueOak-1.0.0" }, + "node_modules/papaparse": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", + "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==", + "license": "MIT" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/package.json b/package.json index c79fea7..349e83f 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "@tabler/icons-react": "^3.19.0", "@tanstack/react-query": "^5.59.0", "@types/jsonwebtoken": "^9.0.7", + "@types/papaparse": "^5.3.15", "@types/uuid": "^10.0.0", "@vanilla-extract/css": "^1.16", "@vanilla-extract/next-plugin": "^2.4", @@ -88,6 +89,7 @@ "mantine-form-zod-resolver": "^1.1.0", "mantine-react-table": "^2.0.0-beta.6", "next-swagger-doc": "^0.4", + "papaparse": "^5.4.1", "pg": "^8.13.0", "react-icons": "^5.3.0", "remeda": "^2.14.0", diff --git a/src/app/dl/results/[...slug]/route.ts b/src/app/dl/results/[...slug]/route.ts index 63eda2d..0cabc20 100644 --- a/src/app/dl/results/[...slug]/route.ts +++ b/src/app/dl/results/[...slug]/route.ts @@ -1,8 +1,8 @@ import { b64toUUID } from '@/lib/uuid' import { NextResponse } from 'next/server' -import { db } from '@/database' import { urlOrPathToResultsFile } from '@/server/results' import { MinimalRunResultsInfo } from '@/lib/types' +import { queryRunResult } from '@/server/queries' export const GET = async ( _: Request, @@ -15,26 +15,13 @@ export const GET = async ( const runId = b64toUUID(runIdentifier) // TODO: check if the run is owned by the researcher - const run = await db - .selectFrom('studyRun') - .innerJoin('study', 'study.id', 'studyRun.studyId') - .innerJoin('member', 'study.memberId', 'member.id') - .select(['studyRun.id as studyRunId', 'studyId', 'resultsPath', 'member.identifier as memberIdentifier']) - .where('studyRun.id', '=', runId) - .where('studyRun.status', '=', 'COMPLETED') - .where('studyRun.resultsPath', 'is not', null) - .executeTakeFirst() + const run = await queryRunResult(runId) if (!run) { return NextResponse.json({ error: 'run not found', runId }, { status: 404 }) } - if (!run.resultsPath) { - return NextResponse.json({ error: 'Results not available yet' }, { status: 404 }) - } - const location = await urlOrPathToResultsFile(run as MinimalRunResultsInfo) - if (location.content) { return new NextResponse(location.content) } else if (location.url) { diff --git a/src/app/researcher/study/[encodedStudyId]/review/actions.ts b/src/app/researcher/study/[encodedStudyId]/review/actions.ts index fe8b66a..2d94a1d 100644 --- a/src/app/researcher/study/[encodedStudyId]/review/actions.ts +++ b/src/app/researcher/study/[encodedStudyId]/review/actions.ts @@ -1,12 +1,15 @@ 'use server' +import { promises as fs } from 'fs' import { db } from '@/database' import { StudyStatus } from '@/database/types' -import { uuidToB64, b64toUUID } from '@/lib/uuid' +import { uuidToB64 } from '@/lib/uuid' import { revalidatePath } from 'next/cache' -import { attachSimulatedResultsToStudyRun } from '@/server/results' +import { attachSimulatedResultsToStudyRun, storageForResultsFile } from '@/server/results' import { sleep } from '@/lib/util' import { SIMULATE_RESULTS_UPLOAD, USING_CONTAINER_REGISTRY } from '@/server/config' +import { queryRunResult } from '@/server/queries' +import { fetchStudyRunResults } from '@/server/aws' const AllowedStatusChanges: Array = ['APPROVED', 'REJECTED'] as const @@ -42,27 +45,6 @@ export const onFetchStudyRunsAction = async (studyId: string) => { return runs } -export const getLatestStudyRunAction = async ({ encodedStudyId }: { encodedStudyId: string }) => { - return await db - .selectFrom('study') - .innerJoin('member', 'member.id', 'study.memberId') - .select([ - 'study.id', - 'study.title', - 'study.containerLocation', - 'member.name as memberName', - ({ selectFrom }) => - selectFrom('studyRun') - .whereRef('study.id', '=', 'studyRun.studyId') - .select('id as runId') - .orderBy('study.createdAt desc') - .limit(1) - .as('pendingRunId'), - ]) - .where('study.id', '=', uuidToB64(encodedStudyId)) - .executeTakeFirst() -} - export const onStudyRunCreateAction = async (studyId: string) => { const studyRun = await db .insertInto('studyRun') @@ -91,3 +73,23 @@ export const onStudyRunCreateAction = async (studyId: string) => { return studyRun.id } + +export const fetchRunResultsAction = async (runId: string) => { + const run = await queryRunResult(runId) + if (!run) { + throw new Error(`Run ${runId} not found or does not have results`) + } + const storage = await storageForResultsFile(run) + let csv = '' + if (storage.s3) { + const body = await fetchStudyRunResults(run) + // TODO: handle other types of results that are not string/CSV + csv = await body.transformToString('utf-8') + } + + if (storage.file) { + csv = await fs.readFile(storage.file, 'utf-8') + } + + return csv +} diff --git a/src/app/researcher/study/[encodedStudyId]/review/page.tsx b/src/app/researcher/study/[encodedStudyId]/review/page.tsx index 9b5eddb..20f13b9 100644 --- a/src/app/researcher/study/[encodedStudyId]/review/page.tsx +++ b/src/app/researcher/study/[encodedStudyId]/review/page.tsx @@ -1,4 +1,4 @@ -import { Paper, Center, Title, Stack, Group, Button } from '@mantine/core' +import { Paper, Center, Title, Stack, Group } from '@mantine/core' import { db } from '@/database' import { uuidToB64 } from '@/lib/uuid' import { StudyPanel } from './panel' @@ -29,13 +29,6 @@ export default async function StudyReviewPage({ {study.title} - {/* - - - - - - */} diff --git a/src/app/researcher/study/[encodedStudyId]/review/panel.tsx b/src/app/researcher/study/[encodedStudyId]/review/panel.tsx index 77d6b40..1c99bb8 100644 --- a/src/app/researcher/study/[encodedStudyId]/review/panel.tsx +++ b/src/app/researcher/study/[encodedStudyId]/review/panel.tsx @@ -3,7 +3,7 @@ import React from 'react' import { useState } from 'react' import { useMutation } from '@tanstack/react-query' -import { Button, Group, Accordion, Stack, Text, Flex, TextInput, Textarea, Checkbox } from '@mantine/core' +import { Group, Accordion, Stack, Text, Flex, TextInput, Textarea, Checkbox } from '@mantine/core' import { labelStyle, inputStyle } from './style.css' import { ErrorAlert } from '@/components/errors' import { useRouter } from 'next/navigation' @@ -22,11 +22,7 @@ export const StudyPanel: React.FC<{ encodedStudyId: string; study: Study; studyI const backPath = `/researcher/studies/review` - const { - mutate: updateStudy, - isPending, - error, - } = useMutation({ + const { error } = useMutation({ mutationFn: (status: StudyStatus) => updateStudyStatusAction(study?.id || '', status), onSettled(error) { if (!error) { diff --git a/src/app/researcher/study/[encodedStudyId]/review/results.tsx b/src/app/researcher/study/[encodedStudyId]/review/results.tsx new file mode 100644 index 0000000..66b8e3a --- /dev/null +++ b/src/app/researcher/study/[encodedStudyId]/review/results.tsx @@ -0,0 +1,90 @@ +'use client' + +import { FC } from 'react' +import Link from 'next/link' +import { Button, LoadingOverlay, Modal, ScrollArea } from '@mantine/core' +import { useDisclosure } from '@mantine/hooks' +import { useQuery } from '@tanstack/react-query' +import { uuidToB64 } from '@/lib/uuid' +import Papa from 'papaparse' +import { DataTable } from 'mantine-datatable' +import { fetchRunResultsAction } from './actions' +import { ErrorAlert } from '@/components/errors' +import { IconDownload } from '@tabler/icons-react' + +type RunResultsProps = { + run: { id: string } +} + +const ViewCSV: FC = ({ run }) => { + const { + data: csv, + isLoading, + isError, + error, + } = useQuery({ + queryKey: ['run-results', run.id], + queryFn: async () => { + const csv = await fetchRunResultsAction(run.id) + return Papa.parse>(csv, { + header: true, + complete: (results) => { + results.data.forEach((row, i) => { + row['INDEX_KEY_FOR_RENDERING'] = i + }) + return results + }, + }) + }, + }) + + if (isError) { + return + } + + if (isLoading || !csv) { + return + } + + return ( + ({ accessor: f }))} + /> + ) +} + +export const PreviewCSVResultsBtn: FC = ({ run }) => { + const [opened, { open, close }] = useDisclosure(false) + + return ( + <> + + + + } + scrollAreaComponent={ScrollArea.Autosize} + overlayProps={{ + backgroundOpacity: 0.55, + blur: 3, + }} + centered + > + + + + + + ) +} diff --git a/src/app/researcher/study/[encodedStudyId]/review/runs-table.tsx b/src/app/researcher/study/[encodedStudyId]/review/runs-table.tsx index 7bdb15a..a1b4a56 100644 --- a/src/app/researcher/study/[encodedStudyId]/review/runs-table.tsx +++ b/src/app/researcher/study/[encodedStudyId]/review/runs-table.tsx @@ -1,6 +1,6 @@ 'use client' -import { useState, useEffect } from 'react' +import { useState } from 'react' import { Table, Accordion, @@ -9,20 +9,18 @@ import { AccordionItem, Button, Modal, - Text, Group, Center, } from '@mantine/core' import { useDisclosure } from '@mantine/hooks' +import { PushInstructions } from '@/components/push-instructions' import { IconPlus } from '@tabler/icons-react' import { useQuery, useQueryClient, useMutation } from '@tanstack/react-query' -import Link from 'next/link' -import { uuidToB64 } from '@/lib/uuid' import { onFetchStudyRunsAction } from './actions' import { humanizeStatus } from '@/lib/status' import { AlertNotFound } from '@/components/errors' -import { PushInstructions } from '@/components/push-instructions' -import { getLatestStudyRunAction, onStudyRunCreateAction } from './actions' +import { onStudyRunCreateAction } from './actions' +import { PreviewCSVResultsBtn } from './results' export type Study = { id: string @@ -33,17 +31,16 @@ export type Study = { } type RunsTableProps = { - // studyIdentifier: string encodedStudyId: string isActive: boolean study: Study } -const RunsTable: React.FC = ({ encodedStudyId, isActive, study }) => { +const RunsTable: React.FC = ({ isActive, study }) => { const queryClient = useQueryClient() - const [viewingRunId, setViewingRunId] = useState(null) + const [__, setViewingRunId] = useState(null) - const { mutate: insertRun, error: insertError } = useMutation({ + const { mutate: insertRun } = useMutation({ mutationFn: () => onStudyRunCreateAction(study.id), onSuccess: async (runId) => { setViewingRunId(runId) @@ -56,26 +53,6 @@ const RunsTable: React.FC = ({ encodedStudyId, isActive, study } enabled: isActive, queryFn: () => onFetchStudyRunsAction(study.id), }) - encodedStudyId = uuidToB64(study.id) - const [run, setRun] = useState<{ - id: string - title: string - containerLocation: string - memberName: string - pendingRunId: string | null - } | null>(null) - - useEffect(() => { - const fetchRun = async () => { - if (encodedStudyId) { - const latestRun = await getLatestStudyRunAction({ encodedStudyId }) - if (latestRun) { - setRun(latestRun) - } - } - } - fetchRun() - }, [encodedStudyId]) const [opened, { open, close }] = useDisclosure(false) @@ -101,21 +78,20 @@ const RunsTable: React.FC = ({ encodedStudyId, isActive, study } {run.startedAt?.toISOString() || ''} - {run.status != 'INITIATED' && ( - <> - + + {run.status == 'INITIATED' && ( + <> - Instructions will go here! - {/* */} + - - - - - - - )} + + )} + {run.status == 'COMPLETED' && } + ))} diff --git a/src/server/aws-ecr-policy.ts b/src/server/aws-ecr-policy.ts index b6a0399..24b84ba 100644 --- a/src/server/aws-ecr-policy.ts +++ b/src/server/aws-ecr-policy.ts @@ -1,29 +1,47 @@ import { type AwsIAMPolicy, AwsEcrActions as Action } from 'aws-iam-policy-types' import { ENCLAVE_AWS_ACCOUNT_NUMBERS } from './config' -export const EcrPolicy: AwsIAMPolicy = { +export const getECRPolicy = (awsAccountId: string): AwsIAMPolicy => ({ Version: '2012-10-17', Statement: [ { - Condition: { - StringEquals: { - 'ecr:ResourceTag/Target': 'si:analysis', - }, - }, Action: [ Action.BatchCheckLayerAvailability, Action.BatchGetImage, Action.DescribeImages, + Action.GetDownloadUrlForLayer, Action.GetAuthorizationToken, Action.ListTagsForResource, Action.GetDownloadUrlForLayer, ], Principal: { - Service: ['ecs-tasks.amazonaws.com'], - AWS: ENCLAVE_AWS_ACCOUNT_NUMBERS.map((acct) => `arn:aws:iam::${acct}:root`), + Service: ['ecs-tasks.amazonaws.com', 'lambda.amazonaws.com'], + AWS: ENCLAVE_AWS_ACCOUNT_NUMBERS.map((acct) => `arn:aws:iam::${acct}:root`).concat( + `arn:aws:iam::${awsAccountId}:root`, + ), }, Effect: 'Allow', Sid: 'AllowEnclaveECSTaskToPullImages', }, + { + Action: [ + Action.BatchCheckLayerAvailability, + Action.BatchGetImage, + Action.DescribeImages, + Action.GetDownloadUrlForLayer, + Action.GetAuthorizationToken, + Action.ListTagsForResource, + + Action.InitiateLayerUpload, + Action.UploadLayerPart, + Action.CompleteLayerUpload, + Action.PutImage, + ], + Principal: { + AWS: `arn:aws:iam::${awsAccountId}:root`, + }, + Effect: 'Allow', + Sid: 'AllowUsersToUploadPullImages', + }, ], -} +}) diff --git a/src/server/aws.ts b/src/server/aws.ts index eaea880..f863ad9 100644 --- a/src/server/aws.ts +++ b/src/server/aws.ts @@ -10,7 +10,7 @@ import { uuidToB64 } from '@/lib/uuid' import { Readable } from 'stream' import { createHash } from 'crypto' import { CodeManifest, MinimalRunInfo, MinimalRunResultsInfo } from '@/lib/types' -import { EcrPolicy } from './aws-ecr-policy' +import { getECRPolicy } from './aws-ecr-policy' const DEFAULT_TAGS: Record = { Environment: 'sandbox', @@ -66,6 +66,7 @@ export const getAWSInfo = async () => { export async function createAnalysisRepository(repositoryName: string, tags: Record = {}) { const ecrClient = getECRClient() + const { accountId } = await getAWSInfo() const resp = await ecrClient.send( new CreateRepositoryCommand({ repositoryName, @@ -75,11 +76,12 @@ export async function createAnalysisRepository(repositoryName: string, tags: Rec if (!resp?.repository?.repositoryUri) { throw new Error('Failed to create repository') } + await ecrClient.send( new SetRepositoryPolicyCommand({ repositoryName, registryId: resp.repository.registryId, - policyText: JSON.stringify(EcrPolicy), + policyText: JSON.stringify(getECRPolicy(accountId)), }), ) return resp.repository.repositoryUri @@ -148,3 +150,10 @@ export async function urlForResults(info: MinimalRunResultsInfo) { }) return url } + +export async function fetchStudyRunResults(info: MinimalRunResultsInfo) { + const path = pathForStudyRunResults(info) + const result = await getS3Client().send(new GetObjectCommand({ Bucket: s3BucketName(), Key: path })) + if (!result.Body) throw new Error(`no file recieved from s3 for run result ${info.studyRunId}`) + return result.Body +} diff --git a/src/server/config.ts b/src/server/config.ts index 6f7db3f..e446dfc 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -16,7 +16,7 @@ export const SIMULATE_RESULTS_UPLOAD = process.env.SIMULATE_RESULTS_UPLOAD === 't' || (process.env.SIMULATE_RESULTS_UPLOAD != 'f' && DEV_ENV) export const ENCLAVE_AWS_ACCOUNT_NUMBERS = [ - '337909745635', //prod + '337909745635', // prod '536697261124', // staging '084375557107', // dev '354918363956', // sandbox diff --git a/src/server/queries.ts b/src/server/queries.ts new file mode 100644 index 0000000..e391df3 --- /dev/null +++ b/src/server/queries.ts @@ -0,0 +1,13 @@ +import { db } from '@/database' +import { MinimalRunResultsInfo } from '@/lib/types' + +export const queryRunResult = async (runId: string) => + (await db + .selectFrom('studyRun') + .innerJoin('study', 'study.id', 'studyRun.studyId') + .innerJoin('member', 'study.memberId', 'member.id') + .select(['studyRun.id as studyRunId', 'studyId', 'resultsPath', 'member.identifier as memberIdentifier']) + .where('studyRun.id', '=', runId) + .where('studyRun.status', '=', 'COMPLETED') + .where('studyRun.resultsPath', 'is not', null) + .executeTakeFirst()) as MinimalRunResultsInfo | undefined diff --git a/src/server/results.ts b/src/server/results.ts index 12201a1..4d97be3 100644 --- a/src/server/results.ts +++ b/src/server/results.ts @@ -46,13 +46,21 @@ export async function attachSimulatedResultsToStudyRun(info: MinimalRunInfo) { await attachResultsToStudyRun(info, file) } -export async function urlOrPathToResultsFile(info: MinimalRunResultsInfo) { +export async function storageForResultsFile(info: MinimalRunResultsInfo) { if (USING_S3_STORAGE) { - return { url: await urlForResults(info) } + return { s3: true } } else { - const filePath = path.join(getUploadTmpDirectory(), pathForStudyRun(info), 'results', info.resultsPath) - return { - content: await fs.promises.readFile(filePath, 'utf-8'), - } + return { file: path.join(getUploadTmpDirectory(), pathForStudyRun(info), 'results', info.resultsPath) } + } +} + +export async function urlOrPathToResultsFile(info: MinimalRunResultsInfo) { + const storage = await storageForResultsFile(info) + if (storage.s3) { + return { url: await urlForResults(info) } + } + if (storage.file) { + return { content: await fs.promises.readFile(storage.file, 'utf-8') } } + throw new Error(`unknown storage type for results file ${JSON.stringify(storage)}`) }