recodex archiver script

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+	"deno.enable": true,
+	"deno.unstable": true
+}

README.md

@@ -29,3 +29,9 @@ anyexec2c -b tools/memtest.c -t C# > memtest.cs
 # or a Python program :)
 anyexec2c -b tools/memtest.c -t python > memtest.py
 ```
+
+
+## Other utilities
+
+* [`tools/memtest.c`](tools/memtest.c) - will measure the memory limit. The exit code is the number of blocks allocated successfully. If your environment doesn't allow C, we also have Python version, or you can obviously compile it using anyexec2c to any supported target (like C#)
+* [`recodex-utils/archiver.ts`](recodex-utils) - script to download all your submissions from ReCodex

recodex-utils/.gitignore

@@ -0,0 +1,2 @@
+# default output of the archiver script
+recodex-archive

recodex-utils/README.md

@@ -0,0 +1,26 @@
+## ReCodex API utilities
+
+
+This folder contains Deno/web JS client for the recodex API, and some utilities
+
+### `archiver.ts`
+
+Usage
+
+```
+./archiver.ts --token YOUR_BEARER_TOKEN --path recodex-archive
+```
+
+It will download all your submissions from all courses into the `recodex-archive` directory.
+
+The token can be obtained by catching some of the ReCodex's requests in DevTools / Network tab.
+ReCodex also supports generating some API tokens, maybe it will also work, I don't know.
+
+You'll need to install Deno to use it.
+
+
+### `api.ts`
+
+This is a JS module with some ReCodex API functions exposed, currently only functions needed for the archiver are implemented.
+Note that ReCodex is open source and the frontend is also written in JS, so they should have a more comprehensive SDK.
+This one was just easier to write than finding + understanding + using the existing implementation.

recodex-utils/api.ts

@@ -0,0 +1,239 @@
+// deno-lint-ignore-file no-explicit-any
+
+
+export function createApi({ token, baseUrl }: { token: string, baseUrl?: string }) {
+	baseUrl ??= 'https://recodex.mff.cuni.cz/'
+	baseUrl = baseUrl.replace(/\/*$/, '/')
+
+	const defaultHeaders = { 'Authorization': 'Bearer ' + token }
+
+	function listArchivedCourses(): Promise<RecodexResponse<RecodexCourse[]>> {
+		return fetch(`${baseUrl}api/v1/groups?ancestors=1&archived=1`, { headers: defaultHeaders })
+			.then(res => res.json())
+	}
+
+	function listAssignments(courseId: string): Promise<RecodexResponse<RecodexAssignment[]>> {
+		return fetch(`${baseUrl}api/v1/groups/${courseId}/assignments`, { headers: defaultHeaders })
+			.then(res => res.json())
+	}
+
+	function courseStats(courseId: string): Promise<RecodexResponse<RecodexCourseStats[]>> {
+		return fetch(`${baseUrl}api/v1/groups/${courseId}/students/stats`, { headers: defaultHeaders })
+			.then(res => res.json())
+	}
+	function getUsers(userIds: string[]): Promise<RecodexResponse<RecodexUserInfo[]>> {
+		return fetch(`${baseUrl}api/v1/users/list`, { headers: { ...defaultHeaders, 'Content-Type': "application/json" }, method: "POST", body: JSON.stringify({ ids: userIds }) })
+			.then(res => res.json())
+	}
+
+	function assignmentSubmissions(assignmentId: string, userId: string): Promise<RecodexResponse<AssignmentSubmission[]>> {
+		return fetch(`${baseUrl}api/v1/exercise-assignments/${assignmentId}/users/${userId}/solutions`, { headers: defaultHeaders })
+			.then(res => res.json())
+	}
+
+	function getSubmissionFiles(submissionId: string): Promise<RecodexResponse<SubmissionFile[]>> {
+		return fetch(`${baseUrl}api/v1/assignment-solutions/${submissionId}/files`, { headers: defaultHeaders })
+			.then(res => res.json())
+	}
+
+	function getFileContent(fileId: string): Promise<Blob> {
+		return fetch(`${baseUrl}api/v1/uploaded-files/${fileId}/download`, { headers: defaultHeaders })
+			.then(res => res.blob())
+	}
+
+
+	return {
+		listArchivedCourses,
+		listAssignments,
+		courseStats,
+		getUsers,
+		assignmentSubmissions,
+		getSubmissionFiles,
+		getFileContent,
+	}
+}
+
+type UnixTime = number
+
+export type RecodexResponse<T> = {
+	code: number
+	success: boolean
+	payload: T
+}
+
+export type LocalizedText<T> = {
+	id: string
+	locale: string
+	createdAt: string
+} & T
+
+export type RecodexCourse = {
+	/** Use for `listAssignments(this.id)` */
+	id: string
+	externalId: string
+	organizational: boolean
+	archived: boolean
+	public: boolean
+	directlyArchived: boolean
+	localizedTexts: LocalizedText<{ name: string, description: string }>[]
+	primaryAdminsIds: string[]
+	parentGroupId: string
+	parentGroupIds: string[]
+	childGroups: string[]
+	permissionHints: RecodexCoursePermissionHints
+	/** WTF, it's public anyways... */
+	privateData: CoursePrivateData
+}
+
+export type CoursePrivateData = {
+	admins: string[]
+	supervisors: string[]
+	observers: string[]
+	students: string[]
+	instance: boolean
+	assignments: string[]
+	shadowAssignments: string[]
+	publicStats: boolean
+	detaining: boolean
+	bindings: {
+		/** Seznam rozvrhových lístků, např 20aNPFL129x02 */
+		sis?: string[]
+	}
+}
+
+export type RecodexCoursePermissionHints = {
+	viewAssignments: boolean
+	viewDetail: boolean
+	viewSubgroups: boolean
+	viewStudents: boolean
+	viewMembers: boolean
+	inviteStudents: boolean
+	viewStats: boolean
+	addSubgroup: boolean
+	update: boolean
+	remove: boolean
+	archive: boolean
+	relocate: boolean
+	viewExercises: boolean
+	assignExercise: boolean
+	createExercise: boolean
+	createShadowAssignment: boolean
+	viewPublicDetail: boolean
+	becomeMember: boolean
+	sendEmail: boolean
+	viewInvitations: boolean
+	acceptInvitation: boolean
+	editInvitations: boolean
+}
+
+export type RecodexAssignment = {
+	id: string
+	version: number
+	isPublic: boolean
+	createdAt: UnixTime
+	updatedAt: UnixTime
+	localizedTexts: LocalizedText<{ name: string, text: string, link: string, studentHint: string }>[]
+	exerciseId: string
+	groupId: string
+	firstDeadline: UnixTime
+	secondDeadline: UnixTime
+	allowSecondDeadline: boolean
+	maxPointsBeforeFirstDeadline: number
+	maxPointsBeforeSecondDeadline: number
+	maxPointsDeadlineInterpolation: false
+	/** TODO: ? */
+	visibleFrom: any
+	submissionsCountLimit: number
+	runtimeEnvironmentIds: string[]
+	disabledRuntimeEnvironmentIds: string[]
+	canViewLimitRatios: boolean, canViewJudgeStdout: boolean, canViewJudgeStderr: boolean, mergeJudgeLogs: boolean
+	isBonus: boolean
+	pointsPercentualThreshold: number
+	solutionFilesLimit: null
+	solutionSizeLimit: null
+	exerciseSynchronizationInfo: any // probably useless
+}
+
+export type RecodexCourseStats = {
+	userId: string
+	groupId: string
+	points: { total: number, gained: number }
+	hasLimit: boolean
+	passesLimit: boolean
+	assignments: {
+		id: string
+		status: "done" | null
+		points: { total: number, gained: number | null, bonus: number | null }
+		bestSolutionId: string | null
+		accepted: boolean
+	}[]
+}
+
+export type AssignmentSubmission = {
+	id: string
+	/** From 1 */
+	attemptIndex: number
+	node: string
+	assignmentId: string
+	userId: string
+	createdAt: UnixTime
+	runtimeEnvironmentId: string
+	maxPoints: number
+	accepted: boolean
+	reviewed: boolean
+	isBestSolution: boolean
+	actualPoints: number
+	bonusPoints: number
+	overriddenPoints: number | null
+	submissions: string[]
+	lastSubmission: {
+		id: string
+		assignmentSolutionId: string
+		evaluationStatus: "done" | "failed" | "TODO"
+		isCorrect: boolean
+		submittedBy: string
+		submittedAt: UnixTime
+		isDebug: boolean
+
+		evaluation: {
+			id: string
+			evaluatedAt: UnixTime
+			score: number
+			points: number
+			initFailed: boolean
+			/** Compiler output */
+			initiationOutputs: string
+			testResults: {
+				id: number
+				testName: string
+				score: number
+				// TODO
+			}[]
+		}
+
+
+	}
+	commentStats: any // TODO?
+	permissionHints: any
+}
+
+export type SubmissionFile = {
+	id: string
+	name: string
+	size: number
+	uploadedAt: UnixTime
+	userId: string
+	isPublic: boolean
+}
+
+export type RecodexUserInfo = {
+	id: string
+	fullName: string
+	avatarUrl: string | null
+	isVerified: boolean
+	/** Normally null, only visible for the current user (unless you are admin or something) */
+	privateData: null | {
+		email: string
+		createdAt: UnixTime
+	}
+}

recodex-utils/archiver.ts

@@ -0,0 +1,93 @@
+#!/usr/bin/env -S deno run --allow-net --allow-env --allow-write --allow-read
+import { parse } from "https://deno.land/[email protected]/flags/mod.ts";
+import { createApi, RecodexResponse } from "./api.ts";
+
+const flags = parse(Deno.args, {
+	boolean: [],
+	string: ["token", "path"],
+	default: { path: "recodex-archive" },
+});
+
+
+const token = flags.token
+
+if (!token) {
+	console.error("Missing --token BEARER_TOKEN")
+	console.error("You can get it from the network tab of your browser when you are logged in to ReCodex, or by looking for \"jwt\": in ReCodex HTML.")
+	Deno.exit(1)
+}
+
+function unwrap<T>(msg: string, res: RecodexResponse<T>): T {
+	if (!res.success) {
+		console.error(`API call ${msg} failed:`, res)
+		Deno.exit(1)
+	}
+	return res.payload
+}
+
+const api = createApi({ token })
+
+const courses = unwrap("List courses", await api.listArchivedCourses())
+
+for (const course of courses) {
+	if (!course.permissionHints.viewAssignments) {
+		console.log(`Skipping ${course.externalId}: ${course.localizedTexts[0].name}, no permission to view assignments`)
+		continue
+	}
+	console.log(`Listing ${course.externalId}: ${course.localizedTexts[0].name}`)
+	const courseDir = `${flags.path}/${course.externalId ?? course.localizedTexts[0]?.name ?? course.id}`
+
+	const assignments = unwrap("List assignments", await api.listAssignments(course.id))
+	const stats = unwrap("Get course stats", await api.courseStats(course.id))
+	const courseUsers = new Map(unwrap("Get users", await api.getUsers(stats.map(s => s.userId)))
+		.map(u => [u.id, u] as [string, typeof u]))
+
+	await Deno.mkdir(courseDir, { recursive: true })
+	await Deno.writeTextFile(courseDir + "/assignments.json", JSON.stringify(assignments, null, '\t'))
+	await Deno.writeTextFile(courseDir + "/users.json", JSON.stringify(courseUsers, null, '\t'))
+	await Deno.writeTextFile(courseDir + "/stats.json", JSON.stringify(stats, null, '\t'))
+
+
+	const statsByAId = new Map(
+		stats.filter(s => courseUsers.get(s.userId)?.privateData != null)
+			 .flatMap(s => s.assignments.map(a => [a.id, { userId: s.userId, ...a}] as [string, typeof a & { userId: string }])))
+	for (const assignment of assignments) {
+		const stat = statsByAId.get(assignment.id)
+		if (!stat || stat.status == null) {
+			// skip assignments without submissions
+			continue
+		}
+
+
+		const assignmentDir = courseDir + "/" + assignment.localizedTexts[0].name
+
+		await Deno.mkdir(assignmentDir, { recursive: true })
+		await Deno.writeTextFile(assignmentDir + "/assignment.json", JSON.stringify(assignment, null, '\t'))
+
+		const submissions = unwrap(
+			`List submissions of ${assignment.localizedTexts[0].name} [${assignment.id}]`,
+			await api.assignmentSubmissions(assignment.id, stat.userId))
+
+		console.log(`  Loading ${course.externalId} / ${assignment.localizedTexts[0].name} (${submissions.length} submissions)`)
+
+		await Deno.writeTextFile(assignmentDir + "/submissions.json", JSON.stringify(submissions, null, '\t'))
+
+		for (const submission of submissions) {
+			const submissionDir = assignmentDir + "/" + submission.attemptIndex
+			await Deno.mkdir(submissionDir, { recursive: true })
+
+			await Deno.writeTextFile(submissionDir + "/submission.json", JSON.stringify(submission, null, '\t'))
+
+			const files = unwrap(
+				`List files of ${assignment.localizedTexts[0].name} [${assignment.id}] submission #${submission.attemptIndex}`,
+				await api.getSubmissionFiles(submission.id))
+
+			for (const file of files) {
+				const content = await api.getFileContent(file.id)
+
+				await Deno.writeFile(submissionDir + "/" + file.name, new Uint8Array(await content.arrayBuffer()))
+			}
+		}
+	}
+}
+