Skip to content

Commit

Permalink
feat: quickjs - state in Memory
Browse files Browse the repository at this point in the history
  • Loading branch information
@ppedziwiatr
ppedziwiatr committed Mar 29, 2024
1 parent 8a67881 commit 9e22b89
Show file tree
Hide file tree
Showing 9 changed files with 324 additions and 323 deletions.
2 changes: 1 addition & 1 deletion warp-contracts-plugin-quickjs/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
},
"dependencies": {
"quickjs-emscripten": "^0.27.0",
"warp-contracts": "1.4.36-ao.3"
"warp-contracts": "1.4.36-ao.12"
},
"files": [
"build/"
Expand Down
61 changes: 13 additions & 48 deletions warp-contracts-plugin-quickjs/src/QuickJsHandlerApi.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { QuickJSContext, QuickJSHandle, QuickJSRuntime, QuickJSWASMModule } from 'quickjs-emscripten';
import { AoInteractionResult, InteractionResult, LoggerFactory, QuickJsPluginMessage } from 'warp-contracts';
import { errorEvalAndDispose, joinBuffers } from './utils';
import { DELIMITER, VARIANT_TYPE } from '.';
import { errorEvalAndDispose } from './utils';

export class QuickJsHandlerApi<State> {
private readonly logger = LoggerFactory.INST.create('QuickJsHandlerApi');
Expand All @@ -10,26 +9,21 @@ export class QuickJsHandlerApi<State> {
private readonly vm: QuickJSContext,
private readonly runtime: QuickJSRuntime,
private readonly quickJS: QuickJSWASMModule,
private readonly wasmMemory?: Buffer,
private readonly compress?: boolean
) {}

async handle<Result>(message: QuickJsPluginMessage): Promise<InteractionResult<State, Result>> {
if (!this.wasmMemory) {
this.init(message);
async handle<Result>(message: QuickJsPluginMessage, state?: State): Promise<InteractionResult<State, Result>> {
if (state) {
this.initState(state);
}

return await this.runContractFunction(message);
return this.runContractFunction(message);
}

initState(state: State): void {
if (!this.wasmMemory) {
const initStateResult = this.vm.evalCode(`__initState(${JSON.stringify(state)})`);
if (initStateResult.error) {
errorEvalAndDispose('initState', this.logger, this.vm, initStateResult.error);
} else {
initStateResult.value.dispose();
}
const initStateResult = this.vm.evalCode(`__initState(${JSON.stringify(state)})`);
if (initStateResult.error) {
errorEvalAndDispose('initState', this.logger, this.vm, initStateResult.error);
} else {
initStateResult.value.dispose();
}
}

Expand All @@ -41,7 +35,7 @@ export class QuickJsHandlerApi<State> {
} else {
const result: AoInteractionResult<Result> = this.disposeResult(evalInteractionResult);
return {
Memory: await this.getWasmMemory(),
Memory: await this.currentState(),
Error: '',
Messages: result.Messages,
Spawns: result.Spawns,
Expand All @@ -52,15 +46,15 @@ export class QuickJsHandlerApi<State> {
} catch (err: any) {
if (err.name.includes('ProcessError')) {
return {
Memory: await this.getWasmMemory(),
Memory: await this.currentState(),
Error: `${err.message} ${JSON.stringify(err.stack)}`,
Messages: null,
Spawns: null,
Output: null
};
} else {
return {
Memory: await this.getWasmMemory(),
Memory: await this.currentState(),
Error: `${(err && JSON.stringify(err.stack)) || (err && err.message) || err}`,
Messages: null,
Spawns: null,
Expand Down Expand Up @@ -145,33 +139,4 @@ export class QuickJsHandlerApi<State> {
}
}

private async getWasmMemory() {
let wasmMemoryBuffer: ArrayBuffer;
wasmMemoryBuffer = this.quickJS.getWasmMemory().buffer;

const headers = {
variantType: VARIANT_TYPE,
// @ts-ignore
vmPointer: this.vm.ctx.value,
// @ts-ignore
runtimePointer: this.vm.rt.value,
compressed: this.compress
};
if (this.compress) {
const compressionStream = new CompressionStream('gzip');
const uint8WasmMemoryBuffer = new Uint8Array(wasmMemoryBuffer);
const stream = new ReadableStream({
start(controller) {
controller.enqueue(uint8WasmMemoryBuffer);
controller.close();
}
});
const compressedStream = stream.pipeThrough(compressionStream);
wasmMemoryBuffer = await new Response(compressedStream).arrayBuffer();
}

const buffers: Buffer[] = [Buffer.from(JSON.stringify(headers)), Buffer.from(wasmMemoryBuffer)];

return joinBuffers(buffers, DELIMITER);
}
}
5 changes: 1 addition & 4 deletions warp-contracts-plugin-quickjs/src/eval/evalCode/decorator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,8 @@ export const decorateProcessFn = (processCode: string) => {
${processCode}
function __handleDecorator(message) {
console.log('before handle test');
console.log('message', message);
console.log('test');
ao.clearOutbox();
handle(currentState, message);
console.log('after handle', ao.outbox);
return JSON.stringify(ao.outbox);
}
`;
Expand Down
50 changes: 25 additions & 25 deletions warp-contracts-plugin-quickjs/src/eval/evalCode/globals.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ let ao = {
}
if (this.authorities.length < 1) {
for (const tag of env.process.tags) {
for (const tag of env.process.Tags) {
if (tag.name == 'Authority') {
this.authorities.push(tag.value);
}
Expand Down Expand Up @@ -69,11 +69,11 @@ let ao = {
this._ref = this._ref + 1;
const message = {
target: msg.target,
data: msg.data,
Target: msg.Target,
Data: msg.Data,
// what's happening here?
anchor: '%032d' + ao._ref,
tags: [
Anchor: '%032d' + ao._ref,
Tags: [
{ name: 'Data-Protocol', value: 'ao' },
{ name: 'Variant', value: 'ao.TN.1' },
{ name: 'Type', value: 'Message' },
Expand All @@ -84,19 +84,19 @@ let ao = {
};
for (const key in msg) {
if (!['target', 'data', 'anchor', 'tags'].includes(key)) {
message.tags.push({ name: key, value: msg[key] });
if (!['Target', 'Data', 'Anchor', 'Tags'].includes(key)) {
message.Tags.push({ name: key, value: msg[key] });
}
}
if (msg.tags) {
if (Array.isArray(msg.tags)) {
msg.tags.forEach((t) => {
message.tags.push(t);
if (Array.isArray(msg.Tags)) {
msg.Tags.forEach((t) => {
message.Tags.push(t);
});
} else {
for (const key in msg.tags) {
message.tags.push({ name: key, value: msg.tags[key] });
for (const key in msg.Tags) {
message.Tags.push({ name: key, value: msg.Tags[key] });
}
}
}
Expand All @@ -108,36 +108,36 @@ let ao = {
throw new Error('Module source is required');
}
ao._ref = ao_.ref + 1;
ao._ref = ao._ref + 1;
const spawn = {
data: msg.Data || 'NODATA',
anchor: '%032d' + ao._ref,
tags: [
Data: msg.Data || 'NODATA',
Anchor: '%032d' + ao._ref,
Tags: [
{name: "Data-Protocol", value: "ao"},
{name: "Variant", value: "ao.TN.1"},
{name: "Type", value: "Process"},
{name: "From-Process", value: ao.id},
{name: "From-Module", value: ao._module},
{name: "Module", value: module},
{name: "Ref_", value: tostring(ao._ref)}
{name: "Ref_", value: toString(ao._ref)}
]
}
for (const key in msg) {
if (!['target', 'data', 'anchor', 'tags'].includes(key)) {
spawn.tags.push({ name: key, value: msg[key] });
if (!['Target', 'Data', 'Anchor', 'Tags'].includes(key)) {
spawn.Tags.push({ name: key, value: msg[key] });
}
}
if (msg.tags) {
if (Array.isArray(msg.tags)) {
msg.tags.forEach((t) => {
spawn.tags.push(t);
if (msg.Tags) {
if (Array.isArray(msg.Tags)) {
msg.Tags.forEach((t) => {
spawn.Tags.push(t);
});
} else {
for (const key in msg.tags) {
spawn.tags.push({ name: key, value: msg.tags[key] });
for (const key in msg.Tags) {
spawn.Tags.push({ name: key, value: msg.Tags[key] });
}
}
}
Expand Down
113 changes: 19 additions & 94 deletions warp-contracts-plugin-quickjs/src/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
import { LoggerFactory, WarpPlugin, WarpPluginType, QuickJsPluginInput, QuickJsOptions } from 'warp-contracts';
import {
Lifetime,
LoggerFactory,
WarpPlugin,
WarpPluginType,
QuickJsPluginInput,
QuickJsOptions,
QuickJsBinaryType
} from 'warp-contracts';
import {
QuickJSContext,
QuickJSRuntime,
QuickJSWASMModule,
Expand All @@ -11,12 +17,11 @@ import {
import { QuickJsHandlerApi } from './QuickJsHandlerApi';
import { decorateProcessFn } from './eval/evalCode/decorator';
import { globals } from './eval/evalCode/globals';
import { WasmMemoryBuffer, WasmMemoryHeaders, WasmModuleConfig } from './types';
import { splitBuffer, vmIntrinsics } from './utils';
import { WasmModuleConfig } from './types';
import { vmIntrinsics } from './utils';
import { QuickJsEvaluator } from './eval/QuickJsEvaluator';

export const DELIMITER = '|||';
export const VARIANT_TYPE = RELEASE_SYNC;
const MEMORY_LIMIT = 1024 * 640;
const MAX_STACK_SIZE = 1024 * 320;
const INTERRUPT_CYCLES = 1024;
Expand All @@ -36,17 +41,16 @@ export class QuickJsPlugin<State> implements WarpPlugin<QuickJsPluginInput, Prom
QuickJS: this.QuickJS,
runtime: this.runtime,
vm: this.vm
} = input.wasmMemory ? await this.configureExistingWasmModule(input.wasmMemory) : await this.configureWasmModule());
} = await this.configureWasmModule(input.binaryType));
this.setRuntimeOptions();

const quickJsEvaluator = new QuickJsEvaluator(this.vm);

quickJsEvaluator.evalGlobalsCode(globals);
quickJsEvaluator.evalHandleFnCode(decorateProcessFn, input.contractSource);
quickJsEvaluator.evalLogging();
if (!input.wasmMemory) {
quickJsEvaluator.evalGlobalsCode(globals);
quickJsEvaluator.evalHandleFnCode(decorateProcessFn, input.contractSource);
}

return new QuickJsHandlerApi(this.vm, this.runtime, this.QuickJS, input.wasmMemory, this.quickJsOptions.compress);
return new QuickJsHandlerApi(this.vm, this.runtime, this.QuickJS);
}

setRuntimeOptions() {
Expand All @@ -58,94 +62,15 @@ export class QuickJsPlugin<State> implements WarpPlugin<QuickJsPluginInput, Prom
);
}

async configureExistingWasmModule(wasmMemory: Buffer): Promise<WasmModuleConfig> {
try {
const splittedBuffer = splitBuffer(wasmMemory, DELIMITER);
const headers: WasmMemoryHeaders = JSON.parse(splittedBuffer[WasmMemoryBuffer.HEADERS].toString());
const existingVariantType = JSON.stringify(headers.variantType);
const variantType = JSON.stringify(VARIANT_TYPE);

if (existingVariantType != variantType) {
throw new Error(
`Trying to configure WASM module with non-compatible variant type. Existing variant type: ${existingVariantType}, variant type: ${variantType}.`
);
}
let memory: Buffer;
memory = splittedBuffer[WasmMemoryBuffer.MEMORY];
const runtimePointer = headers.runtimePointer;
const vmPointer = headers.vmPointer;

if (headers.compressed) {
const compressedMemoryView = new Uint8Array(memory);
const decompressionStream = new DecompressionStream('gzip');
const compressedStream = new ReadableStream({
start(controller) {
controller.enqueue(compressedMemoryView);
controller.close();
}
});
const decompressedStream = compressedStream.pipeThrough(decompressionStream);
const decompressedBuffer = await new Response(decompressedStream).arrayBuffer();
memory = Buffer.from(decompressedBuffer);
}

const existingMemoryView = new Uint8Array(memory);
const pageSize = MEMORY_INITIAL_PAGE_SIZE;
const numPages = Math.ceil(memory.byteLength / pageSize);
const newWasmMemory = new WebAssembly.Memory({
initial: numPages,
maximum: MEMORY_MAXIMUM_PAGE_SIZE
});
const newWasmMemoryView = new Uint8Array(newWasmMemory.buffer);

newWasmMemoryView.set(existingMemoryView);

const variant = newVariant(VARIANT_TYPE, {
wasmMemory: newWasmMemory
});

const QuickJS = await newQuickJSWASMModule(variant);
const rt = new Lifetime(runtimePointer, undefined, (rt_ptr) => {
//@ts-ignore
QuickJS.callbacks.deleteRuntime(rt_ptr);
//@ts-ignore
QuickJS.ffi.QTS_FreeRuntime(rt_ptr);
});
const runtime = new QuickJSRuntime({
//@ts-ignore
module: QuickJS.module,
//@ts-ignore
callbacks: QuickJS.callbacks,
//@ts-ignore
ffi: QuickJS.ffi,
//@ts-ignore
rt
});

const vm = runtime.newContext({
//@ts-ignore
contextPointer: vmPointer,
intrinsics: vmIntrinsics
});

return {
QuickJS,
runtime,
vm
};
} catch (e: any) {
this.logger.error(e);
throw new Error(`Could not create WASM module from existing memory. ${JSON.stringify(e.message)}`);
}
}

async configureWasmModule(): Promise<WasmModuleConfig> {
async configureWasmModule(binaryType: QuickJsBinaryType): Promise<WasmModuleConfig> {
try {
const initialWasmMemory = new WebAssembly.Memory({
initial: 256, //*65536
maximum: 2048 //*65536
});
const variant = newVariant(VARIANT_TYPE, {

// TODO: set variant depending on the binaryType
const variant = newVariant(RELEASE_SYNC, {
wasmMemory: initialWasmMemory
});
const QuickJS = await newQuickJSWASMModule(variant);
Expand Down
Loading

0 comments on commit 9e22b89

Please sign in to comment.