From 2e5e8a18d23259071876b1c7ed11e95df32e1360 Mon Sep 17 00:00:00 2001 From: Joe Heffernan Date: Wed, 27 Nov 2024 10:07:17 -0800 Subject: [PATCH 1/5] distinguish between playback and streaming methods in controller to facilitate prefetching --- .../src/Components/CacheAndStreamingLogs.tsx | 86 +++++++++++++++++++ examples/src/Viewer.tsx | 83 ++++++++++++------ src/constants.ts | 1 + src/controller/index.ts | 85 +++++++++++------- src/simularium/VisData.ts | 44 ++++++++++ src/simularium/VisDataCache.ts | 19 +++- src/test/AgentSimController.test.ts | 4 +- src/test/SimulariumController.test.ts | 4 +- src/viewport/index.tsx | 11 ++- 9 files changed, 275 insertions(+), 62 deletions(-) create mode 100644 examples/src/Components/CacheAndStreamingLogs.tsx diff --git a/examples/src/Components/CacheAndStreamingLogs.tsx b/examples/src/Components/CacheAndStreamingLogs.tsx new file mode 100644 index 00000000..deb45166 --- /dev/null +++ b/examples/src/Components/CacheAndStreamingLogs.tsx @@ -0,0 +1,86 @@ +import React, { useEffect, useState } from "react"; + +interface StreamingReadoutProps { + playbackState: string; + streamingState: string; + cacheSize: number; + cacheEnabled: boolean; + maxSize: number; + firstFrameNumber: number; + lastFrameNumber: number; + currentPlaybackFrame: number; + totalDuration: number; + getFirstFrameNumber: () => number; + getLastFrameNumber: () => number; +} + +const CacheAndStreamingLogs: React.FC = ({ + playbackState, + streamingState, + cacheSize, + cacheEnabled, + maxSize, + firstFrameNumber, + lastFrameNumber, + currentPlaybackFrame, + totalDuration, + getFirstFrameNumber, + getLastFrameNumber, +}) => { + const [isOpen, setIsOpen] = useState(false); + const [debugInfo, setDebugInfo] = useState({ + directFirst: -1, + functionFirst: -1, + renderCount: 0, + }); + + useEffect(() => { + setDebugInfo((prev) => ({ + directFirst: firstFrameNumber, + functionFirst: getFirstFrameNumber(), + renderCount: prev.renderCount + 1, + })); + }, [firstFrameNumber, getFirstFrameNumber, lastFrameNumber]); + + const calculateRatio = () => { + const first = Math.max(firstFrameNumber, getFirstFrameNumber()); + const last = Math.max(lastFrameNumber, getLastFrameNumber()); + if (last <= first || currentPlaybackFrame < first) return 0; + return (last - currentPlaybackFrame) / (last - first); + }; + + return ( +
+ + + {isOpen && ( + <> +
Playback State: {playbackState}
+
Streaming State: {streamingState}
+
Cache Size: {cacheSize}
+
Cache Enabled: {cacheEnabled ? "Yes" : "No"}
+
Max Size: {maxSize}
+
First Frame Number: {getFirstFrameNumber()} this is buggy and shows a stale value even when i can log a different value with hasFrames()
+
Last Frame Number: {getLastFrameNumber()}
+
Total Duration: {totalDuration}
+
+
Debug Info:
+
+ Render Count: {debugInfo.renderCount} +
+
+ Direct First Frame: {debugInfo.directFirst} +
+
+ Function First Frame: {debugInfo.functionFirst} +
+
+ + )} +
+ ); +}; + +export default CacheAndStreamingLogs; diff --git a/examples/src/Viewer.tsx b/examples/src/Viewer.tsx index f5fa2fb4..e33b7f17 100644 --- a/examples/src/Viewer.tsx +++ b/examples/src/Viewer.tsx @@ -57,9 +57,7 @@ import { UI_TEMPLATE_DOWNLOAD_URL_ROOT, UI_TEMPLATE_URL_ROOT, } from "./api-settings"; - -import "../../style/style.css"; -import "./style.css"; +import CacheAndStreamingLogs from "./Components/CacheAndStreamingLogs"; let playbackFile = "TEST_LIVEMODE_API"; let queryStringFile = ""; @@ -392,7 +390,7 @@ class Viewer extends React.Component { } this.setState({ currentFrame, currentTime }); if (currentFrame < 0) { - simulariumController.pause(); + simulariumController.pauseStreaming(); } } @@ -454,8 +452,12 @@ class Viewer extends React.Component { }); } + public handlePause(): void { + simulariumController.pausePlayback(); + } + public handleScrubTime(event): void { - simulariumController.gotoTime(parseFloat(event.target.value)); + simulariumController.movePlaybackTime(parseFloat(event.target.value)); } public handleUIDisplayData(uiDisplayData: UIDisplayData): void { @@ -487,13 +489,13 @@ class Viewer extends React.Component { } public gotoNextFrame(): void { - simulariumController.gotoTime( + simulariumController.movePlaybackTime( this.state.currentTime + this.state.timeStep ); } public gotoPreviousFrame(): void { - simulariumController.gotoTime( + simulariumController.movePlaybackTime( this.state.currentTime - this.state.timeStep ); } @@ -516,8 +518,8 @@ class Viewer extends React.Component { } private configureAndLoad() { - simulariumController.configureNetwork(this.netConnectionSettings); if (playbackFile.startsWith("http")) { + simulariumController.configureNetwork(this.netConnectionSettings); return this.loadFromUrl(playbackFile); } if (playbackFile === "TEST_LIVEMODE_API") { @@ -694,7 +696,7 @@ class Viewer extends React.Component {
{ - simulariumController.pauseStreaming(); + this.handlePauseStreaming(); playbackFile = event.target.value; this.configureAndLoad(); }} @@ -782,10 +810,12 @@ class Viewer extends React.Component { Load a smoldyn trajectory
- + -
diff --git a/src/controller/index.ts b/src/controller/index.ts index be2f9c73..6683a22b 100644 --- a/src/controller/index.ts +++ b/src/controller/index.ts @@ -53,8 +53,6 @@ export default class SimulariumController { public onError?: (error: FrontEndError) => void; private networkEnabled: boolean; - public isPlaybackPaused: boolean; - public isStreaming: boolean; public isFileChanging: boolean; private playBackFile: string; @@ -103,8 +101,6 @@ export default class SimulariumController { } this.networkEnabled = true; - this.isPlaybackPaused = false; - this.isStreaming = false; this.isFileChanging = false; this.playBackFile = params.trajectoryPlaybackFile || ""; this.zoomIn = this.zoomIn.bind(this); @@ -162,9 +158,9 @@ export default class SimulariumController { this.handleTrajectoryInfo(trajFileInfo); } ); - this.visData.setOnCacheLimitReached((lastFrameTime: number) => { + this.visData.setOnCacheLimitReached(() => { this.pauseStreaming(); - this.moveRemoteSimulationTime(lastFrameTime); + // this.simulator?.requestSingleFrame(this.visData.currentStreamingHead); }); } @@ -206,7 +202,7 @@ export default class SimulariumController { // switch back to 'networked' playback this.networkEnabled = true; - this.isPlaybackPaused = true; + this.visData.isPlaying = false; this.visData.clearCache(); // todo renaming of initalize/playback methods in ISimulator @@ -220,7 +216,7 @@ export default class SimulariumController { public abortRemoteSimulation(): void { if (this.simulator) { this.simulator.abortRemoteSim(); - this.isStreaming = false; + this.visData.updateStreamingState(false); } } @@ -259,15 +255,11 @@ export default class SimulariumController { public pauseStreaming(): void { if (this.networkEnabled && this.simulator) { + this.visData.updateStreamingState(false); this.simulator.pauseRemoteSim(); - this.isStreaming = false; } } - public playbackPaused(): boolean { - return this.isPlaybackPaused; - } - public initializeTrajectoryFile(): void { if (this.simulator) { this.simulator.requestTrajectoryFileInfo(this.playBackFile); @@ -282,47 +274,70 @@ export default class SimulariumController { this.resumeStreaming(); } else { if (this.networkEnabled && this.simulator) { - this.visData.clearCache(); - this.moveRemoteSimulationTime(time); - this.resumeStreaming(); + this.simulator.gotoRemoteSimulationTime(time); + // get frame number for time + const frameNumber = time; + // time is framenumber *timestep + + this.visData.currentFrameNumber = frameNumber; + // this.resumeStreaming(); } } } - public moveRemoteSimulationTime(time: number): void { - if (this.networkEnabled && this.simulator) { - this.simulator.gotoRemoteSimulationTime(time); + public movePlaybackFrame(frameNumber: number): void { + // If in the middle of changing files, ignore any gotoTime requests + console.log("movePlaybackFrame", frameNumber); + if (this.isFileChanging === true) return; + if (this.visData.hasLocalCacheForFrame(frameNumber)) { + this.visData.gotoFrame(frameNumber); + this.resumeStreaming(); + } else { + if (this.networkEnabled && this.simulator) { + this.clearLocalCache(); + this.visData.WaitForFrame(frameNumber); + this.visData.currentFrameNumber = frameNumber; + this.resumeStreaming(frameNumber); + } } } public playFromTime(time: number): void { this.movePlaybackTime(time); - this.isPlaybackPaused = false; + this.visData.isPlaying = true; } public initalizeStreaming(): void { if (this.simulator) { this.simulator.requestSingleFrame(0); this.simulator.resumeRemoteSim(); - this.isStreaming = true; + this.visData.updateStreamingState(true); } } - public resumeStreaming(): void { + public resumeStreaming(startFrame?: number): void { + let requestFrame: number | null = null; + if (startFrame !== undefined) { + requestFrame = startFrame; + } else if (this.visData.remoteStreamingHeadPotentiallyOutOfSync) { + requestFrame = this.visData.currentStreamingHead; + } if (this.networkEnabled && this.simulator) { + if (requestFrame !== null) { + this.simulator.requestSingleFrame(requestFrame); + } this.simulator.resumeRemoteSim(); - this.isStreaming = true; + this.visData.updateStreamingState(true); + this.visData.remoteStreamingHeadPotentiallyOutOfSync = false; } } public pausePlayback(): void { - this.isPlaybackPaused = true; - this.visData.setPlaybackPaused(true); + this.visData.isPlaying = false; } public resumePlayback(): void { - this.isPlaybackPaused = false; - this.visData.setPlaybackPaused(false); + this.visData.isPlaying = true; } public clearFile(): void { @@ -366,11 +381,10 @@ export default class SimulariumController { this.simulator.handleError = () => noop; } + this.abortRemoteSimulation(); this.visData.WaitForFrame(0); this.visData.clearForNewTrajectory(); - this.abortRemoteSimulation(); - // don't create simulator if client wants to keep remote simulator and the // current simulator is a remote simulator if ( @@ -385,7 +399,7 @@ export default class SimulariumController { connectionParams.geoAssets ); this.networkEnabled = true; // This confuses me, because local files also go through this code path - this.isPlaybackPaused = true; + this.visData.isPlaying = false; } else { // caught in following block, not sent to front end throw new Error("incomplete simulator config provided"); @@ -395,7 +409,7 @@ export default class SimulariumController { this.simulator = undefined; console.warn(error.message); this.networkEnabled = false; - this.isPlaybackPaused = false; + this.visData.isPlaying = false; } } @@ -567,6 +581,22 @@ export default class SimulariumController { public setCameraType(ortho: boolean): void { this.visGeometry?.setCameraType(ortho); } + + public isStreaming(): boolean { + return this.visData.isStreaming; + } + + public isPlaying(): boolean { + return this.visData.isPlaying; + } + + public currentPlaybackHead(): number { + return this.visData.currentFrameNumber; + } + + public currentStreamingHead(): number { + return this.visData.currentStreamingHead; + } } export { SimulariumController }; diff --git a/src/simularium/VisData.ts b/src/simularium/VisData.ts index 6cce6e09..a747e1ee 100644 --- a/src/simularium/VisData.ts +++ b/src/simularium/VisData.ts @@ -14,12 +14,13 @@ class VisData { private frameToWaitFor: number; private lockedForFrame: boolean; - private currentFrameNumber: number; - private isPlaybackPaused: boolean; - public setPlaybackPaused(paused: boolean): void { - this.isPlaybackPaused = paused; - } - public onCacheLimitReached: (latestFrame: number) => void; + public currentFrameNumber: number; // playback head + public currentStreamingHead: number; + public remoteStreamingHeadPotentiallyOutOfSync: boolean; + public isPlaying: boolean; + public isStreaming: boolean; + public onStreamingChange: (streaming: boolean) => void; + public onCacheLimitReached: () => void; public timeStepSize: number; public totalSteps: number; @@ -44,14 +45,18 @@ class VisData { public constructor() { this.currentFrameNumber = -1; + this.currentStreamingHead = -1; + this.remoteStreamingHeadPotentiallyOutOfSync = false; this.frameCache = new VisDataCache(); this.frameToWaitFor = 0; this.lockedForFrame = false; this.timeStepSize = 0; this.totalSteps = 0; - this.isPlaybackPaused = false; + this.isPlaying = false; + this.isStreaming = false; this.onError = noop; + this.onStreamingChange = noop; this.onCacheLimitReached = noop; } @@ -59,9 +64,13 @@ class VisData { this.onError = onError; } - public setOnCacheLimitReached( - onCacheLimitReached: (latestFrame: number) => void + public setOnStreamingChange( + onStreamingChange: (streaming: boolean) => void ): void { + this.onStreamingChange = onStreamingChange; + } + + public setOnCacheLimitReached(onCacheLimitReached: () => void): void { this.onCacheLimitReached = onCacheLimitReached; } @@ -93,6 +102,10 @@ class VisData { return this.frameCache.containsTime(time); } + public hasLocalCacheForFrame(frameNumber: number): boolean { + return this.frameCache.containsFrameAtFrameNumber(frameNumber); + } + public gotoTime(time: number): void { const frameNumber = this.frameCache.getFrameAtTime(time)?.frameNumber; if (frameNumber !== undefined) { @@ -100,6 +113,12 @@ class VisData { } } + public gotoFrame(frameNumber: number): void { + if (this.hasLocalCacheForFrame(frameNumber)) { + this.currentFrameNumber = frameNumber; + } + } + public atLatestFrame(): boolean { return this.currentFrameNumber >= this.frameCache.getLastFrameNumber(); } @@ -110,6 +129,11 @@ class VisData { } } + public updateStreamingState(isStreaming: boolean): void { + this.isStreaming = isStreaming; + this.onStreamingChange(isStreaming); + } + /** * Data management * */ @@ -161,7 +185,7 @@ class VisData { this.frameExceedsCacheSizeError(parsedMsg.size); return; } - this.addFrameToCache(parsedMsg); + this.validateAndProcessFrame(parsedMsg); } public parseAgentsFromFrameData(msg: VisDataMessage | ArrayBuffer): void { @@ -170,7 +194,7 @@ class VisData { if (frame.frameNumber === 0) { this.clearCache(); // new data has arrived } - this.addFrameToCache(frame); + this.validateAndProcessFrame(frame); return; } this.parseAgentsFromVisDataMessage(msg); @@ -192,7 +216,10 @@ class VisData { this.parseAgentsFromFrameData(msg); } - private addFrameToCache(frame: CachedFrame): void { + /** + * Incoming frame management + */ + private handleOversizedFrame(frame: CachedFrame): void { if ( this.frameCache.cacheSizeLimited && frame.size > this.frameCache.maxSize @@ -200,36 +227,53 @@ class VisData { this.frameExceedsCacheSizeError(frame.size); return; } + } - // TODO: the code below and associated callbacks - // are not finished/finalized - // we currently need controller to tell visData whether playback - // is ongoing and to send a callback to correct the "backend time" - // when it gets out of sync with the end of the cache - // as vidata does not directly interact with the simulator - if (this.frameCache.size + frame.size > this.frameCache.maxSize) { - // if playback is ongoing we can trim from the beginning of the cache - // and add frames to the end of the cache - if (!this.isPlaybackPaused) { - const playbackFrame = this.currentFrameData; - if ( - playbackFrame.frameNumber > - this.frameCache.getFirstFrameNumber() - ) { - this.frameCache.trimCache(playbackFrame.size); - this.frameCache.addFrame(frame); - } else { - // If playback is not advancing, we are in prefetch mode - // (streaming but playback paused) - // Stop streaming, reject the frame, reset backend "time" - // to the latest frame in the cache - this.onCacheLimitReached( - this.frameCache.getLastFrameTime() - ); - } - } + private trimAndAddFrame(frame: CachedFrame): void { + this.frameCache.trimCache(this.currentFrameData.size); + this.frameCache.addFrame(frame); + } + + private resetCacheWithFrame(frame: CachedFrame): void { + this.clearCache(); + this.frameCache.addFrame(frame); + } + + private handleCacheOverflow(frame: CachedFrame): boolean { + if (frame.size + this.frameCache.size <= this.frameCache.maxSize) { + return false; + } + const playbackFrame = this.currentFrameData; + const isCacheHeadBehindPlayback = + playbackFrame.frameNumber > this.frameCache.getFirstFrameNumber(); + + if (isCacheHeadBehindPlayback) { + this.trimAndAddFrame(frame); + } else if (this.isPlaying) { + // if currently playing, and cache head is ahead of playback head + // we clear the cache and add the frame + this.resetCacheWithFrame(frame); + } else { + // if paused we run out of space we need to stop streaming + // which is handled by the controller via a callback + this.currentStreamingHead = frame.frameNumber; + this.remoteStreamingHeadPotentiallyOutOfSync = true; + this.onCacheLimitReached(); } + return true; + } + + private validateAndProcessFrame(frame: CachedFrame): void { + this.handleOversizedFrame(frame); + + if (!this.handleCacheOverflow(frame)) { + this.addFrameToCache(frame); + } + } + + private addFrameToCache(frame: CachedFrame): void { this.frameCache.addFrame(frame); + this.currentStreamingHead = this.frameCache.getLastFrameNumber(); } private frameExceedsCacheSizeError(frameSize: number): void { diff --git a/src/viewport/index.tsx b/src/viewport/index.tsx index 06b867bd..8047e8a2 100644 --- a/src/viewport/index.tsx +++ b/src/viewport/index.tsx @@ -50,6 +50,7 @@ type ViewportProps = { onFollowObjectChanged?: (agentData: AgentData) => void; // passes agent data about the followed agent to the front end maxCacheSize?: number; onCacheUpdate?: (log: CacheLog) => void; + onStreamingChange?: (streaming: boolean) => void; } & Partial; const defaultProps = { @@ -137,6 +138,11 @@ class Viewport extends React.Component< if (props.onError) { this.props.simulariumController.visData.setOnError(props.onError); } + if (props.onStreamingChange) { + this.props.simulariumController.visData.setOnStreamingChange( + props.onStreamingChange + ); + } this.props.simulariumController.visData.clearCache(); this.visGeometry.createMaterials(props.agentColors); this.vdomRef = React.createRef(); @@ -661,7 +667,7 @@ class Viewport extends React.Component< } } - if (!visData.atLatestFrame() && !simulariumController.playbackPaused()) { + if (!visData.atLatestFrame() && simulariumController.isPlaying()) { visData.gotoNextFrame(); } this.stats.begin();