diff --git a/backend/api/EventHandlers/MqttEventHandler.cs b/backend/api/EventHandlers/MqttEventHandler.cs index 9fc099a33..b0401bd51 100644 --- a/backend/api/EventHandlers/MqttEventHandler.cs +++ b/backend/api/EventHandlers/MqttEventHandler.cs @@ -58,6 +58,7 @@ public override void Subscribe() MqttService.MqttIsarPressureReceived += OnIsarPressureUpdate; MqttService.MqttIsarPoseReceived += OnIsarPoseUpdate; MqttService.MqttIsarCloudHealthReceived += OnIsarCloudHealthUpdate; + MqttService.MqttIsarMediaConfigReceived += OnIsarMediaConfigUpdate; } public override void Unsubscribe() @@ -71,6 +72,7 @@ public override void Unsubscribe() MqttService.MqttIsarPressureReceived -= OnIsarPressureUpdate; MqttService.MqttIsarPoseReceived -= OnIsarPoseUpdate; MqttService.MqttIsarCloudHealthReceived -= OnIsarCloudHealthUpdate; + MqttService.MqttIsarMediaConfigReceived -= OnIsarMediaConfigUpdate; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await stoppingToken; } @@ -485,5 +487,25 @@ private async void OnIsarCloudHealthUpdate(object? sender, MqttReceivedArgs mqtt TeamsMessageService.TriggerTeamsMessageReceived(new TeamsMessageEventArgs(message)); } + + private async void OnIsarMediaConfigUpdate(object? sender, MqttReceivedArgs mqttArgs) + { + var isarTelemetyUpdate = (IsarMediaConfigMessage)mqttArgs.Message; + + var robot = await RobotService.ReadByIsarId(isarTelemetyUpdate.IsarId); + if (robot == null) + { + _logger.LogInformation("Received message from unknown ISAR instance {Id} with robot name {Name}", isarTelemetyUpdate.IsarId, isarTelemetyUpdate.RobotName); + return; + } + await SignalRService.SendMessageAsync("Media stream config received", robot.CurrentInstallation, + new MediaConfig + { + Url = isarTelemetyUpdate.Url, + Token = isarTelemetyUpdate.Token, + RobotId = robot.Id, + MediaConnectionType = isarTelemetyUpdate.MediaConnectionType + }); + } } } diff --git a/backend/api/MQTT/MessageModels/IsarMediaConfig.cs b/backend/api/MQTT/MessageModels/IsarMediaConfig.cs new file mode 100644 index 000000000..214a4420a --- /dev/null +++ b/backend/api/MQTT/MessageModels/IsarMediaConfig.cs @@ -0,0 +1,25 @@ +using System.Text.Json.Serialization; +using Api.Services.Models; + +namespace Api.Mqtt.MessageModels +{ +#nullable disable + public class IsarMediaConfigMessage : MqttMessage + { + [JsonPropertyName("robot_name")] + public string RobotName { get; set; } + + [JsonPropertyName("isar_id")] + public string IsarId { get; set; } + + [JsonPropertyName("url")] + public string Url { get; set; } + + [JsonPropertyName("token")] + public string Token { get; set; } + + [JsonPropertyName("mediaConnectionType")] + public MediaConnectionType MediaConnectionType { get; set; } + + } +} diff --git a/backend/api/MQTT/MqttService.cs b/backend/api/MQTT/MqttService.cs index 480430f7e..0a5743318 100644 --- a/backend/api/MQTT/MqttService.cs +++ b/backend/api/MQTT/MqttService.cs @@ -91,6 +91,7 @@ public MqttService(ILogger logger, IConfiguration config) public static event EventHandler? MqttIsarPressureReceived; public static event EventHandler? MqttIsarPoseReceived; public static event EventHandler? MqttIsarCloudHealthReceived; + public static event EventHandler? MqttIsarMediaConfigReceived; protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -152,6 +153,9 @@ private Task OnMessageReceived(MqttApplicationMessageReceivedEventArgs messageRe case Type type when type == typeof(IsarCloudHealthMessage): OnIsarTopicReceived(content); break; + case Type type when type == typeof(IsarMediaConfigMessage): + OnIsarTopicReceived(content); + break; default: _logger.LogWarning( "No callback defined for MQTT message type '{type}'", @@ -301,6 +305,7 @@ private void OnIsarTopicReceived(string content) where T : MqttMessage _ when type == typeof(IsarPressureMessage) => MqttIsarPressureReceived, _ when type == typeof(IsarPoseMessage) => MqttIsarPoseReceived, _ when type == typeof(IsarCloudHealthMessage) => MqttIsarCloudHealthReceived, + _ when type == typeof(IsarMediaConfigMessage) => MqttIsarMediaConfigReceived, _ => throw new NotImplementedException( $"No event defined for message type '{typeof(T).Name}'" diff --git a/backend/api/MQTT/MqttTopics.cs b/backend/api/MQTT/MqttTopics.cs index 3d6fcbf42..9a499cc26 100644 --- a/backend/api/MQTT/MqttTopics.cs +++ b/backend/api/MQTT/MqttTopics.cs @@ -42,6 +42,9 @@ public static class MqttTopics }, { "isar/+/cloud_health", typeof(IsarCloudHealthMessage) + }, + { + "isar/+/media_config", typeof(IsarMediaConfigMessage) } }; diff --git a/backend/api/Services/Models/MediaConfig.cs b/backend/api/Services/Models/MediaConfig.cs new file mode 100644 index 000000000..8d5a077f2 --- /dev/null +++ b/backend/api/Services/Models/MediaConfig.cs @@ -0,0 +1,20 @@ +using System.Text.Json.Serialization; +namespace Api.Services.Models +{ + public struct MediaConfig + { + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonPropertyName("token")] + public string? Token { get; set; } + + [JsonPropertyName("robotId")] + public string? RobotId { get; set; } + + [JsonPropertyName("mediaConnectionType")] + public MediaConnectionType MediaConnectionType { get; set; } + } + + public enum MediaConnectionType { LiveKit }; +} diff --git a/backend/api/appsettings.Development.json b/backend/api/appsettings.Development.json index ec30fbcd2..e9955ddc6 100644 --- a/backend/api/appsettings.Development.json +++ b/backend/api/appsettings.Development.json @@ -32,7 +32,8 @@ "isar/+/battery", "isar/+/pressure", "isar/+/pose", - "isar/+/cloud_health" + "isar/+/cloud_health", + "isar/+/media_config" ], "MaxRetryAttempts": 5, "ShouldFailOnMaxRetries": false diff --git a/backend/api/appsettings.Local.json b/backend/api/appsettings.Local.json index 448c08de6..ba15e5f55 100644 --- a/backend/api/appsettings.Local.json +++ b/backend/api/appsettings.Local.json @@ -32,7 +32,8 @@ "isar/+/battery", "isar/+/pressure", "isar/+/pose", - "isar/+/cloud_health" + "isar/+/cloud_health", + "isar/+/media_config" ], "MaxRetryAttempts": 5, "ShouldFailOnMaxRetries": false diff --git a/backend/api/appsettings.Production.json b/backend/api/appsettings.Production.json index 022550f1c..2339c5fbe 100644 --- a/backend/api/appsettings.Production.json +++ b/backend/api/appsettings.Production.json @@ -28,7 +28,8 @@ "isar/+/battery", "isar/+/pressure", "isar/+/pose", - "isar/+/cloud_health" + "isar/+/cloud_health", + "isar/+/media_config" ], "MaxRetryAttempts": 15, "ShouldFailOnMaxRetries": true diff --git a/backend/api/appsettings.Staging.json b/backend/api/appsettings.Staging.json index 1fca017fa..e0e78b682 100644 --- a/backend/api/appsettings.Staging.json +++ b/backend/api/appsettings.Staging.json @@ -32,7 +32,8 @@ "isar/+/battery", "isar/+/pressure", "isar/+/pose", - "isar/+/cloud_health" + "isar/+/cloud_health", + "isar/+/media_config" ], "MaxRetryAttempts": 15, "ShouldFailOnMaxRetries": true diff --git a/backend/api/appsettings.Test.json b/backend/api/appsettings.Test.json index 392c3687f..fc712c030 100644 --- a/backend/api/appsettings.Test.json +++ b/backend/api/appsettings.Test.json @@ -30,7 +30,8 @@ "isar/+/battery", "isar/+/pressure", "isar/+/pose", - "isar/+/cloud_health" + "isar/+/cloud_health", + "isar/+/media_config" ], "MaxRetryAttempts": 15, "ShouldFailOnMaxRetries": true diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e94c89bbf..d322a1759 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -13,6 +13,7 @@ "@equinor/eds-core-react": "^0.36.1", "@equinor/eds-icons": "^0.21.0", "@equinor/eds-tokens": "^0.9.2", + "@livekit/components-styles": "^1.0.12", "@microsoft/applicationinsights-web": "^3.1.2", "@microsoft/signalr": "^8.0.0", "@testing-library/jest-dom": "^6.4.2", @@ -23,12 +24,14 @@ "@types/react": "^18.2.75", "@types/react-dom": "^18.2.24", "date-fns": "^3.6.0", + "livekit-client": "^2.5.1", "ovenplayer": "^0.10.35", "prettier": "^3.2.5", "react": "^18.2.0", "react-dom": "^18.2.0", "react-error-boundary": "^4.0.13", "react-modal": "^3.15.1", + "react-player": "^2.16.0", "react-router-dom": "^6.22.3", "react-scripts": "^5.0.1", "styled-components": "^6.1.8", @@ -2028,6 +2031,11 @@ "version": "0.2.3", "license": "MIT" }, + "node_modules/@bufbuild/protobuf": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.10.0.tgz", + "integrity": "sha512-QDdVFLoN93Zjg36NoQPZfsVH9tZew7wKDKyV5qRdj8ntT4wQCOradQjRaTdwMhWUYsgKsvCINKKm87FdEk96Ag==" + }, "node_modules/@csstools/normalize.css": { "version": "12.0.0", "license": "CC0-1.0" @@ -2453,12 +2461,12 @@ } }, "node_modules/@floating-ui/dom": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.3.tgz", - "integrity": "sha512-RnDthu3mzPlQ31Ss/BTwQ1zjzIhr3lk1gZB1OC56h/1vEtaXkESrOqL5fQVMfXpwGtRwX+YsZBdyHtJMQnkArw==", + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.8.tgz", + "integrity": "sha512-kx62rP19VZ767Q653wsP1XZCGIirkE09E0QUGNYTM/ttbbQHqcGPdSfWFxUyyNLc/W6aoJRBajOSXhP6GXjC0Q==", "dependencies": { - "@floating-ui/core": "^1.0.0", - "@floating-ui/utils": "^0.2.0" + "@floating-ui/core": "^1.6.0", + "@floating-ui/utils": "^0.2.5" } }, "node_modules/@floating-ui/react": { @@ -2488,9 +2496,9 @@ } }, "node_modules/@floating-ui/utils": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.1.tgz", - "integrity": "sha512-9TANp6GPoMtYzQdt54kfAyMmz1+osLlXdg2ENroU7zzrtflTLrrC/lgrIfaSe+Wu0b89GKccT7vxXA0MoAIO+Q==" + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.7.tgz", + "integrity": "sha512-X8R8Oj771YRl/w+c1HqAC1szL8zWQRwFvgDwT129k9ACdBoud/+/rX9V0qiMl6LWUdP9voC2nDVZYPMQQsb6eA==" }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.13", @@ -2766,6 +2774,14 @@ "version": "2.0.4", "license": "MIT" }, + "node_modules/@livekit/components-styles": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/@livekit/components-styles/-/components-styles-1.0.12.tgz", + "integrity": "sha512-Hsxkfq240w0tMPtkQTHQotpkYfIY4lhP2pzegvOIIV/nYxj8LeRYypUjxJpFw3s6jQcV/WQS7oCYmFQdy98Jtw==", + "engines": { + "node": ">=18" + } + }, "node_modules/@microsoft/applicationinsights-analytics-js": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@microsoft/applicationinsights-analytics-js/-/applicationinsights-analytics-js-3.1.2.tgz", @@ -9829,6 +9845,34 @@ "version": "1.2.4", "license": "MIT" }, + "node_modules/livekit-client": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.5.1.tgz", + "integrity": "sha512-D7BzGoO3nc7/H2pEP9hseTjpzfgUoQ1AdeUNdlM7XNEywvorY1UR6RhOWH9UvycM/D5tIIRx7OvhxzpVfHE3TA==", + "dependencies": { + "@livekit/protocol": "1.20.1", + "events": "^3.3.0", + "loglevel": "^1.8.0", + "sdp-transform": "^2.14.1", + "ts-debounce": "^4.0.0", + "tslib": "2.6.3", + "typed-emitter": "^2.1.0", + "webrtc-adapter": "^9.0.0" + } + }, + "node_modules/livekit-client/node_modules/@livekit/protocol": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.20.1.tgz", + "integrity": "sha512-TgyuwOx+XJn9inEYT9OKfFNs9YIPS4BdLa4pF5FDf9MhWRnahKwPe7jxr/+sVdWxYbZmy9hRrH58jSAFu0ONHw==", + "dependencies": { + "@bufbuild/protobuf": "^1.7.2" + } + }, + "node_modules/load-script": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/load-script/-/load-script-1.0.0.tgz", + "integrity": "sha512-kPEjMFtZvwL9TaZo0uZ2ml+Ye9HUMmPwbYRJ324qF9tqMejwykJ5ggTyvzmrbBeapCAbk98BSbTeovHEEP1uCA==" + }, "node_modules/loader-runner": { "version": "4.3.0", "license": "MIT", @@ -9885,6 +9929,18 @@ "version": "4.5.0", "license": "MIT" }, + "node_modules/loglevel": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.1.tgz", + "integrity": "sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==", + "engines": { + "node": ">= 0.6.0" + }, + "funding": { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/loglevel" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "license": "MIT", @@ -9995,6 +10051,11 @@ "node": ">= 4.0.0" } }, + "node_modules/memoize-one": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", + "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==" + }, "node_modules/merge-descriptors": { "version": "1.0.1", "license": "MIT" @@ -12302,6 +12363,11 @@ "version": "6.0.11", "license": "MIT" }, + "node_modules/react-fast-compare": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-3.2.2.tgz", + "integrity": "sha512-nsO+KSNgo1SbJqJEYRE9ERzo7YtYbou/OqjSQKxV7jcKox7+usiUVZOAC+XnDOABXggQTno0Y1CpVnuWEc1boQ==" + }, "node_modules/react-is": { "version": "18.2.0", "license": "MIT" @@ -12327,6 +12393,21 @@ "react-dom": "^0.14.0 || ^15.0.0 || ^16 || ^17 || ^18" } }, + "node_modules/react-player": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/react-player/-/react-player-2.16.0.tgz", + "integrity": "sha512-mAIPHfioD7yxO0GNYVFD1303QFtI3lyyQZLY229UEAp/a10cSW+hPcakg0Keq8uWJxT2OiT/4Gt+Lc9bD6bJmQ==", + "dependencies": { + "deepmerge": "^4.0.0", + "load-script": "^1.0.0", + "memoize-one": "^5.1.1", + "prop-types": "^15.7.2", + "react-fast-compare": "^3.0.1" + }, + "peerDependencies": { + "react": ">=16.6.0" + } + }, "node_modules/react-refresh": { "version": "0.11.0", "license": "MIT", @@ -13755,6 +13836,15 @@ "individual": "^2.0.0" } }, + "node_modules/rxjs": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", + "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", + "optional": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/safe-array-concat": { "version": "1.0.1", "license": "MIT", @@ -13888,6 +13978,19 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/sdp": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/sdp/-/sdp-3.2.0.tgz", + "integrity": "sha512-d7wDPgDV3DDiqulJjKiV2865wKsJ34YI+NDREbm+FySq6WuKOikwyNQcm+doLAZ1O6ltdO0SeKle2xMpN3Brgw==" + }, + "node_modules/sdp-transform": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.14.2.tgz", + "integrity": "sha512-icY6jVao7MfKCieyo1AyxFYm1baiM+fA00qW/KrNNVlkxHAd34riEKuEkUe4bBb3gJwLJZM+xT60Yj1QL8rHiA==", + "bin": { + "sdp-verify": "checker.js" + } + }, "node_modules/select-hose": { "version": "2.0.0", "license": "MIT" @@ -15053,6 +15156,11 @@ "node": ">=14.0.0" } }, + "node_modules/ts-debounce": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/ts-debounce/-/ts-debounce-4.0.0.tgz", + "integrity": "sha512-+1iDGY6NmOGidq7i7xZGA4cm8DAa6fqdYcvO5Z6yBevH++Bdo9Qt/mN0TzHUgcCcKv1gmh9+W5dHqz8pMWbCbg==" + }, "node_modules/ts-interface-checker": { "version": "0.1.13", "license": "Apache-2.0" @@ -15085,8 +15193,9 @@ } }, "node_modules/tslib": { - "version": "2.6.2", - "license": "0BSD" + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" }, "node_modules/tsutils": { "version": "3.21.0", @@ -15200,6 +15309,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz", + "integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==", + "optionalDependencies": { + "rxjs": "*" + } + }, "node_modules/typedarray-to-buffer": { "version": "3.1.5", "license": "MIT", @@ -15795,6 +15912,18 @@ "node": ">=4.0" } }, + "node_modules/webrtc-adapter": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.1.tgz", + "integrity": "sha512-1AQO+d4ElfVSXyzNVTOewgGT/tAomwwztX/6e3totvyyzXPvXIIuUUjAmyZGbKBKbZOXauuJooZm3g6IuFuiNQ==", + "dependencies": { + "sdp": "^3.2.0" + }, + "engines": { + "node": ">=6.0.0", + "npm": ">=3.10.0" + } + }, "node_modules/websocket-driver": { "version": "0.7.4", "license": "Apache-2.0", diff --git a/frontend/package.json b/frontend/package.json index 327ef8117..9b61c2255 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,6 +9,7 @@ "@equinor/eds-core-react": "^0.36.1", "@equinor/eds-icons": "^0.21.0", "@equinor/eds-tokens": "^0.9.2", + "@livekit/components-styles": "^1.0.12", "@microsoft/applicationinsights-web": "^3.1.2", "@microsoft/signalr": "^8.0.0", "@testing-library/jest-dom": "^6.4.2", @@ -19,12 +20,14 @@ "@types/react": "^18.2.75", "@types/react-dom": "^18.2.24", "date-fns": "^3.6.0", + "livekit-client": "^2.5.1", "ovenplayer": "^0.10.35", "prettier": "^3.2.5", "react": "^18.2.0", "react-dom": "^18.2.0", "react-error-boundary": "^4.0.13", "react-modal": "^3.15.1", + "react-player": "^2.16.0", "react-router-dom": "^6.22.3", "react-scripts": "^5.0.1", "styled-components": "^6.1.8", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 394c647f2..c2862ca0c 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -14,6 +14,7 @@ import { SignalRProvider } from 'components/Contexts/SignalRContext' import { RobotProvider } from 'components/Contexts/RobotContext' import { config } from 'config' import { MissionDefinitionsProvider } from 'components/Contexts/MissionDefinitionsContext' +import { MediaStreamProvider } from 'components/Contexts/MediaStreamContext' const appInsights = new ApplicationInsights({ config: { @@ -45,7 +46,9 @@ const App = () => ( - + + + diff --git a/frontend/src/api/ApiCaller.tsx b/frontend/src/api/ApiCaller.tsx index 77c8bd9f7..9597daa80 100644 --- a/frontend/src/api/ApiCaller.tsx +++ b/frontend/src/api/ApiCaller.tsx @@ -1,7 +1,6 @@ import { config } from 'config' import { Mission } from 'models/Mission' import { Robot } from 'models/Robot' -import { VideoStream } from 'models/VideoStream' import { filterRobots } from 'utils/filtersAndSorts' import { MissionRunQueryParameters } from 'models/MissionRunQueryParameters' import { MissionDefinitionQueryParameters } from 'models/MissionDefinitionQueryParameters' @@ -264,12 +263,6 @@ export class BackendAPICaller { return result.content } - static async getVideoStreamsByRobotId(robotId: string): Promise { - const path: string = 'robots/' + robotId + '/video-streams' - const result = await BackendAPICaller.GET(path).catch(BackendAPICaller.handleError('GET', path)) - return result.content - } - static async getPlantInfo(): Promise { const path: string = 'mission-loader/plants' const result = await BackendAPICaller.GET(path).catch(BackendAPICaller.handleError('GET', path)) diff --git a/frontend/src/components/Contexts/MediaStreamContext.tsx b/frontend/src/components/Contexts/MediaStreamContext.tsx new file mode 100644 index 000000000..a7d31ce07 --- /dev/null +++ b/frontend/src/components/Contexts/MediaStreamContext.tsx @@ -0,0 +1,113 @@ +import { createContext, FC, useContext, useEffect, useState } from 'react' +import { SignalREventLabels, useSignalRContext } from './SignalRContext' +import { useRobotContext } from './RobotContext' +import { + ConnectionState, + RemoteParticipant, + RemoteTrack, + RemoteTrackPublication, + Room, + RoomEvent, +} from 'livekit-client' +import { MediaConnectionType, MediaStreamConfig } from 'models/VideoStream' + +type MediaStreamDictionaryType = { + [robotId: string]: MediaStreamConfig & { streams: MediaStreamTrack[] } +} + +interface IMediaStreamContext { + mediaStreams: MediaStreamDictionaryType +} + +interface Props { + children: React.ReactNode +} + +const defaultMediaStreamInterface = { + mediaStreams: {}, +} + +export const MediaStreamContext = createContext(defaultMediaStreamInterface) + +export const MediaStreamProvider: FC = ({ children }) => { + const [mediaStreams, setMediaStreams] = useState( + defaultMediaStreamInterface.mediaStreams + ) + const { registerEvent, connectionReady } = useSignalRContext() + const { enabledRobots } = useRobotContext() + + const addTrackToConnection = (newTrack: MediaStreamTrack, robotId: string) => { + setMediaStreams((oldStreams) => { + if (!Object.keys(oldStreams).includes(robotId)) { + return oldStreams + } else { + const newStreams = { ...oldStreams } + return { + ...oldStreams, + [robotId]: { ...newStreams[robotId], streams: [...oldStreams[robotId].streams, newTrack] }, + } + } + }) + } + + const createLiveKitConnection = async (config: MediaStreamConfig) => { + const room = new Room() + room.on(RoomEvent.TrackSubscribed, handleTrackSubscribed) + + function handleTrackSubscribed( + track: RemoteTrack, + publication: RemoteTrackPublication, + participant: RemoteParticipant + ) { + addTrackToConnection(track.mediaStreamTrack, config.robotId) + } + if (room.state === ConnectionState.Disconnected) { + room.connect(config.url, config.token) + .then(() => console.log(JSON.stringify(room.state))) + .catch((error) => console.warn('Error connecting to LiveKit Room, may already be connected:', error)) + } + } + + const createMediaConnection = async (config: MediaStreamConfig) => { + switch (config.mediaConnectionType) { + case MediaConnectionType.LiveKit: + return await createLiveKitConnection(config) + default: + console.error('Invalid media connection type received') + } + return undefined + } + + // Register a signalR event handler that listens for new media stream connections + useEffect(() => { + if (connectionReady) { + registerEvent(SignalREventLabels.mediaStreamConfigReceived, (username: string, message: string) => { + const newMediaConfig: MediaStreamConfig = JSON.parse(message) + setMediaStreams((oldStreams) => { + if (Object.keys(oldStreams).includes(newMediaConfig.robotId)) { + return oldStreams + } else { + createMediaConnection(newMediaConfig) + return { + ...oldStreams, + [newMediaConfig.robotId]: { ...newMediaConfig, streams: [] }, + } + } + }) + }) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [registerEvent, connectionReady, enabledRobots]) + + return ( + + {children} + + ) +} + +export const useMediaStreamContext = () => useContext(MediaStreamContext) diff --git a/frontend/src/components/Contexts/SignalRContext.tsx b/frontend/src/components/Contexts/SignalRContext.tsx index dd11aca6c..f2859576b 100644 --- a/frontend/src/components/Contexts/SignalRContext.tsx +++ b/frontend/src/components/Contexts/SignalRContext.tsx @@ -122,4 +122,5 @@ export enum SignalREventLabels { robotDeleted = 'Robot deleted', inspectionUpdated = 'Inspection updated', alert = 'Alert', + mediaStreamConfigReceived = 'Media stream config received', } diff --git a/frontend/src/components/Pages/MissionPage/MissionPage.tsx b/frontend/src/components/Pages/MissionPage/MissionPage.tsx index dc34eb2c8..e9387c457 100644 --- a/frontend/src/components/Pages/MissionPage/MissionPage.tsx +++ b/frontend/src/components/Pages/MissionPage/MissionPage.tsx @@ -1,7 +1,6 @@ import { TaskTable } from 'components/Pages/MissionPage/TaskOverview/TaskTable' import { VideoStreamWindow } from 'components/Pages/MissionPage/VideoStream/VideoStreamWindow' import { Mission } from 'models/Mission' -import { VideoStream } from 'models/VideoStream' import { useEffect, useState } from 'react' import { useParams } from 'react-router-dom' import styled from 'styled-components' @@ -15,6 +14,7 @@ import { AlertType, useAlertContext } from 'components/Contexts/AlertContext' import { useLanguageContext } from 'components/Contexts/LanguageContext' import { FailedRequestAlertContent, FailedRequestAlertListContent } from 'components/Alerts/FailedRequestAlert' import { AlertCategory } from 'components/Alerts/AlertsBanner' +import { useMediaStreamContext } from 'components/Contexts/MediaStreamContext' const StyledMissionPage = styled.div` display: flex; @@ -32,7 +32,7 @@ const TaskAndMapSection = styled.div` padding-top: 16px; padding-bottom: 16px; ` -const VideoStreamSection = styled.div` +export const VideoStreamSection = styled.div` display: grid; gap: 1rem; ` @@ -41,9 +41,10 @@ export const MissionPage = () => { const { missionId } = useParams() const { TranslateText } = useLanguageContext() const { setAlert, setListAlert } = useAlertContext() - const [videoStreams, setVideoStreams] = useState([]) + const [videoMediaStreams, setVideoMediaStreams] = useState([]) const [selectedMission, setSelectedMission] = useState() const { registerEvent, connectionReady } = useSignalRContext() + const { mediaStreams } = useMediaStreamContext() useEffect(() => { if (connectionReady) { @@ -56,18 +57,18 @@ export const MissionPage = () => { }, [connectionReady]) useEffect(() => { - const updateVideoStreams = (mission: Mission) => - BackendAPICaller.getVideoStreamsByRobotId(mission.robot.id) - .then((streams) => setVideoStreams(streams)) - .catch((e) => { - console.warn(`Failed to get video stream with robot ID ${mission.robot.id}`) - }) + if (selectedMission && mediaStreams && Object.keys(mediaStreams).includes(selectedMission?.robot.id)) { + const mediaStreamConfig = mediaStreams[selectedMission?.robot.id] + if (mediaStreamConfig && mediaStreamConfig.streams.length > 0) + setVideoMediaStreams(mediaStreamConfig.streams) + } + }, [selectedMission, mediaStreams]) + useEffect(() => { if (missionId) BackendAPICaller.getMissionRunById(missionId) .then((mission) => { setSelectedMission(mission) - updateVideoStreams(mission) }) .catch((e) => { setAlert( @@ -101,7 +102,9 @@ export const MissionPage = () => { - {videoStreams.length > 0 && } + {videoMediaStreams && videoMediaStreams.length > 0 && ( + + )} )} diff --git a/frontend/src/components/Pages/MissionPage/VideoStream/FullScreenVideo.tsx b/frontend/src/components/Pages/MissionPage/VideoStream/FullScreenVideo.tsx index bda3442ec..caf206f32 100644 --- a/frontend/src/components/Pages/MissionPage/VideoStream/FullScreenVideo.tsx +++ b/frontend/src/components/Pages/MissionPage/VideoStream/FullScreenVideo.tsx @@ -1,10 +1,8 @@ -import { VideoPlayerOvenPlayer, isValidOvenPlayerType } from './VideoPlayerOvenPlayer' -import { VideoPlayerSimple } from './VideoPlayerSimple' -import { VideoStream } from 'models/VideoStream' import styled from 'styled-components' import { Typography, Button, Icon } from '@equinor/eds-core-react' import { tokens } from '@equinor/eds-tokens' import { Icons } from 'utils/icons' +import { VideoPlayerSimpleStream } from './VideoPlayerSimpleStream' const FullscreenExitButton = styled(Button)` position: absolute; @@ -28,26 +26,17 @@ const FullScreenCard = styled.div` padding: 1rem; ` -// Styles for rotation -const FullScreenCardRotated = styled.div` - transform: rotate(270deg); - padding: 1rem; -` -const RotateText = styled.div` - writing-mode: vertical-rl; -` - -const PositionText = styled.div` - display: flex; - flex-direction: row-reverse; -` - interface IFullScreenVideoStreamCardProps { - videoStream: VideoStream + videoStream: MediaStream + videoStreamName: string toggleFullScreenMode: VoidFunction } -export const FullScreenVideoStreamCard = ({ videoStream, toggleFullScreenMode }: IFullScreenVideoStreamCardProps) => { +export const FullScreenVideoStreamCard = ({ + videoStream, + videoStreamName, + toggleFullScreenMode, +}: IFullScreenVideoStreamCardProps) => { const cardWidth = () => { const availableInnerHeight = window.innerHeight - 9 * 16 const availableInnerWidth = window.innerWidth - 2 * 16 @@ -57,15 +46,6 @@ export const FullScreenVideoStreamCard = ({ videoStream, toggleFullScreenMode }: Math.min(coverageFactor * availableInnerWidth, aspectRatio * coverageFactor * availableInnerHeight) ) } - const rotatedCardWidth = () => { - const availableInnerHeight = window.innerHeight - 7.5 * 16 - const availableInnerWidth = window.innerWidth + 0.5 * 16 - const coverageFactor = 0.9 - const aspectRatio = 9 / 16 - return Math.round( - Math.min(coverageFactor * availableInnerHeight, aspectRatio * coverageFactor * availableInnerWidth) - ) - } const fullScreenExitButton = (shouldRotate270Clockwise: boolean) => { if (shouldRotate270Clockwise) { @@ -82,33 +62,11 @@ export const FullScreenVideoStreamCard = ({ videoStream, toggleFullScreenMode }: ) } - if (isValidOvenPlayerType(videoStream)) { - if (videoStream.shouldRotate270Clockwise) { - return ( - - - - {videoStream.name} - - - - {fullScreenExitButton(true)} - - ) - } - return ( - - {videoStream.name} - - {fullScreenExitButton(false)} - - ) - } // Rotated stream is not supported for simpleplayer return ( - {videoStream.name} - + {videoStreamName} + {fullScreenExitButton(false)} ) diff --git a/frontend/src/components/Pages/MissionPage/VideoStream/VideoPlayerOvenPlayer.tsx b/frontend/src/components/Pages/MissionPage/VideoStream/VideoPlayerOvenPlayer.tsx deleted file mode 100644 index 9c5fccb18..000000000 --- a/frontend/src/components/Pages/MissionPage/VideoStream/VideoPlayerOvenPlayer.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import { useEffect } from 'react' - -// import OvenPlayer from 'ovenplayer' - -// Styles -import 'video.js/dist/video-js.css' -import { VideoStream } from 'models/VideoStream' - -interface IVideoPlayerProps { - videoStream: VideoStream -} - -// TODO: Video player is not used at the moment, commented out for now -export const VideoPlayerOvenPlayer = ({ videoStream }: IVideoPlayerProps) => { - useEffect(() => { - // const aspectRatio = videoStream.shouldRotate270Clockwise ? '9:16' : '16:9' - switch (videoStream.type) { - case 'webrtc': - case 'hls': - case 'llhls': - case 'dash': - case 'lldash': - case 'mp4': - // const player = OvenPlayer.create(videoStream.id, { - // aspectRatio: aspectRatio, - // controls: false, - // mute: true, - // autoStart: true, - // expandFullScreenUI: false, - // sources: [ - // { - // label: videoStream.name, - // type: videoStream.type, - // file: videoStream.url, - // }, - // ], - // }) - } - }, [videoStream.id, videoStream.name, videoStream.shouldRotate270Clockwise, videoStream.type, videoStream.url]) - - return
-} - -export const isValidOvenPlayerType = (videoStream: VideoStream) => { - const validTypes = ['webrtc', 'hls', 'llhls', 'dash', 'lldash', 'mp4'] - return validTypes.includes(videoStream.type) -} diff --git a/frontend/src/components/Pages/MissionPage/VideoStream/VideoPlayerSimple.tsx b/frontend/src/components/Pages/MissionPage/VideoStream/VideoPlayerSimple.tsx deleted file mode 100644 index 8da14bfdb..000000000 --- a/frontend/src/components/Pages/MissionPage/VideoStream/VideoPlayerSimple.tsx +++ /dev/null @@ -1,9 +0,0 @@ -import { VideoStream } from 'models/VideoStream' - -interface IVideoPlayerProps { - videoStream: VideoStream -} - -export const VideoPlayerSimple = ({ videoStream }: IVideoPlayerProps) => { - return {videoStream.name -} diff --git a/frontend/src/components/Pages/MissionPage/VideoStream/VideoPlayerSimpleStream.tsx b/frontend/src/components/Pages/MissionPage/VideoStream/VideoPlayerSimpleStream.tsx new file mode 100644 index 000000000..43763ac3d --- /dev/null +++ b/frontend/src/components/Pages/MissionPage/VideoStream/VideoPlayerSimpleStream.tsx @@ -0,0 +1,15 @@ +interface IVideoPlayerProps { + videoStream: MediaStream + videoStreamName: string +} + +export const VideoPlayerSimpleStream = ({ videoStream, videoStreamName }: IVideoPlayerProps) => ( +