From dbe1fb5fb4d1342266d0fc1fb57b00a3c71ebf68 Mon Sep 17 00:00:00 2001 From: smasherprog Date: Sat, 30 Apr 2016 08:39:16 -0700 Subject: [PATCH 1/6] Working on Server Reciving Input Works for mouse Movements right now and probably button clicks. Working on web browser support for sending mouse inputs as well. --- Core/ClientNetworkDriver.cpp | 23 +++--------- Core/ClientNetworkDriver.h | 5 ++- Core/Core.vcxitems | 1 - Core/Core.vcxitems.filters | 4 --- Core/IClientDriver.h | 4 ++- Core/IServerDriver.h | 8 +++-- Core/Mouse.cpp | 64 +++++++++++++++++++++++---------- Core/Mouse.h | 25 +++++++++++-- Core/MouseInput.h | 16 --------- Core/Server.cpp | 9 ++--- Core/ServerNetworkDriver.cpp | 20 ++--------- Core/WebSocket.h | 9 +++-- Desktop_Client/ViewerWindow.cpp | 55 ++++++++++++++-------------- wwwroot/Core.ts | 34 +++++++++++++++++- 14 files changed, 155 insertions(+), 122 deletions(-) delete mode 100644 Core/MouseInput.h diff --git a/Core/ClientNetworkDriver.cpp b/Core/ClientNetworkDriver.cpp index 13434a11..926c8605 100644 --- a/Core/ClientNetworkDriver.cpp +++ b/Core/ClientNetworkDriver.cpp @@ -111,17 +111,9 @@ namespace SL { } } - void SendMouse(Utilities::Point& pos) { - Packet p(static_cast(PACKET_TYPES::MOUSEPOS), sizeof(pos)); - auto dst = (unsigned char*)p.Payload; - memcpy(dst, &pos, sizeof(pos)); - _Socket->send(p); - } - void SendMouse(Input::MouseEvents ev, Input::MousePress press) { - Packet p(static_cast(PACKET_TYPES::MOUSEEVENT), sizeof(press)+ sizeof(ev)); - auto dst = (unsigned char*)p.Payload; - *dst++ = ev; - *dst++ = press; + void SendMouse(const Input::MouseEvent& m) { + Packet p(static_cast(PACKET_TYPES::MOUSEEVENT), sizeof(m)); + memcpy(p.Payload, &m, sizeof(m)); _Socket->send(p); } }; @@ -150,12 +142,7 @@ void SL::Remote_Access_Library::Network::ClientNetworkDriver::Stop() _ClientNetworkDriverImpl->Stop(); } -void SL::Remote_Access_Library::Network::ClientNetworkDriver::SendMouse(Utilities::Point & pos) -{ - _ClientNetworkDriverImpl->SendMouse(pos); -} - -void SL::Remote_Access_Library::Network::ClientNetworkDriver::SendMouse(Input::MouseEvents ev, Input::MousePress press) +void SL::Remote_Access_Library::Network::ClientNetworkDriver::SendMouse(const Input::MouseEvent& m) { - _ClientNetworkDriverImpl->SendMouse(ev, press); + _ClientNetworkDriverImpl->SendMouse(m); } diff --git a/Core/ClientNetworkDriver.h b/Core/ClientNetworkDriver.h index d9372cf6..73824b59 100644 --- a/Core/ClientNetworkDriver.h +++ b/Core/ClientNetworkDriver.h @@ -1,7 +1,7 @@ #pragma once #include "IBaseNetworkDriver.h" #include -#include "MouseInput.h" +#include "Mouse.h" namespace SL { namespace Remote_Access_Library { @@ -24,8 +24,7 @@ namespace SL { //Before calling Stop, you must ensure that any external references to shared_ptr have been released void Stop(); - void SendMouse(Utilities::Point& pos); - void SendMouse(Input::MouseEvents ev, Input::MousePress press); + void SendMouse(const Input::MouseEvent& m); }; } } diff --git a/Core/Core.vcxitems b/Core/Core.vcxitems index d433bde2..fcc20725 100644 --- a/Core/Core.vcxitems +++ b/Core/Core.vcxitems @@ -92,7 +92,6 @@ - diff --git a/Core/Core.vcxitems.filters b/Core/Core.vcxitems.filters index 0cf28187..cbc9aa50 100644 --- a/Core/Core.vcxitems.filters +++ b/Core/Core.vcxitems.filters @@ -28,9 +28,6 @@ {486bb838-570a-425a-aaa3-9d059b12d301} - - {ccaac4c1-f65a-4e95-a385-603275123ebb} - @@ -232,7 +229,6 @@ Compression - diff --git a/Core/IClientDriver.h b/Core/IClientDriver.h index 792c1b46..6a5072ec 100644 --- a/Core/IClientDriver.h +++ b/Core/IClientDriver.h @@ -3,11 +3,12 @@ namespace SL { namespace Remote_Access_Library { + namespace Utilities { class Image; + class Point; } namespace Network { - class Rect; class IClientDriver : public IBaseNetworkDriver { public: IClientDriver() {} @@ -16,6 +17,7 @@ namespace SL { virtual void OnReceive_Image(const std::shared_ptr& socket,std::shared_ptr& img) = 0; virtual void OnReceive_MouseImage(const std::shared_ptr& socket, std::shared_ptr& img) = 0; virtual void OnReceive_MousePos(const std::shared_ptr& socket, Utilities::Point* pos) = 0; + }; } } diff --git a/Core/IServerDriver.h b/Core/IServerDriver.h index 750f4b2b..7e665cba 100644 --- a/Core/IServerDriver.h +++ b/Core/IServerDriver.h @@ -1,9 +1,12 @@ #pragma once #include "IBaseNetworkDriver.h" -#include "MouseInput.h" + namespace SL { namespace Remote_Access_Library { + namespace Input { + struct MouseEvent; + } namespace Utilities { class Point; } @@ -17,8 +20,7 @@ namespace SL { virtual ~IServerDriver() {} - virtual void OnMouse(Utilities::Point& pos) = 0; - virtual void OnMouse(Input::MouseEvents ev, Input::MousePress press) = 0; + virtual void OnMouse(Input::MouseEvent* m) = 0; }; } diff --git a/Core/Mouse.cpp b/Core/Mouse.cpp index 21ca8447..319831c3 100644 --- a/Core/Mouse.cpp +++ b/Core/Mouse.cpp @@ -5,6 +5,11 @@ #include #include "Logging.h" +#if __linux__ +#include +#include +#endif + namespace SL { namespace Remote_Access_Library @@ -127,10 +132,10 @@ namespace SL #error Applie specific implementation of CaptureMouse has not been written yet. You can help out by writing it! #elif __ANDROID__ - + std::shared_ptr CaptureMouseImage() { - //this should never be ran + //this should never be ran assert(true); return Utilities::Image::CreateImage(0, 0); } @@ -141,8 +146,7 @@ namespace SL return Utilities::Point(0, 0); } #elif __linux__ -#include -#include + std::shared_ptr CaptureMouseImage() { auto display = XOpenDisplay(NULL); @@ -181,33 +185,55 @@ namespace SL return Utilities::Point(x, y); } #endif - bool SetCursorPosition(Utilities::Point p) - { + + void SetMouseEvent(const Input::MouseEvent& m) { #if defined _WIN32 - return SetCursorPos(p.X, p.Y) == TRUE; + + INPUT input; + input.type = INPUT_MOUSE; + input.mi.mouseData = m.ScrollDelta / 120; + input.mi.dx = static_cast(static_cast(m.Pos.X)*(65536.0f / static_cast(GetSystemMetrics(SM_CXSCREEN))));//x being coord in pixels + input.mi.dy = static_cast(static_cast(m.Pos.Y)*(65536.0f / static_cast(GetSystemMetrics(SM_CYSCREEN))));//y being coord in pixels + input.mi.dwFlags = MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE; + + switch (m.EventData) { + case Input::MouseEvents::LEFT: + input.mi.dwFlags |= m.PressData == Input::MousePress::UP ? MOUSEEVENTF_LEFTUP : MOUSEEVENTF_LEFTDOWN; + break; + case Input::MouseEvents::MIDDLE: + input.mi.dwFlags |= m.PressData == Input::MousePress::UP ? MOUSEEVENTF_MIDDLEUP : MOUSEEVENTF_MIDDLEDOWN; + break; + case Input::MouseEvents::RIGHT: + input.mi.dwFlags |= m.PressData == Input::MousePress::UP ? MOUSEEVENTF_RIGHTUP : MOUSEEVENTF_RIGHTDOWN; + break; + case Input::MouseEvents::SCROLL: + input.mi.dwFlags |= MOUSEEVENTF_WHEEL; + break; + default: + break; + } + + SendInput(1, &input, sizeof(input)); + #elif defined __APPLE__ CGPoint new_pos; CGEventErr err; - new_pos.x = p.X; - new_pos.y = p.Y; - return !CGWarpMouseCursorPosition(new_pos); -#elif __ANDROID__ - return true; + new_pos.x = m.Pos.X; + new_pos.y = m.Pos.Y; + !CGWarpMouseCursorPosition(new_pos); #elif __linux__ -#include + auto display = XOpenDisplay(NULL); auto root = DefaultRootWindow(display); - - XWarpPointer(display, None, root, 0, 0, 0, 0, p.X, p.Y); - + XWarpPointer(display, None, root, 0, 0, 0, 0, m.Pos.X, m.Pos.Y); XCloseDisplay(display); - return true; -#else - return false; // Fail + #endif } + + } namespace INTERNAL { diff --git a/Core/Mouse.h b/Core/Mouse.h index 549cb313..9f37d33d 100644 --- a/Core/Mouse.h +++ b/Core/Mouse.h @@ -5,7 +5,26 @@ namespace SL { namespace Remote_Access_Library { - + namespace Input { + enum MouseEvents : unsigned char { + LEFT, + RIGHT, + MIDDLE, + SCROLL, + NO_EVENTDATA + }; + enum MousePress : unsigned char { + UP, + DOWN, + NO_PRESS_DATA + }; + struct MouseEvent { + MouseEvents EventData; + Utilities::Point Pos; + int ScrollDelta; + MousePress PressData; + }; + } namespace Utilities { class Image; } @@ -20,7 +39,9 @@ namespace SL { Mouse(std::function)> img_func, std::function pos_func, int img_dely = 1000, int pos_dely = 20); ~Mouse(); }; - bool SetCursorPosition(Utilities::Point p); + + void SetMouseEvent(const Input::MouseEvent& m); } + } } \ No newline at end of file diff --git a/Core/MouseInput.h b/Core/MouseInput.h deleted file mode 100644 index c4624152..00000000 --- a/Core/MouseInput.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -namespace SL { - namespace Remote_Access_Library { - namespace Input { - enum MouseEvents : unsigned char { - LEFT = 1, - RIGHT, - MIDDLE - }; - enum MousePress : unsigned char { - UP=MouseEvents::MIDDLE +1, DOWN - }; - } - } -} diff --git a/Core/Server.cpp b/Core/Server.cpp index 44b75d17..3377b54b 100644 --- a/Core/Server.cpp +++ b/Core/Server.cpp @@ -103,14 +103,9 @@ namespace SL { LastMouse = img; } - virtual void OnMouse(Utilities::Point& pos) override { - UNUSED(pos); + virtual void OnMouse(Input::MouseEvent* m) override { + Capturing::SetMouseEvent(*m); } - virtual void OnMouse(Input::MouseEvents ev, Input::MousePress press) override { - UNUSED(ev); - UNUSED(press); - } - int Run() { Status = Server_Status::SERVER_RUNNING; _ServerNetworkDriver.Start(); diff --git a/Core/ServerNetworkDriver.cpp b/Core/ServerNetworkDriver.cpp index 54cc3ef6..ecff39d2 100644 --- a/Core/ServerNetworkDriver.cpp +++ b/Core/ServerNetworkDriver.cpp @@ -13,7 +13,6 @@ #include "turbojpeg.h" #include "Mouse.h" #include -#include "MouseInput.h" namespace SL { namespace Remote_Access_Library { @@ -35,21 +34,9 @@ namespace SL { std::vector _CompressBuffer; - void MousePos(const std::shared_ptr& socket, std::shared_ptr& p) { - assert(p->Payload_Length == sizeof(Utilities::Point)); - Utilities::Point point; - memcpy( &point, p->Payload,sizeof(point)); - _IServerDriver->OnMouse(point); - } void MouseEvent(const std::shared_ptr& socket, std::shared_ptr& p) { - - assert(p->Payload_Length == sizeof(Input::MouseEvents::MIDDLE) + sizeof(Input::MousePress::UP)); - auto val((unsigned char*)p->Payload); - - Input::MouseEvents ev = static_cast(*val++); - Input::MousePress evp = static_cast(*val++); - - _IServerDriver->OnMouse(ev, evp); + assert(p->Payload_Length == sizeof(Input::MouseEvent)); + _IServerDriver->OnMouse((Input::MouseEvent*) p->Payload); } public: ServerNetworkDriverImpl(Server_Config& config, IServerDriver* svrd) : _IServerDriver(svrd), _Config(config) { @@ -73,9 +60,6 @@ namespace SL { { switch (p->Packet_Type) { - case static_cast(PACKET_TYPES::MOUSEPOS) : - MousePos(socket, p); - break; case static_cast(PACKET_TYPES::MOUSEEVENT) : MouseEvent(socket, p); break; diff --git a/Core/WebSocket.h b/Core/WebSocket.h index c2b86868..6c54468b 100644 --- a/Core/WebSocket.h +++ b/Core/WebSocket.h @@ -29,6 +29,7 @@ namespace SL { this->close_Socket("~WebSocket"); } virtual SocketTypes get_type() const override { return SocketTypes::WEBSOCKET; } + //function below is needed to accomidate the android compiler.. void Android_writeheader(std::shared_ptr packet) { this->writeexpire_from_now(this->_SocketImpl.writetimeout); auto self(this->shared_from_this()); @@ -143,6 +144,7 @@ namespace SL { asio::async_read(this->_socket, asio::buffer(_readheaderbuffer, 2), [this, self](const std::error_code& ec, size_t bytes_transferred) { UNUSED(bytes_transferred); if (!ec) { + assert(bytes_transferred == 2); this->_SocketImpl.ReadPacketHeader.Payload_Length = 0; _recv_fin_rsv_opcode = _readheaderbuffer[0]; //Close connection if unmasked message from client (protocol error) @@ -155,9 +157,9 @@ namespace SL { auto readbytes = (_readheaderbuffer[1] & 127) == 126 ? 2 : ((_readheaderbuffer[1] & 127) == 127 ? 8 : 0); if (readbytes != 0) { asio::async_read(this->_socket, asio::buffer(_readheaderbuffer, readbytes), [this, self, readbytes](const std::error_code& ec, size_t bytes_transferred) { - UNUSED(bytes_transferred); + if (!ec) { - + assert(readbytes == bytes_transferred); for (int c = 0; c < readbytes; c++) { this->_SocketImpl.ReadPacketHeader.Payload_Length += _readheaderbuffer[c] << (8 * (readbytes - 1 - c)); } @@ -188,8 +190,9 @@ namespace SL { auto p(this->_SocketImpl.get_ReadBuffer()); auto size(this->_SocketImpl.get_ReadBufferSize()); asio::async_read(this->_socket, asio::buffer(p, size), [this, self](const std::error_code& ec, size_t bytes_transferred) { - UNUSED(bytes_transferred); + if (!ec) { + assert(this->_SocketImpl.get_ReadBufferSize() == bytes_transferred); auto packet(this->_SocketImpl.GetNextReadPacket()); //If connection close if ((_recv_fin_rsv_opcode & 0x0f) == 8) { diff --git a/Desktop_Client/ViewerWindow.cpp b/Desktop_Client/ViewerWindow.cpp index af23f645..6d178771 100644 --- a/Desktop_Client/ViewerWindow.cpp +++ b/Desktop_Client/ViewerWindow.cpp @@ -17,6 +17,7 @@ #include #include #include "../Core/Logging.h" +#include namespace SL { namespace Remote_Access_Library { @@ -116,7 +117,7 @@ namespace SL { } } float GetScaleFactor() const { - if (_OriginalImage) { + if (_OriginalImage && _ScaleImage) { auto pheight = this->parent()->h() - SCROLLBARSIZE;//16 is the scrollbars size if (pheight < 0) pheight = 48;//cannot make image smaller than this.. return static_cast(pheight) / static_cast(_OriginalImage->Height()); @@ -184,16 +185,14 @@ namespace SL { switch (e) { case FL_PUSH: - handle_mousebutton(Fl::event_button(), true); - handle_mousemove(Fl::event_x(), Fl::event_y()); + handle_mouse(Fl::event_button(), Input::MousePress::DOWN, Fl::event_x(), Fl::event_y()); break; case FL_RELEASE: - handle_mousebutton(Fl::event_button(), false); - handle_mousemove(Fl::event_x(), Fl::event_y()); + handle_mouse(Fl::event_button(), Input::MousePress::UP, Fl::event_x(), Fl::event_y()); break; case FL_DRAG: case FL_MOVE: - handle_mousemove(Fl::event_x(), Fl::event_y()); + handle_mouse(Fl::event_button(), Input::MousePress::NO_PRESS_DATA, Fl::event_x(), Fl::event_y()); break; case FL_FOCUS: _HasFocus = true; @@ -204,28 +203,31 @@ namespace SL { }; return Fl_Window::handle(e); } - void handle_mousebutton(int button, bool pushed) { + void handle_mouse(int button, Input::MousePress press, int x, int y) { + + auto scale = _MyCanvas->GetScaleFactor(); + + Input::MouseEvent ev; + ev.Pos = Utilities::Point(static_cast(static_cast(x) / scale), static_cast(static_cast(y) / scale)); + ev.ScrollDelta = 0; + ev.PressData = press; + + if (press == Input::MousePress::NO_PRESS_DATA) button = 0; switch (button) { - case FL_LEFT_MOUSE: - _ClientNetworkDriver.SendMouse(Input::MouseEvents::LEFT, pushed ? Input::MousePress::DOWN : Input::MousePress::UP); - break; - case FL_MIDDLE_MOUSE: - _ClientNetworkDriver.SendMouse(Input::MouseEvents::MIDDLE, pushed ? Input::MousePress::DOWN : Input::MousePress::UP); - break; - case FL_RIGHT_MOUSE: - _ClientNetworkDriver.SendMouse(Input::MouseEvents::RIGHT, pushed ? Input::MousePress::DOWN : Input::MousePress::UP); - break; + case FL_LEFT_MOUSE: + ev.EventData = Input::MouseEvents::LEFT; + break; + case FL_MIDDLE_MOUSE: + ev.EventData = Input::MouseEvents::MIDDLE; + break; + case FL_RIGHT_MOUSE: + ev.EventData = Input::MouseEvents::RIGHT; + break; + default: + ev.EventData = Input::MouseEvents::NO_EVENTDATA; }; - } - void handle_mousemove(int x, int y) { - if (!_HasFocus && _CursorHidden) { - this->cursor(Fl_Cursor::FL_CURSOR_ARROW); - _CursorHidden = false; - } - auto scale = _MyCanvas->GetScaleFactor(); - _ClientNetworkDriver.SendMouse(Utilities::Point( - static_cast(static_cast(x) / scale), - static_cast(static_cast(y) / scale))); + + _ClientNetworkDriver.SendMouse(ev); } virtual ~ViewerWindowImpl() { _ClientNetworkDriver.Stop(); @@ -307,6 +309,7 @@ namespace SL { _MyCanvas->SetMouseImage(img); Fl::awake(awakenredraw, this); } + virtual void OnReceive_MousePos(const std::shared_ptr& socket, Utilities::Point* pos)override { _MyCanvas->SetMousePosition(*pos); } diff --git a/wwwroot/Core.ts b/wwwroot/Core.ts index 9031477d..64a92a56 100644 --- a/wwwroot/Core.ts +++ b/wwwroot/Core.ts @@ -21,6 +21,26 @@ module SL { } } } + export module Input { + enum MouseEvents { + LEFT, + RIGHT, + MIDDLE, + SCROLL, + NO_EVENTDATA + }; + enum MousePress { + UP, + DOWN, + NO_PRESS_DATA + }; + export class MouseEvent { + EventData: MouseEvents; + Pos: Utilities.Point; + ScrollDelta: number; + PressData: MousePress; + }; + } export module Network { enum PACKET_TYPES { INVALID, @@ -30,6 +50,7 @@ module SL { MOUSEPOS, MOUSEIMAGE, KEYEVENT, + MOUSEEVENT, //use LAST_PACKET_TYPE as the starting point of your custom packet types. Everything before this is used internally by the library LAST_PACKET_TYPE } @@ -66,10 +87,18 @@ module SL { constructor(private _Screen_Canvas_Id: string, private _Mouse_Canvas_Id: string) { window.addEventListener("resize", this.onresize); + window.addEventListener("click", this.onclick); + window.addEventListener("mousemove", this.onmove); } public ScaleView = (b: boolean): void => { this._ScaleImage = b; + } + onclick = (ev: MouseEvent): void => { + + } + onmove = (ev: MouseEvent): void => { + } onresize = (ev: UIEvent): void => { @@ -122,7 +151,7 @@ module SL { var i = new Image(); i.src = "data:image/jpeg;base64," + img; - + var self = this; i.onload = function () { var elem = document.getElementById(self._Screen_Canvas_Id); @@ -205,6 +234,9 @@ module SL { public Stop = (): void => { this._Socket.close(1001, "Web Browser called Stop()"); this._Socket = null; + } + public SendMouse = (m: Input.MouseEvent): void => { + } OnMessage = (ev: MessageEvent): void => { var t0 = performance.now(); From c608ff2a5c8ec99daaa2729f271f35c44957eef8 Mon Sep 17 00:00:00 2001 From: smasherprog Date: Mon, 2 May 2016 05:51:54 -0700 Subject: [PATCH 2/6] Updating typescript/html Cleaned up the public API for javascript users. Preping for sending mouse/keyboard input to server from web browser. --- wwwroot/Core.js | 129 +++++++++++++++++++++++++++++++++------------ wwwroot/Core.ts | 96 ++++++++++++++++++++++----------- wwwroot/index.html | 16 +----- 3 files changed, 160 insertions(+), 81 deletions(-) diff --git a/wwwroot/Core.js b/wwwroot/Core.js index da1fd588..6550ac7e 100644 --- a/wwwroot/Core.js +++ b/wwwroot/Core.js @@ -39,6 +39,32 @@ var SL; Utilities.Rect = Rect; })(Remote_Access_Library.Utilities || (Remote_Access_Library.Utilities = {})); var Utilities = Remote_Access_Library.Utilities; + (function (Input) { + var MouseEvents; + (function (MouseEvents) { + MouseEvents[MouseEvents["LEFT"] = 0] = "LEFT"; + MouseEvents[MouseEvents["RIGHT"] = 1] = "RIGHT"; + MouseEvents[MouseEvents["MIDDLE"] = 2] = "MIDDLE"; + MouseEvents[MouseEvents["SCROLL"] = 3] = "SCROLL"; + MouseEvents[MouseEvents["NO_EVENTDATA"] = 4] = "NO_EVENTDATA"; + })(MouseEvents || (MouseEvents = {})); + ; + var MousePress; + (function (MousePress) { + MousePress[MousePress["UP"] = 0] = "UP"; + MousePress[MousePress["DOWN"] = 1] = "DOWN"; + MousePress[MousePress["NO_PRESS_DATA"] = 2] = "NO_PRESS_DATA"; + })(MousePress || (MousePress = {})); + ; + var MouseEvent = (function () { + function MouseEvent() { + } + return MouseEvent; + })(); + Input.MouseEvent = MouseEvent; + ; + })(Remote_Access_Library.Input || (Remote_Access_Library.Input = {})); + var Input = Remote_Access_Library.Input; (function (Network) { var PACKET_TYPES; (function (PACKET_TYPES) { @@ -49,9 +75,10 @@ var SL; PACKET_TYPES[PACKET_TYPES["MOUSEPOS"] = 4] = "MOUSEPOS"; PACKET_TYPES[PACKET_TYPES["MOUSEIMAGE"] = 5] = "MOUSEIMAGE"; PACKET_TYPES[PACKET_TYPES["KEYEVENT"] = 6] = "KEYEVENT"; + PACKET_TYPES[PACKET_TYPES["MOUSEEVENT"] = 7] = "MOUSEEVENT"; //use LAST_PACKET_TYPE as the starting point of your custom packet types. Everything before this is used internally by the library - PACKET_TYPES[PACKET_TYPES["LAST_PACKET_TYPE"] = 7] = "LAST_PACKET_TYPE"; + PACKET_TYPES[PACKET_TYPES["LAST_PACKET_TYPE"] = 8] = "LAST_PACKET_TYPE"; })(PACKET_TYPES || (PACKET_TYPES = {})); var PacketHeader = (function () { function PacketHeader(d) { @@ -73,27 +100,61 @@ var SL; })(); Network.SocketStats = SocketStats; var ClientDriver = (function () { - function ClientDriver(_Screen_Canvas_Id, _Mouse_Canvas_Id) { + function ClientDriver(_dst_host, _dst_port) { var _this = this; - this._Screen_Canvas_Id = _Screen_Canvas_Id; - this._Mouse_Canvas_Id = _Mouse_Canvas_Id; + this._dst_host = _dst_host; + this._dst_port = _dst_port; this._ScaleImage = false; + this._DivRootId = 'SLRATROOTID123'; + this.Start = function () { + var testroot = document.getElementById(_this._DivRootId); + if (testroot !== null) { + document.removeChild(testroot); + } + _this._HTMLDivRoot = document.createElement('div'); + _this._HTMLDivRoot.id = _this._DivRootId; + _this._HTMLDivRoot.style.position = 'relative'; + _this._HTMLCanvasScreenImage = document.createElement('canvas'); + _this._HTMLCanvasScreenImage.style.position = 'absolute'; + _this._HTMLCanvasScreenImage.style.left = _this._HTMLCanvasScreenImage.style.top = _this._HTMLCanvasScreenImage.style.zIndex = '0'; + + _this._HTMLCanvasMouseImage = document.createElement('canvas'); + + _this._HTMLCanvasMouseImage.style.left = _this._HTMLCanvasMouseImage.style.top = '0'; + _this._HTMLCanvasMouseImage.style.zIndex = '1'; + + _this._HTMLDivRoot.appendChild(_this._HTMLCanvasScreenImage); + _this._HTMLDivRoot.appendChild(_this._HTMLCanvasMouseImage); + document.body.appendChild(_this._HTMLDivRoot); //add to the dom + _this._ClientNetworkDriver = new ClientNetworkDriver(_this, _this._dst_host, _this._dst_port); + _this._ClientNetworkDriver.Start(); + }; + this.Stop = function () { + _this._ClientNetworkDriver.Stop(); + _this._ClientNetworkDriver = null; + var testroot = document.getElementById(_this._DivRootId); + if (testroot !== null) { + document.removeChild(testroot); + } + }; this.ScaleView = function (b) { _this._ScaleImage = b; }; + this.onclick = function (ev) { + }; + this.onmove = function (ev) { + }; this.onresize = function (ev) { if (_this._ScaleImage && _this._OriginalImage != null) { - var elem = document.getElementById(_this._Screen_Canvas_Id); var scale = _this.GetScalingFactor(); - elem.width = _this._OriginalImage.width * scale; - elem.height = _this._OriginalImage.height * scale; - elem.getContext("2d").drawImage(_this._OriginalImage, 0, 0, elem.width, elem.height); + _this._HTMLCanvasScreenImage.width = _this._OriginalImage.width * scale; + _this._HTMLCanvasScreenImage.height = _this._OriginalImage.height * scale; + _this._HTMLCanvasScreenImage.getContext("2d").drawImage(_this._OriginalImage, 0, 0, _this._HTMLCanvasScreenImage.width, _this._HTMLCanvasScreenImage.height); } else if (!_this._ScaleImage && _this._OriginalImage != null) { - var elem = document.getElementById(_this._Screen_Canvas_Id); - if (elem.height != _this._OriginalImage.height || elem.width != _this._OriginalImage.width) { - elem.width = _this._OriginalImage.width; - elem.height = _this._OriginalImage.height; - elem.getContext("2d").drawImage(_this._OriginalImage, 0, 0); + if (_this._HTMLCanvasScreenImage.height != _this._OriginalImage.height || _this._HTMLCanvasScreenImage.width != _this._OriginalImage.width) { + _this._HTMLCanvasScreenImage.width = _this._OriginalImage.width; + _this._HTMLCanvasScreenImage.height = _this._OriginalImage.height; + _this._HTMLCanvasScreenImage.getContext("2d").drawImage(_this._OriginalImage, 0, 0); } } }; @@ -105,12 +166,11 @@ var SL; i.src = "data:image/jpeg;base64," + img; var self = _this; i.onload = function () { - var elem = document.getElementById(self._Screen_Canvas_Id); if (self._ScaleImage) { var scale = self.GetScalingFactor(); - elem.getContext("2d").drawImage(i, rect.Origin.X * scale, rect.Origin.Y * scale, rect.Width * scale, rect.Height * scale); + self._HTMLCanvasScreenImage.getContext("2d").drawImage(i, rect.Origin.X * scale, rect.Origin.Y * scale, rect.Width * scale, rect.Height * scale); } else { - elem.getContext("2d").drawImage(i, rect.Origin.X, rect.Origin.Y); + self._HTMLCanvasScreenImage.getContext("2d").drawImage(i, rect.Origin.X, rect.Origin.Y); } // console.log("ctx.drawImage" + coords.Y, " " + coords.X); }; @@ -128,17 +188,15 @@ var SL; var self = _this; i.onload = function () { - var elem = document.getElementById(self._Screen_Canvas_Id); - if (self._ScaleImage) { var scale = self.GetScalingFactor(); - elem.width = i.width * scale; - elem.height = i.height * scale; - elem.getContext("2d").drawImage(i, 0, 0, elem.width, elem.height); + self._HTMLCanvasScreenImage.width = i.width * scale; + self._HTMLCanvasScreenImage.height = i.height * scale; + self._HTMLCanvasScreenImage.getContext("2d").drawImage(i, 0, 0, self._HTMLCanvasScreenImage.width, self._HTMLCanvasScreenImage.height); } else { - elem.width = i.width; - elem.height = i.height; - elem.getContext("2d").drawImage(i, 0, 0); + self._HTMLCanvasScreenImage.width = i.width; + self._HTMLCanvasScreenImage.height = i.height; + self._HTMLCanvasScreenImage.getContext("2d").drawImage(i, 0, 0); } self._OriginalImage = i; // console.log("ctx.drawImage" + coords.Y, " " + coords.X); @@ -151,11 +209,10 @@ var SL; "use strict"; //console.log('coords' + coords.X + ' ' + coords.Y + ' ' + coords.Width + ' ' + coords.Height); - var elem = document.getElementById(_this._Mouse_Canvas_Id); - elem.width = point.X; - elem.height = point.Y; + _this._HTMLCanvasMouseImage.width = point.X; + _this._HTMLCanvasMouseImage.height = point.Y; try { - _this._Cursor = elem.getContext("2d").createImageData(point.X, point.Y); + _this._Cursor = _this._HTMLCanvasMouseImage.getContext("2d").createImageData(point.X, point.Y); for (var i = 0; i < _this._Cursor.data.length; i += 4) { _this._Cursor.data[i + 0] = img[i + 0]; @@ -163,23 +220,24 @@ var SL; _this._Cursor.data[i + 2] = img[i + 2]; _this._Cursor.data[i + 3] = img[i + 3]; } - elem.getContext("2d").putImageData(_this._Cursor, 0, 0); + _this._HTMLCanvasMouseImage.getContext("2d").putImageData(_this._Cursor, 0, 0); } catch (e) { console.log(e.message); } }; this.OnReceive_MousePos = function (socket, pos) { - var elem = document.getElementById(_this._Mouse_Canvas_Id); if (_this._ScaleImage) { var scale = _this.GetScalingFactor(); - elem.style.top = (pos.Y * scale) + "px"; - elem.style.left = (pos.X * scale) + "px"; + _this._HTMLCanvasMouseImage.style.top = (pos.Y * scale) + "px"; + _this._HTMLCanvasMouseImage.style.left = (pos.X * scale) + "px"; } else { - elem.style.top = pos.Y + "px"; - elem.style.left = pos.X + "px"; + _this._HTMLCanvasMouseImage.style.top = pos.Y + "px"; + _this._HTMLCanvasMouseImage.style.left = pos.X + "px"; } }; window.addEventListener("resize", this.onresize); + window.addEventListener("click", this.onclick); + window.addEventListener("mousemove", this.onmove); } ClientDriver.prototype.GetScalingFactor = function () { if (this._OriginalImage != null) { @@ -216,6 +274,8 @@ var SL; _this._Socket.close(1001, "Web Browser called Stop()"); _this._Socket = null; }; + this.SendMouse = function (m) { + }; this.OnMessage = function (ev) { var t0 = performance.now(); var packetheader = new PacketHeader(ev.data); @@ -326,7 +386,6 @@ var SL; }; return ClientNetworkDriver; })(); - Network.ClientNetworkDriver = ClientNetworkDriver; })(Remote_Access_Library.Network || (Remote_Access_Library.Network = {})); var Network = Remote_Access_Library.Network; })(SL.Remote_Access_Library || (SL.Remote_Access_Library = {})); diff --git a/wwwroot/Core.ts b/wwwroot/Core.ts index 64a92a56..80d4389e 100644 --- a/wwwroot/Core.ts +++ b/wwwroot/Core.ts @@ -84,13 +84,50 @@ module SL { _Cursor: ImageData; _ScaleImage = false; _OriginalImage: HTMLImageElement; + _ClientNetworkDriver: ClientNetworkDriver; + _HTMLDivRoot: HTMLDivElement; + _HTMLCanvasScreenImage: HTMLCanvasElement; + _HTMLCanvasMouseImage: HTMLCanvasElement; + + _DivRootId = 'SLRATROOTID123';//this is an id used internally to double check if a canvas has allready been inserted + + constructor(private _dst_host: string, private _dst_port: string) { - constructor(private _Screen_Canvas_Id: string, private _Mouse_Canvas_Id: string) { window.addEventListener("resize", this.onresize); window.addEventListener("click", this.onclick); window.addEventListener("mousemove", this.onmove); } + public Start = (): void => { + var testroot = document.getElementById(this._DivRootId); + if (testroot !== null) { + document.removeChild(testroot); + } + this._HTMLDivRoot = document.createElement('div'); + this._HTMLDivRoot.id = this._DivRootId; + this._HTMLDivRoot.style.position = 'relative'; + this._HTMLCanvasScreenImage = document.createElement('canvas'); + this._HTMLCanvasScreenImage.style.position = 'absolute'; + this._HTMLCanvasScreenImage.style.left = this._HTMLCanvasScreenImage.style.top = this._HTMLCanvasScreenImage.style.zIndex = '0'; + + this._HTMLCanvasMouseImage = document.createElement('canvas'); + + this._HTMLCanvasMouseImage.style.left = this._HTMLCanvasMouseImage.style.top = '0'; + this._HTMLCanvasMouseImage.style.zIndex = '1'; + this._HTMLDivRoot.appendChild(this._HTMLCanvasScreenImage); + this._HTMLDivRoot.appendChild(this._HTMLCanvasMouseImage); + document.body.appendChild(this._HTMLDivRoot);//add to the dom + this._ClientNetworkDriver = new ClientNetworkDriver(this, this._dst_host, this._dst_port); + this._ClientNetworkDriver.Start(); + } + public Stop = (): void => { + this._ClientNetworkDriver.Stop(); + this._ClientNetworkDriver = null; + var testroot = document.getElementById(this._DivRootId); + if (testroot !== null) { + document.removeChild(testroot); + } + } public ScaleView = (b: boolean): void => { this._ScaleImage = b; } @@ -103,17 +140,16 @@ module SL { onresize = (ev: UIEvent): void => { if (this._ScaleImage && this._OriginalImage != null) { - var elem = document.getElementById(this._Screen_Canvas_Id); + var scale = this.GetScalingFactor(); - elem.width = this._OriginalImage.width * scale; - elem.height = this._OriginalImage.height * scale; - elem.getContext("2d").drawImage(this._OriginalImage, 0, 0, elem.width, elem.height); + this._HTMLCanvasScreenImage.width = this._OriginalImage.width * scale; + this._HTMLCanvasScreenImage.height = this._OriginalImage.height * scale; + this._HTMLCanvasScreenImage.getContext("2d").drawImage(this._OriginalImage, 0, 0, this._HTMLCanvasScreenImage.width, this._HTMLCanvasScreenImage.height); } else if (!this._ScaleImage && this._OriginalImage != null) { - var elem = document.getElementById(this._Screen_Canvas_Id); - if (elem.height != this._OriginalImage.height || elem.width != this._OriginalImage.width) { - elem.width = this._OriginalImage.width; - elem.height = this._OriginalImage.height; - elem.getContext("2d").drawImage(this._OriginalImage, 0, 0); + if (this._HTMLCanvasScreenImage.height != this._OriginalImage.height || this._HTMLCanvasScreenImage.width != this._OriginalImage.width) { + this._HTMLCanvasScreenImage.width = this._OriginalImage.width; + this._HTMLCanvasScreenImage.height = this._OriginalImage.height; + this._HTMLCanvasScreenImage.getContext("2d").drawImage(this._OriginalImage, 0, 0); } } } @@ -131,12 +167,11 @@ module SL { i.src = "data:image/jpeg;base64," + img var self = this; i.onload = function () { - var elem = document.getElementById(self._Screen_Canvas_Id); if (self._ScaleImage) { var scale = self.GetScalingFactor(); - elem.getContext("2d").drawImage(i, rect.Origin.X * scale, rect.Origin.Y * scale, rect.Width * scale, rect.Height * scale); + self._HTMLCanvasScreenImage.getContext("2d").drawImage(i, rect.Origin.X * scale, rect.Origin.Y * scale, rect.Width * scale, rect.Height * scale); } else { - elem.getContext("2d").drawImage(i, rect.Origin.X, rect.Origin.Y); + self._HTMLCanvasScreenImage.getContext("2d").drawImage(i, rect.Origin.X, rect.Origin.Y); } // console.log("ctx.drawImage" + coords.Y, " " + coords.X); }; @@ -154,17 +189,15 @@ module SL { var self = this; i.onload = function () { - var elem = document.getElementById(self._Screen_Canvas_Id); - if (self._ScaleImage) { var scale = self.GetScalingFactor(); - elem.width = i.width * scale; - elem.height = i.height * scale; - elem.getContext("2d").drawImage(i, 0, 0, elem.width, elem.height); + self._HTMLCanvasScreenImage.width = i.width * scale; + self._HTMLCanvasScreenImage.height = i.height * scale; + self._HTMLCanvasScreenImage.getContext("2d").drawImage(i, 0, 0, self._HTMLCanvasScreenImage.width, self._HTMLCanvasScreenImage.height); } else { - elem.width = i.width; - elem.height = i.height; - elem.getContext("2d").drawImage(i, 0, 0); + self._HTMLCanvasScreenImage.width = i.width; + self._HTMLCanvasScreenImage.height = i.height; + self._HTMLCanvasScreenImage.getContext("2d").drawImage(i, 0, 0); } self._OriginalImage = i; // console.log("ctx.drawImage" + coords.Y, " " + coords.X); @@ -176,11 +209,11 @@ module SL { public OnReceive_MouseImage = (socket: WebSocket, point: Utilities.Point, img: Uint8Array): void => { "use strict"; //console.log('coords' + coords.X + ' ' + coords.Y + ' ' + coords.Width + ' ' + coords.Height); - var elem = document.getElementById(this._Mouse_Canvas_Id); - elem.width = point.X; - elem.height = point.Y; + + this._HTMLCanvasMouseImage.width = point.X; + this._HTMLCanvasMouseImage.height = point.Y; try { - this._Cursor = elem.getContext("2d").createImageData(point.X, point.Y); + this._Cursor = this._HTMLCanvasMouseImage.getContext("2d").createImageData(point.X, point.Y); for (var i = 0; i < this._Cursor.data.length; i += 4) { this._Cursor.data[i + 0] = img[i + 0]; @@ -188,27 +221,26 @@ module SL { this._Cursor.data[i + 2] = img[i + 2]; this._Cursor.data[i + 3] = img[i + 3]; } - elem.getContext("2d").putImageData(this._Cursor, 0, 0); + this._HTMLCanvasMouseImage.getContext("2d").putImageData(this._Cursor, 0, 0); } catch (e) { console.log(e.message); } } public OnReceive_MousePos = (socket: WebSocket, pos: Utilities.Point): void => { - var elem = document.getElementById(this._Mouse_Canvas_Id); if (this._ScaleImage) { var scale = this.GetScalingFactor(); - elem.style.top = (pos.Y * scale) + "px"; - elem.style.left = (pos.X * scale) + "px"; + this._HTMLCanvasMouseImage.style.top = (pos.Y * scale) + "px"; + this._HTMLCanvasMouseImage.style.left = (pos.X * scale) + "px"; } else { - elem.style.top = pos.Y + "px"; - elem.style.left = pos.X + "px"; + this._HTMLCanvasMouseImage.style.top = pos.Y + "px"; + this._HTMLCanvasMouseImage.style.left = pos.X + "px"; } } } - export class ClientNetworkDriver { + class ClientNetworkDriver { _Socket: WebSocket; _SocketStats: SocketStats; _TotalMemoryUsed: number; diff --git a/wwwroot/index.html b/wwwroot/index.html index 463a9296..04d0de24 100644 --- a/wwwroot/index.html +++ b/wwwroot/index.html @@ -14,27 +14,15 @@ } document.addEventListener("DOMContentLoaded", function (event) { if ("WebSocket" in window) { - - var clientdriver = new SL.Remote_Access_Library.Network.ClientDriver("myCanvas", "mousecanvas"); + var clientdriver = new SL.Remote_Access_Library.Network.ClientDriver(window.location.hostname, "6001"); clientdriver.ScaleView(getURLParameter('ScaleImage') === 'true'); - - SL_Client = new SL.Remote_Access_Library.Network.ClientNetworkDriver(clientdriver, window.location.hostname, "6001"); - - - SL_Client.Start(); + clientdriver.Start(); } else { // The browser doesn't support WebSocket alert("WebSocket NOT supported by your Browser!"); } }); - - -
- - -
- From 5ee008cf1078ae2f0f1df17b224041bd06e685a0 Mon Sep 17 00:00:00 2001 From: smasherprog Date: Wed, 4 May 2016 19:32:41 -0700 Subject: [PATCH 3/6] Sending Mouse input from browser --- Core/ClientNetworkDriver.cpp | 3 +- Core/Core.vcxitems | 7 +- Core/Core.vcxitems.filters | 6 +- Core/HttpHeader.cpp | 2 + Core/Mouse.cpp | 3 +- Core/Server.cpp | 17 +-- Core/ServerNetworkDriver.cpp | 3 +- Core/WebSocketListener.cpp | 1 - Desktop_Client/ViewerWindow.cpp | 54 +++++---- Desktop_Server/Desktop_Server.vcxproj | 22 ++-- Desktop_Server/packages.config | 8 +- wwwroot/Core.js | 155 ++++++++++++++++++++++---- wwwroot/Core.ts | 151 ++++++++++++++++++++----- 13 files changed, 331 insertions(+), 101 deletions(-) diff --git a/Core/ClientNetworkDriver.cpp b/Core/ClientNetworkDriver.cpp index 926c8605..3f43d544 100644 --- a/Core/ClientNetworkDriver.cpp +++ b/Core/ClientNetworkDriver.cpp @@ -19,13 +19,12 @@ namespace SL { std::shared_ptr> _Socket; std::unique_ptr _IO_Runner; std::string _dst_host, _dst_port; - - void MouseImage(const std::shared_ptr& socket, std::shared_ptr& p) { auto imgsize = (Utilities::Point*)p->Payload; auto img(Utilities::Image::CreateImage(imgsize->Y, imgsize->X, p->Payload + sizeof(Utilities::Rect), 4)); _IClientDriver->OnReceive_MouseImage(socket, img); } + void MousePos(const std::shared_ptr& socket, std::shared_ptr& p) { assert(p->Payload_Length == sizeof(Utilities::Point)); _IClientDriver->OnReceive_MousePos(socket, (Utilities::Point*)p->Payload); diff --git a/Core/Core.vcxitems b/Core/Core.vcxitems index fcc20725..c17e75ab 100644 --- a/Core/Core.vcxitems +++ b/Core/Core.vcxitems @@ -107,7 +107,12 @@ - + + Document + tsc $(SolutionDir)wwwroot\Core.ts + $(OutDir)wwwroot\Core.js + true + diff --git a/Core/Core.vcxitems.filters b/Core/Core.vcxitems.filters index cbc9aa50..9d4a76b2 100644 --- a/Core/Core.vcxitems.filters +++ b/Core/Core.vcxitems.filters @@ -234,8 +234,8 @@ wwwroot - - wwwroot - + + + \ No newline at end of file diff --git a/Core/HttpHeader.cpp b/Core/HttpHeader.cpp index f1b03829..3eb17b27 100644 --- a/Core/HttpHeader.cpp +++ b/Core/HttpHeader.cpp @@ -14,6 +14,7 @@ namespace SL { { if (in[i] == '%') { + if (i + 3 <= in.size()) { int value = 0; @@ -25,6 +26,7 @@ namespace SL { } else { + return std::string("/"); } } diff --git a/Core/Mouse.cpp b/Core/Mouse.cpp index 319831c3..d6ac9175 100644 --- a/Core/Mouse.cpp +++ b/Core/Mouse.cpp @@ -188,6 +188,7 @@ namespace SL void SetMouseEvent(const Input::MouseEvent& m) { + //SL_RAT_LOG(std::string("SetMouseEvent EventData:") + std::to_string(m.EventData) + std::string(" ScrollDelta: ") + std::to_string(m.ScrollDelta) + std::string(" PressData: ") + std::to_string(m.PressData), Utilities::Logging_Levels::INFO_log_level); #if defined _WIN32 INPUT input; @@ -214,7 +215,7 @@ namespace SL break; } - SendInput(1, &input, sizeof(input)); + //SendInput(1, &input, sizeof(input)); #elif defined __APPLE__ CGPoint new_pos; diff --git a/Core/Server.cpp b/Core/Server.cpp index 3377b54b..81d24c7f 100644 --- a/Core/Server.cpp +++ b/Core/Server.cpp @@ -58,20 +58,15 @@ namespace SL { void OnScreen(std::shared_ptr img) { - if (!LastScreen) {//first screen send all! - _ServerNetworkDriver.SendScreenFull(nullptr, *img); + if (!_NewClients.empty()) { + //make sure to send the full screens to any new connects std::lock_guard lock(_NewClientLock); + for (auto& a : _NewClients) { + _ServerNetworkDriver.SendScreenFull(a.get(), *img); + } _NewClients.clear(); } - else {//compare and send all difs along - { - //make sure to send the full screens to any new connects - std::lock_guard lock(_NewClientLock); - for (auto& a : _NewClients) { - _ServerNetworkDriver.SendScreenFull(a.get(), *img); - } - _NewClients.clear(); - } + if (LastScreen) { if (img->data() != LastScreen->data()) { for (auto r : SL::Remote_Access_Library::Utilities::Image::GetDifs(*LastScreen, *img)) { _ServerNetworkDriver.SendScreenDif(nullptr, r, *img); diff --git a/Core/ServerNetworkDriver.cpp b/Core/ServerNetworkDriver.cpp index ecff39d2..723ff610 100644 --- a/Core/ServerNetworkDriver.cpp +++ b/Core/ServerNetworkDriver.cpp @@ -35,7 +35,7 @@ namespace SL { void MouseEvent(const std::shared_ptr& socket, std::shared_ptr& p) { - assert(p->Payload_Length == sizeof(Input::MouseEvent)); + assert(p->Payload_Length == sizeof(Input::MouseEvent::EventData) + sizeof(Input::MouseEvent::Pos) + sizeof(Input::MouseEvent::ScrollDelta) + sizeof(Input::MouseEvent::PressData)); _IServerDriver->OnMouse((Input::MouseEvent*) p->Payload); } public: @@ -58,7 +58,6 @@ namespace SL { virtual void OnReceive(const std::shared_ptr& socket, std::shared_ptr& p) override { - switch (p->Packet_Type) { case static_cast(PACKET_TYPES::MOUSEEVENT) : MouseEvent(socket, p); diff --git a/Core/WebSocketListener.cpp b/Core/WebSocketListener.cpp index 88e4e8e1..f25f8643 100644 --- a/Core/WebSocketListener.cpp +++ b/Core/WebSocketListener.cpp @@ -23,7 +23,6 @@ namespace SL { virtual ~WebSocketListinerImpl() { Stop(); } - virtual void OnConnect(const std::shared_ptr& socket) override { SL_RAT_LOG("websocket OnConnect", Utilities::Logging_Levels::INFO_log_level); socket->set_ReadTimeout(_config.Read_Timeout); diff --git a/Desktop_Client/ViewerWindow.cpp b/Desktop_Client/ViewerWindow.cpp index 6d178771..90fd7868 100644 --- a/Desktop_Client/ViewerWindow.cpp +++ b/Desktop_Client/ViewerWindow.cpp @@ -185,14 +185,10 @@ namespace SL { switch (e) { case FL_PUSH: - handle_mouse(Fl::event_button(), Input::MousePress::DOWN, Fl::event_x(), Fl::event_y()); + handle_mouse(e, Fl::event_button(), Input::MousePress::DOWN, Fl::event_x(), Fl::event_y()); break; case FL_RELEASE: - handle_mouse(Fl::event_button(), Input::MousePress::UP, Fl::event_x(), Fl::event_y()); - break; - case FL_DRAG: - case FL_MOVE: - handle_mouse(Fl::event_button(), Input::MousePress::NO_PRESS_DATA, Fl::event_x(), Fl::event_y()); + handle_mouse(e, Fl::event_button(), Input::MousePress::UP, Fl::event_x(), Fl::event_y()); break; case FL_FOCUS: _HasFocus = true; @@ -200,32 +196,46 @@ namespace SL { case FL_UNFOCUS: _HasFocus = false; break; + case FL_DRAG: + case FL_MOUSEWHEEL: + case FL_MOVE: + handle_mouse(e, Fl::event_button(), Input::MousePress::NO_PRESS_DATA, Fl::event_x(), Fl::event_y()); + break; + default: + break; }; return Fl_Window::handle(e); } - void handle_mouse(int button, Input::MousePress press, int x, int y) { + void handle_mouse(int e, int button, Input::MousePress press, int x, int y) { auto scale = _MyCanvas->GetScaleFactor(); Input::MouseEvent ev; ev.Pos = Utilities::Point(static_cast(static_cast(x) / scale), static_cast(static_cast(y) / scale)); - ev.ScrollDelta = 0; - ev.PressData = press; + if (e == FL_MOUSEWHEEL) { + ev.ScrollDelta = Fl::event_dy(); - if (press == Input::MousePress::NO_PRESS_DATA) button = 0; + } else { + ev.ScrollDelta = 0; + } + + ev.PressData = press; + + switch (button) { - case FL_LEFT_MOUSE: - ev.EventData = Input::MouseEvents::LEFT; - break; - case FL_MIDDLE_MOUSE: - ev.EventData = Input::MouseEvents::MIDDLE; - break; - case FL_RIGHT_MOUSE: - ev.EventData = Input::MouseEvents::RIGHT; - break; - default: - ev.EventData = Input::MouseEvents::NO_EVENTDATA; - }; + case FL_LEFT_MOUSE: + ev.EventData = Input::MouseEvents::LEFT; + break; + case FL_MIDDLE_MOUSE: + ev.EventData = Input::MouseEvents::MIDDLE; + break; + case FL_RIGHT_MOUSE: + ev.EventData = Input::MouseEvents::RIGHT; + break; + default: + ev.EventData = Input::MouseEvents::NO_EVENTDATA; + break; + }; _ClientNetworkDriver.SendMouse(ev); } diff --git a/Desktop_Server/Desktop_Server.vcxproj b/Desktop_Server/Desktop_Server.vcxproj index d99d44dc..bf6fa8f1 100644 --- a/Desktop_Server/Desktop_Server.vcxproj +++ b/Desktop_Server/Desktop_Server.vcxproj @@ -149,6 +149,7 @@ true MultiThreadedDebugDLL NotSet + Default Console @@ -165,6 +166,7 @@ true MultiThreadedDebugDLL NotSet + Default Console @@ -181,6 +183,7 @@ true MultiThreadedDebugDLL NotSet + Default Console @@ -199,6 +202,7 @@ true MultiThreadedDLL StreamingSIMDExtensions2 + 16Bytes Console @@ -219,6 +223,7 @@ true MultiThreadedDLL StreamingSIMDExtensions2 + 16Bytes Console @@ -239,6 +244,7 @@ true MultiThreadedDLL StreamingSIMDExtensions2 + 16Bytes Console @@ -260,13 +266,13 @@ - - - - + + + + @@ -276,12 +282,12 @@ - - - - + + + + \ No newline at end of file diff --git a/Desktop_Server/packages.config b/Desktop_Server/packages.config index 655b1bb4..293d738c 100644 --- a/Desktop_Server/packages.config +++ b/Desktop_Server/packages.config @@ -7,8 +7,8 @@ - - - - + + + + \ No newline at end of file diff --git a/wwwroot/Core.js b/wwwroot/Core.js index 6550ac7e..a1db2eb6 100644 --- a/wwwroot/Core.js +++ b/wwwroot/Core.js @@ -17,6 +17,11 @@ var SL; var arr = new Int32Array(data.slice(0, this.sizeof()).buffer); return new Point(arr[0], arr[1]); }; + Point.prototype.Fill = function (d, offset) { + var dt = new DataView(d, offset); + dt.setInt32(0, this.X, true); + dt.setInt32(4, this.Y, true); + }; return Point; })(); Utilities.Point = Point; @@ -34,31 +39,58 @@ var SL; var arr = new Int32Array(data.slice(0, this.sizeof()).buffer); return new Rect(new Point(arr[0], arr[1]), arr[2], arr[3]); }; + Rect.prototype.Fill = function (d, offset) { + this.Origin.Fill(d, offset); + var dt = new DataView(d, offset + Point.sizeof()); + dt.setInt32(0, this.Height, true); + dt.setInt32(4, this.Width, true); + }; return Rect; })(); Utilities.Rect = Rect; })(Remote_Access_Library.Utilities || (Remote_Access_Library.Utilities = {})); var Utilities = Remote_Access_Library.Utilities; (function (Input) { - var MouseEvents; (function (MouseEvents) { MouseEvents[MouseEvents["LEFT"] = 0] = "LEFT"; MouseEvents[MouseEvents["RIGHT"] = 1] = "RIGHT"; MouseEvents[MouseEvents["MIDDLE"] = 2] = "MIDDLE"; MouseEvents[MouseEvents["SCROLL"] = 3] = "SCROLL"; MouseEvents[MouseEvents["NO_EVENTDATA"] = 4] = "NO_EVENTDATA"; - })(MouseEvents || (MouseEvents = {})); + })(Input.MouseEvents || (Input.MouseEvents = {})); + var MouseEvents = Input.MouseEvents; ; - var MousePress; (function (MousePress) { MousePress[MousePress["UP"] = 0] = "UP"; MousePress[MousePress["DOWN"] = 1] = "DOWN"; MousePress[MousePress["NO_PRESS_DATA"] = 2] = "NO_PRESS_DATA"; - })(MousePress || (MousePress = {})); + })(Input.MousePress || (Input.MousePress = {})); + var MousePress = Input.MousePress; ; var MouseEvent = (function () { - function MouseEvent() { + function MouseEvent(EventData, Pos, ScrollDelta, PressData) { + if (typeof EventData === "undefined") { EventData = 4 /* NO_EVENTDATA */; } + if (typeof Pos === "undefined") { Pos = new Utilities.Point(0, 0); } + if (typeof ScrollDelta === "undefined") { ScrollDelta = 0; } + if (typeof PressData === "undefined") { PressData = 2 /* NO_PRESS_DATA */; } + this.EventData = EventData; + this.Pos = Pos; + this.ScrollDelta = ScrollDelta; + this.PressData = PressData; } + MouseEvent.sizeof = function () { + return 1 + Utilities.Point.sizeof() + 4 + 1; + }; + MouseEvent.prototype.Fill = function (d, offset) { + var dt = new DataView(d, offset); + dt.setUint8(0, this.EventData); + offset += 1; + this.Pos.Fill(d, offset); + offset += Utilities.Point.sizeof(); + dt.setInt32(offset, this.ScrollDelta, true); + offset += 4; + dt.setUint8(offset, this.PressData); + }; return MouseEvent; })(); Input.MouseEvent = MouseEvent; @@ -66,7 +98,6 @@ var SL; })(Remote_Access_Library.Input || (Remote_Access_Library.Input = {})); var Input = Remote_Access_Library.Input; (function (Network) { - var PACKET_TYPES; (function (PACKET_TYPES) { PACKET_TYPES[PACKET_TYPES["INVALID"] = 0] = "INVALID"; PACKET_TYPES[PACKET_TYPES["HTTP_MSG"] = 1] = "HTTP_MSG"; @@ -79,17 +110,27 @@ var SL; //use LAST_PACKET_TYPE as the starting point of your custom packet types. Everything before this is used internally by the library PACKET_TYPES[PACKET_TYPES["LAST_PACKET_TYPE"] = 8] = "LAST_PACKET_TYPE"; - })(PACKET_TYPES || (PACKET_TYPES = {})); + })(Network.PACKET_TYPES || (Network.PACKET_TYPES = {})); + var PACKET_TYPES = Network.PACKET_TYPES; var PacketHeader = (function () { - function PacketHeader(d) { - var data = new DataView(d); - this.Packet_Type = data.getInt32(0, true); - this.Payload_Length = data.getInt32(4, true); - this.UncompressedLength = data.getInt32(8, true); + function PacketHeader(Packet_Type, Payload_Length, UncompressedLength) { + if (typeof Packet_Type === "undefined") { Packet_Type = 0 /* INVALID */; } + if (typeof Payload_Length === "undefined") { Payload_Length = 0; } + if (typeof UncompressedLength === "undefined") { UncompressedLength = 0; } + this.Packet_Type = Packet_Type; + this.Payload_Length = Payload_Length; + this.UncompressedLength = UncompressedLength; + this.Payload = new ArrayBuffer(this.Payload_Length); } PacketHeader.prototype.sizeof = function () { return 12; }; + + PacketHeader.prototype.Fill = function (arr) { + arr[0] = this.Packet_Type; + arr[1] = this.Payload_Length; + arr[2] = this.UncompressedLength; + }; return PacketHeader; })(); Network.PacketHeader = PacketHeader; @@ -140,9 +181,40 @@ var SL; this.ScaleView = function (b) { _this._ScaleImage = b; }; - this.onclick = function (ev) { + this.onmousedown = function (ev) { + _this.handlemouse(ev.button, 1 /* DOWN */, ev.clientX, ev.clientY); + }; + this.onmouseup = function (ev) { + _this.handlemouse(ev.button, 0 /* UP */, ev.clientX, ev.clientY); }; this.onmove = function (ev) { + _this.handlemouse(-1, 2 /* NO_PRESS_DATA */, ev.clientX, ev.clientY); + }; + this.handlemouse = function (button, press, x, y) { + var ev = new Input.MouseEvent(); + var scale = _this.GetScalingFactor(); + ev.Pos.X = x / scale; + ev.Pos.Y = y / scale; + ev.ScrollDelta = 0; + + ev.PressData = press; + + switch (button) { + case 0: + ev.EventData = 0 /* LEFT */; + break; + case 1: + ev.EventData = 2 /* MIDDLE */; + break; + case 2: + ev.EventData = 1 /* RIGHT */; + break; + default: + ev.EventData = 4 /* NO_EVENTDATA */; + break; + } + ; + _this._ClientNetworkDriver.SendMouse(ev); }; this.onresize = function (ev) { if (_this._ScaleImage && _this._OriginalImage != null) { @@ -159,6 +231,8 @@ var SL; } }; this.OnReceive_ImageDif = function (socket, rect, img) { + if (_this._OriginalImage === null) + return; "use strict"; //console.log('coords' + coords.X + ' ' + coords.Y + ' ' + coords.Width + ' ' + coords.Height); @@ -236,11 +310,12 @@ var SL; } }; window.addEventListener("resize", this.onresize); - window.addEventListener("click", this.onclick); + window.addEventListener("mousedown", this.onmousedown); + window.addEventListener("mouseup", this.onmouseup); window.addEventListener("mousemove", this.onmove); } ClientDriver.prototype.GetScalingFactor = function () { - if (this._OriginalImage != null) { + if (this._ScaleImage && this._OriginalImage != null) { return window.innerHeight / this._OriginalImage.height; } else { return 1.0; @@ -275,10 +350,50 @@ var SL; _this._Socket = null; }; this.SendMouse = function (m) { + var pac = new PacketHeader(7 /* MOUSEEVENT */, Input.MouseEvent.sizeof(), Input.MouseEvent.sizeof()); + m.Fill(pac.Payload, 0); + _this.Compress_and_Send(pac); + }; + this.Compress_and_Send = function (p) { + var t0 = performance.now(); + + var srcPtr = Module._malloc(p.Payload_Length); + _this._TotalMemoryUsed += p.Payload_Length; + var srcbuff = new Uint8Array(Module.HEAPU8.buffer, srcPtr, p.Payload_Length); + srcbuff.set(new Uint8Array(p.Payload, p.sizeof())); //copy the data to the newly allocated memory + + var dstsize = _ZSTD_compressBound(p.UncompressedLength + p.sizeof()); + var dsttr = Module._malloc(dstsize); + _this._TotalMemoryUsed += dstsize; + var dstbuff = new Uint8Array(Module.HEAPU8.buffer, dsttr, dstsize); + + p.Payload_Length = _ZSTD_compress(dstbuff.byteOffset + p.sizeof(), dstsize - p.sizeof(), srcbuff.byteOffset, p.Payload_Length, 3); + if (_ZSTD_isError(p.Payload_Length) > 0) { + console.log('zstd error' + _ZSTD_getErrorName(p.Payload_Length)); + } + + p.Fill(new Uint32Array(Module.HEAPU8.buffer, dsttr, p.sizeof())); + + var t1 = performance.now(); + + //comment this line out to see performance issues... My machine takes 0 to 6 ms to complete each receive + // console.log("took " + (t1 - t0) + " milliseconds to Compress the packet") + Module._free(srcPtr); + _this._TotalMemoryUsed -= p.Payload_Length; + var dstsendbuff = new Uint8Array(Module.HEAPU8.buffer, dsttr, p.UncompressedLength + p.sizeof()); + + // this._Socket.send(dstsendbuff); + Module._free(dsttr); + _this._TotalMemoryUsed -= dstsize; }; this.OnMessage = function (ev) { var t0 = performance.now(); - var packetheader = new PacketHeader(ev.data); + + var packetheader = new PacketHeader(); + var data = new DataView(ev.data); + packetheader.Packet_Type = data.getInt32(0, true); + packetheader.Payload_Length = data.getInt32(4, true); + packetheader.UncompressedLength = data.getInt32(8, true); var srcPtr = Module._malloc(packetheader.Payload_Length); _this._TotalMemoryUsed += packetheader.Payload_Length; @@ -296,7 +411,7 @@ var SL; var t1 = performance.now(); //comment this line out to see performance issues... My machine takes 0 to 6 ms to complete each receive - console.log("took " + (t1 - t0) + " milliseconds to Decompress the receive loop"); + // console.log("took " + (t1 - t0) + " milliseconds to Decompress the receive loop") t0 = performance.now(); switch (packetheader.Packet_Type) { case (2 /* SCREENIMAGE */): @@ -321,9 +436,8 @@ var SL; Module._free(srcPtr); _this._TotalMemoryUsed -= packetheader.Payload_Length; t1 = performance.now(); - //comment this line out to see performance issues... My machine takes 0 to 6 ms to complete each receive - console.log("took " + (t1 - t0) + " milliseconds to process the receive loop"); + // console.log("took " + (t1 - t0) + " milliseconds to process the receive loop"); }; this.Image = function (data) { _this._IClientDriver.OnReceive_Image(_this._Socket, Utilities.Rect.FromArray(data), _this._arrayBufferToBase64(data, Utilities.Rect.sizeof())); @@ -332,7 +446,7 @@ var SL; _this._IClientDriver.OnReceive_ImageDif(_this._Socket, Utilities.Rect.FromArray(data), _this._arrayBufferToBase64(data, Utilities.Rect.sizeof())); }; this.MouseImage = function (data) { - _this._IClientDriver.OnReceive_MouseImage(_this._Socket, Utilities.Point.FromArray(data), new Uint8Array(data.slice(Utilities.Point.sizeof()).buffer)); + _this._IClientDriver.OnReceive_MouseImage(_this._Socket, Utilities.Point.FromArray(data), new Uint8Array(data.buffer, Utilities.Point.sizeof())); }; this.MousePos = function (data) { _this._IClientDriver.OnReceive_MousePos(_this._Socket, Utilities.Point.FromArray(data)); @@ -386,6 +500,7 @@ var SL; }; return ClientNetworkDriver; })(); + Network.ClientNetworkDriver = ClientNetworkDriver; })(Remote_Access_Library.Network || (Remote_Access_Library.Network = {})); var Network = Remote_Access_Library.Network; })(SL.Remote_Access_Library || (SL.Remote_Access_Library = {})); diff --git a/wwwroot/Core.ts b/wwwroot/Core.ts index 80d4389e..f93182fe 100644 --- a/wwwroot/Core.ts +++ b/wwwroot/Core.ts @@ -11,6 +11,11 @@ module SL { var arr = new Int32Array(data.slice(0, this.sizeof()).buffer); return new Point(arr[0], arr[1]); } + Fill(d: ArrayBuffer, offset: number): void { + var dt = new DataView(d, offset); + dt.setInt32(0, this.X, true); + dt.setInt32(4, this.Y, true); + } } export class Rect { static sizeof(): number { return 8 + Point.sizeof(); }//actual bytes used @@ -19,30 +24,50 @@ module SL { var arr = new Int32Array(data.slice(0, this.sizeof()).buffer); return new Rect(new Point(arr[0], arr[1]), arr[2], arr[3]); } + Fill(d: ArrayBuffer, offset: number): void { + this.Origin.Fill(d, offset); + var dt = new DataView(d, offset + Point.sizeof()); + dt.setInt32(0, this.Height, true); + dt.setInt32(4, this.Width, true); + } } } export module Input { - enum MouseEvents { + export enum MouseEvents { LEFT, RIGHT, MIDDLE, SCROLL, NO_EVENTDATA }; - enum MousePress { + export enum MousePress { UP, DOWN, NO_PRESS_DATA }; export class MouseEvent { - EventData: MouseEvents; - Pos: Utilities.Point; - ScrollDelta: number; - PressData: MousePress; + + constructor(public EventData = MouseEvents.NO_EVENTDATA, + public Pos = new Utilities.Point(0, 0), + public ScrollDelta = 0, + public PressData = MousePress.NO_PRESS_DATA) { } + + static sizeof() { return 1 + Utilities.Point.sizeof() + 4 + 1; }//actual bytes used + Fill(d: ArrayBuffer, offset: number): void { + + var dt = new DataView(d, offset); + dt.setUint8(0, this.EventData); + offset += 1; + this.Pos.Fill(d, offset); + offset += Utilities.Point.sizeof(); + dt.setInt32(offset, this.ScrollDelta, true); + offset += 4; + dt.setUint8(offset, this.PressData); + } }; } export module Network { - enum PACKET_TYPES { + export enum PACKET_TYPES { INVALID, HTTP_MSG, SCREENIMAGE, @@ -55,15 +80,15 @@ module SL { LAST_PACKET_TYPE } export class PacketHeader { - Packet_Type: PACKET_TYPES; - Payload_Length: number; - UncompressedLength: number; + Payload: ArrayBuffer; sizeof() { return 12; }//actual bytes used - constructor(d: ArrayBuffer) { - var data = new DataView(d); - this.Packet_Type = data.getInt32(0, true); - this.Payload_Length = data.getInt32(4, true); - this.UncompressedLength = data.getInt32(8, true); + constructor(public Packet_Type = PACKET_TYPES.INVALID, public Payload_Length = 0, public UncompressedLength = 0) { + this.Payload = new ArrayBuffer(this.Payload_Length); + } + Fill(arr: Uint32Array): void { + arr[0] = this.Packet_Type; + arr[1] = this.Payload_Length; + arr[2] = this.UncompressedLength; } } export class SocketStats { @@ -94,7 +119,8 @@ module SL { constructor(private _dst_host: string, private _dst_port: string) { window.addEventListener("resize", this.onresize); - window.addEventListener("click", this.onclick); + window.addEventListener("mousedown", this.onmousedown); + window.addEventListener("mouseup", this.onmouseup); window.addEventListener("mousemove", this.onmove); } public Start = (): void => { @@ -131,16 +157,45 @@ module SL { public ScaleView = (b: boolean): void => { this._ScaleImage = b; } - onclick = (ev: MouseEvent): void => { - + onmousedown = (ev: MouseEvent): void => { + this.handlemouse(ev.button, Input.MousePress.DOWN, ev.clientX, ev.clientY); + } + onmouseup = (ev: MouseEvent): void => { + this.handlemouse(ev.button, Input.MousePress.UP, ev.clientX, ev.clientY); } onmove = (ev: MouseEvent): void => { + this.handlemouse(-1, Input.MousePress.NO_PRESS_DATA, ev.clientX, ev.clientY); + + } + private handlemouse = (button: number, press: Input.MousePress, x: number, y: number): void => { + var ev = new Input.MouseEvent(); + var scale = this.GetScalingFactor(); + ev.Pos.X = x / scale; + ev.Pos.Y = y / scale; + ev.ScrollDelta = 0; + + ev.PressData = press; + switch (button) { + case 0: + ev.EventData = Input.MouseEvents.LEFT; + break; + case 1: + ev.EventData = Input.MouseEvents.MIDDLE; + break; + case 2: + ev.EventData = Input.MouseEvents.RIGHT; + break; + default: + ev.EventData = Input.MouseEvents.NO_EVENTDATA; + break; + }; + this._ClientNetworkDriver.SendMouse(ev); } onresize = (ev: UIEvent): void => { if (this._ScaleImage && this._OriginalImage != null) { - + var scale = this.GetScalingFactor(); this._HTMLCanvasScreenImage.width = this._OriginalImage.width * scale; this._HTMLCanvasScreenImage.height = this._OriginalImage.height * scale; @@ -154,13 +209,14 @@ module SL { } } GetScalingFactor(): number { - if (this._OriginalImage != null) { + if (this._ScaleImage && this._OriginalImage != null) { return window.innerHeight / this._OriginalImage.height; } else { return 1.0; } } public OnReceive_ImageDif = (socket: WebSocket, rect: Utilities.Rect, img: string): void => { + if (this._OriginalImage === null) return; "use strict"; //console.log('coords' + coords.X + ' ' + coords.Y + ' ' + coords.Width + ' ' + coords.Height); var i = new Image(); @@ -209,7 +265,7 @@ module SL { public OnReceive_MouseImage = (socket: WebSocket, point: Utilities.Point, img: Uint8Array): void => { "use strict"; //console.log('coords' + coords.X + ' ' + coords.Y + ' ' + coords.Width + ' ' + coords.Height); - + this._HTMLCanvasMouseImage.width = point.X; this._HTMLCanvasMouseImage.height = point.Y; try { @@ -240,7 +296,7 @@ module SL { } - class ClientNetworkDriver { + export class ClientNetworkDriver { _Socket: WebSocket; _SocketStats: SocketStats; _TotalMemoryUsed: number; @@ -268,11 +324,54 @@ module SL { this._Socket = null; } public SendMouse = (m: Input.MouseEvent): void => { + var pac = new PacketHeader(PACKET_TYPES.MOUSEEVENT, Input.MouseEvent.sizeof(), Input.MouseEvent.sizeof()); + m.Fill(pac.Payload, 0); + this.Compress_and_Send(pac); + } + private Compress_and_Send = (p: PacketHeader): void => { + var t0 = performance.now(); + + var srcPtr = Module._malloc(p.Payload_Length); + this._TotalMemoryUsed += p.Payload_Length; + var srcbuff = new Uint8Array(Module.HEAPU8.buffer, srcPtr, p.Payload_Length);//get enough space in the heap + srcbuff.set(new Uint8Array(p.Payload, p.sizeof()));//copy the data to the newly allocated memory + + var dstsize = _ZSTD_compressBound(p.UncompressedLength + p.sizeof()); + var dsttr = Module._malloc(dstsize);//get worst case space requirements for dst buffer + this._TotalMemoryUsed += dstsize; + var dstbuff = new Uint8Array(Module.HEAPU8.buffer, dsttr, dstsize);// dont write to the header portion + + + p.Payload_Length = _ZSTD_compress(dstbuff.byteOffset + p.sizeof(), dstsize - p.sizeof(), srcbuff.byteOffset, p.Payload_Length, 3); + if (_ZSTD_isError(p.Payload_Length) > 0) { + console.log('zstd error' + _ZSTD_getErrorName(p.Payload_Length)); + } + + p.Fill(new Uint32Array(Module.HEAPU8.buffer, dsttr, p.sizeof())); + + var t1 = performance.now(); + //comment this line out to see performance issues... My machine takes 0 to 6 ms to complete each receive + // console.log("took " + (t1 - t0) + " milliseconds to Compress the packet") + + Module._free(srcPtr); + this._TotalMemoryUsed -= p.Payload_Length; + var dstsendbuff = new Uint8Array(Module.HEAPU8.buffer, dsttr, p.UncompressedLength + p.sizeof());// dont write to the header portion + + // this._Socket.send(dstsendbuff); + Module._free(dsttr); + this._TotalMemoryUsed -= dstsize; } OnMessage = (ev: MessageEvent): void => { var t0 = performance.now(); - var packetheader = new PacketHeader(ev.data); + + var packetheader = new PacketHeader(); + var data = new DataView(ev.data); + packetheader.Packet_Type = data.getInt32(0, true); + packetheader.Payload_Length = data.getInt32(4, true); + packetheader.UncompressedLength = data.getInt32(8, true); + + var srcPtr = Module._malloc(packetheader.Payload_Length); this._TotalMemoryUsed += packetheader.Payload_Length; @@ -289,7 +388,7 @@ module SL { } var t1 = performance.now(); //comment this line out to see performance issues... My machine takes 0 to 6 ms to complete each receive - console.log("took " + (t1 - t0) + " milliseconds to Decompress the receive loop") + // console.log("took " + (t1 - t0) + " milliseconds to Decompress the receive loop") t0 = performance.now(); switch (packetheader.Packet_Type) { case (PACKET_TYPES.SCREENIMAGE): @@ -314,7 +413,7 @@ module SL { this._TotalMemoryUsed -= packetheader.Payload_Length; t1 = performance.now(); //comment this line out to see performance issues... My machine takes 0 to 6 ms to complete each receive - console.log("took " + (t1 - t0) + " milliseconds to process the receive loop"); + // console.log("took " + (t1 - t0) + " milliseconds to process the receive loop"); } _arrayBufferToBase64(buffer: Uint8Array, offset: number): string { var binary = ''; @@ -338,7 +437,7 @@ module SL { } MouseImage = (data: Uint8Array): void => { - this._IClientDriver.OnReceive_MouseImage(this._Socket, Utilities.Point.FromArray(data), new Uint8Array(data.slice(Utilities.Point.sizeof()).buffer)); + this._IClientDriver.OnReceive_MouseImage(this._Socket, Utilities.Point.FromArray(data), new Uint8Array(data.buffer, Utilities.Point.sizeof())); } MousePos = (data: Uint8Array): void => { this._IClientDriver.OnReceive_MousePos(this._Socket, Utilities.Point.FromArray(data)); From f09f544b648daa38be21e5e5c1f8885b99668def Mon Sep 17 00:00:00 2001 From: smasherprog Date: Wed, 4 May 2016 20:59:18 -0700 Subject: [PATCH 4/6] Working on javascript serialization --- Core/Core.vcxitems.filters | 4 +--- Core/Server.cpp | 2 +- Core/SocketImpl.cpp | 6 ------ Core/WebSocket.h | 6 ++++-- wwwroot/Core.js | 20 ++++++++++---------- wwwroot/Core.ts | 24 ++++++++++++------------ 6 files changed, 28 insertions(+), 34 deletions(-) diff --git a/Core/Core.vcxitems.filters b/Core/Core.vcxitems.filters index 9d4a76b2..d55edf80 100644 --- a/Core/Core.vcxitems.filters +++ b/Core/Core.vcxitems.filters @@ -234,8 +234,6 @@ wwwroot - - - + \ No newline at end of file diff --git a/Core/Server.cpp b/Core/Server.cpp index 81d24c7f..89ddb6a5 100644 --- a/Core/Server.cpp +++ b/Core/Server.cpp @@ -97,7 +97,6 @@ namespace SL { } LastMouse = img; } - virtual void OnMouse(Input::MouseEvent* m) override { Capturing::SetMouseEvent(*m); } @@ -122,6 +121,7 @@ namespace SL { } } } + Server_Status get_Status() const { return Status; } diff --git a/Core/SocketImpl.cpp b/Core/SocketImpl.cpp index d3381061..feae1846 100644 --- a/Core/SocketImpl.cpp +++ b/Core/SocketImpl.cpp @@ -11,31 +11,25 @@ SL::Remote_Access_Library::Network::SocketImpl::SocketImpl(asio::io_service& io_ readtimeout = 5; writetimeout = 5; } - SL::Remote_Access_Library::Network::SocketImpl::~SocketImpl() { CancelTimers(); } - void SL::Remote_Access_Library::Network::SocketImpl::StartReadTimer(int seconds) { if (seconds <= 0) read_deadline_.expires_at(boost::posix_time::pos_infin); else read_deadline_.expires_from_now(boost::posix_time::seconds(seconds)); } - void SL::Remote_Access_Library::Network::SocketImpl::StartWriteTimer(int seconds) { if (seconds <= 0) write_deadline_.expires_at(boost::posix_time::pos_infin); else write_deadline_.expires_from_now(boost::posix_time::seconds(seconds)); } - void SL::Remote_Access_Library::Network::SocketImpl::CancelTimers() { read_deadline_.cancel(); write_deadline_.cancel(); } - - SL::Remote_Access_Library::Network::IBaseNetworkDriver* SL::Remote_Access_Library::Network::SocketImpl::get_Driver() const { return _IBaseNetworkDriver; diff --git a/Core/WebSocket.h b/Core/WebSocket.h index 6c54468b..d77ddd9d 100644 --- a/Core/WebSocket.h +++ b/Core/WebSocket.h @@ -224,9 +224,11 @@ namespace SL { unsigned char mask[MASKSIZE]; memcpy(mask, packet.Payload, sizeof(mask)); auto startpack = packet.Payload; - + std::vector test; + test.resize(packet.Payload_Length - MASKSIZE); + for (size_t c = 0; c < packet.Payload_Length - MASKSIZE; c++) { - startpack[c] = startpack[c + MASKSIZE] ^ mask[c % MASKSIZE]; + test[c]= startpack[c] = startpack[c + MASKSIZE] ^ mask[c % MASKSIZE]; } } diff --git a/wwwroot/Core.js b/wwwroot/Core.js index a1db2eb6..e920218b 100644 --- a/wwwroot/Core.js +++ b/wwwroot/Core.js @@ -84,12 +84,12 @@ var SL; MouseEvent.prototype.Fill = function (d, offset) { var dt = new DataView(d, offset); dt.setUint8(0, this.EventData); - offset += 1; - this.Pos.Fill(d, offset); - offset += Utilities.Point.sizeof(); - dt.setInt32(offset, this.ScrollDelta, true); - offset += 4; - dt.setUint8(offset, this.PressData); + + this.Pos.Fill(d, offset + 1); + + dt.setInt32(offset + 1 + Utilities.Point.sizeof(), this.ScrollDelta, true); + + dt.setUint8(offset + 1 + Utilities.Point.sizeof() + 4, this.PressData); }; return MouseEvent; })(); @@ -360,7 +360,7 @@ var SL; var srcPtr = Module._malloc(p.Payload_Length); _this._TotalMemoryUsed += p.Payload_Length; var srcbuff = new Uint8Array(Module.HEAPU8.buffer, srcPtr, p.Payload_Length); - srcbuff.set(new Uint8Array(p.Payload, p.sizeof())); //copy the data to the newly allocated memory + srcbuff.set(new Uint8Array(p.Payload, 0, p.Payload_Length)); //copy the data to the newly allocated memory var dstsize = _ZSTD_compressBound(p.UncompressedLength + p.sizeof()); var dsttr = Module._malloc(dstsize); @@ -380,9 +380,9 @@ var SL; // console.log("took " + (t1 - t0) + " milliseconds to Compress the packet") Module._free(srcPtr); _this._TotalMemoryUsed -= p.Payload_Length; - var dstsendbuff = new Uint8Array(Module.HEAPU8.buffer, dsttr, p.UncompressedLength + p.sizeof()); - - // this._Socket.send(dstsendbuff); + var test = dstbuff.buffer.slice(dsttr, dsttr + p.Payload_Length + p.sizeof()); + var teu = new Uint8Array(test, 0, p.Payload_Length + p.sizeof()); + _this._Socket.send(teu); Module._free(dsttr); _this._TotalMemoryUsed -= dstsize; }; diff --git a/wwwroot/Core.ts b/wwwroot/Core.ts index f93182fe..96868290 100644 --- a/wwwroot/Core.ts +++ b/wwwroot/Core.ts @@ -57,12 +57,13 @@ module SL { var dt = new DataView(d, offset); dt.setUint8(0, this.EventData); - offset += 1; - this.Pos.Fill(d, offset); - offset += Utilities.Point.sizeof(); - dt.setInt32(offset, this.ScrollDelta, true); - offset += 4; - dt.setUint8(offset, this.PressData); + + this.Pos.Fill(d, offset + 1); + + dt.setInt32(offset + 1 + Utilities.Point.sizeof(), this.ScrollDelta, true); + + dt.setUint8(offset + 1 + Utilities.Point.sizeof() + 4, this.PressData); + } }; } @@ -334,13 +335,12 @@ module SL { var srcPtr = Module._malloc(p.Payload_Length); this._TotalMemoryUsed += p.Payload_Length; var srcbuff = new Uint8Array(Module.HEAPU8.buffer, srcPtr, p.Payload_Length);//get enough space in the heap - srcbuff.set(new Uint8Array(p.Payload, p.sizeof()));//copy the data to the newly allocated memory - + srcbuff.set(new Uint8Array(p.Payload, 0, p.Payload_Length));//copy the data to the newly allocated memory + var dstsize = _ZSTD_compressBound(p.UncompressedLength + p.sizeof()); var dsttr = Module._malloc(dstsize);//get worst case space requirements for dst buffer this._TotalMemoryUsed += dstsize; var dstbuff = new Uint8Array(Module.HEAPU8.buffer, dsttr, dstsize);// dont write to the header portion - p.Payload_Length = _ZSTD_compress(dstbuff.byteOffset + p.sizeof(), dstsize - p.sizeof(), srcbuff.byteOffset, p.Payload_Length, 3); if (_ZSTD_isError(p.Payload_Length) > 0) { @@ -355,9 +355,9 @@ module SL { Module._free(srcPtr); this._TotalMemoryUsed -= p.Payload_Length; - var dstsendbuff = new Uint8Array(Module.HEAPU8.buffer, dsttr, p.UncompressedLength + p.sizeof());// dont write to the header portion - - // this._Socket.send(dstsendbuff); + var test = dstbuff.buffer.slice(dsttr, dsttr+ p.Payload_Length + p.sizeof()); + var teu = new Uint8Array(test, 0, p.Payload_Length + p.sizeof()); + this._Socket.send(teu); Module._free(dsttr); this._TotalMemoryUsed -= dstsize; From 3fd84b51cf2d9c76922096d640b89e76d0ab86fb Mon Sep 17 00:00:00 2001 From: smasherprog Date: Thu, 5 May 2016 06:26:42 -0700 Subject: [PATCH 5/6] Fixed screen not being sent on first connect Broke mouse visability on web browser. Will fix --- Core/Server.cpp | 17 ++++------------- wwwroot/Core.ts | 2 +- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/Core/Server.cpp b/Core/Server.cpp index 89ddb6a5..109f7809 100644 --- a/Core/Server.cpp +++ b/Core/Server.cpp @@ -43,6 +43,7 @@ namespace SL { { std::lock_guard lock(_NewClientLock); _NewClients.push_back(socket); + } if (_IUserNetworkDriver != nullptr) _IUserNetworkDriver->OnConnect(socket); } @@ -77,21 +78,11 @@ namespace SL { } void OnMouseImg(std::shared_ptr img) { - if (!LastMouse) {//first screen send all! + if (!LastMouse) { _ServerNetworkDriver.SendMouse(nullptr, *img); - std::lock_guard lock(_NewClientLock); - _NewClients.clear(); } - else {//compare and send all difs along - { - //make sure to send the full screens to any new connects - std::lock_guard lock(_NewClientLock); - for (auto& a : _NewClients) { - _ServerNetworkDriver.SendMouse(a.get(), *img); - } - _NewClients.clear(); - } - if (memcmp(img->data(), LastScreen->data(), std::min(LastScreen->size(), img->size())) != 0) { + else { + if (memcmp(img->data(), LastMouse->data(), std::min(LastMouse->size(), img->size())) != 0) { _ServerNetworkDriver.SendMouse(nullptr, *img); } } diff --git a/wwwroot/Core.ts b/wwwroot/Core.ts index 96868290..25aa2b4a 100644 --- a/wwwroot/Core.ts +++ b/wwwroot/Core.ts @@ -271,7 +271,7 @@ module SL { this._HTMLCanvasMouseImage.height = point.Y; try { this._Cursor = this._HTMLCanvasMouseImage.getContext("2d").createImageData(point.X, point.Y); - + for (var i = 0; i < this._Cursor.data.length; i += 4) { this._Cursor.data[i + 0] = img[i + 0]; this._Cursor.data[i + 1] = img[i + 1]; From 4e8f61dd8ecc8463e2642c3b166fd4d46d85f1f7 Mon Sep 17 00:00:00 2001 From: smasherprog Date: Thu, 5 May 2016 20:40:02 -0700 Subject: [PATCH 6/6] Updated typescript and mouse server code Fixed Mouse not displaying in web viewer Updated Mouse struct to account for padding added by compiler. --- Core/ApplicationDirectory.cpp | 1 - Core/Core.vcxitems | 1 + Core/Core.vcxitems.filters | 4 +++- Core/HttpHeader.cpp | 3 +-- Core/Mouse.h | 4 ++-- Core/Server.cpp | 4 ++-- Core/ServerNetworkDriver.cpp | 4 ++-- Core/SocketImpl.cpp | 2 -- wwwroot/Core.js | 17 +++++++---------- wwwroot/Core.ts | 19 +++++++------------ 10 files changed, 25 insertions(+), 34 deletions(-) diff --git a/Core/ApplicationDirectory.cpp b/Core/ApplicationDirectory.cpp index 695d37e4..38978e49 100644 --- a/Core/ApplicationDirectory.cpp +++ b/Core/ApplicationDirectory.cpp @@ -8,7 +8,6 @@ std::string executable_path(const char *argv0) } #else - #include #include #include diff --git a/Core/Core.vcxitems b/Core/Core.vcxitems index c17e75ab..a5d81c25 100644 --- a/Core/Core.vcxitems +++ b/Core/Core.vcxitems @@ -112,6 +112,7 @@ tsc $(SolutionDir)wwwroot\Core.ts $(OutDir)wwwroot\Core.js true + false diff --git a/Core/Core.vcxitems.filters b/Core/Core.vcxitems.filters index d55edf80..cbc9aa50 100644 --- a/Core/Core.vcxitems.filters +++ b/Core/Core.vcxitems.filters @@ -234,6 +234,8 @@ wwwroot - + + wwwroot + \ No newline at end of file diff --git a/Core/HttpHeader.cpp b/Core/HttpHeader.cpp index 3eb17b27..9eb86c2c 100644 --- a/Core/HttpHeader.cpp +++ b/Core/HttpHeader.cpp @@ -14,7 +14,6 @@ namespace SL { { if (in[i] == '%') { - if (i + 3 <= in.size()) { int value = 0; @@ -26,7 +25,6 @@ namespace SL { } else { - return std::string("/"); } } @@ -51,6 +49,7 @@ namespace SL { } } + std::unordered_map SL::Remote_Access_Library::Network::HttpHeader::Parse(std::string defaultheaderversion, std::istream& stream) { std::unordered_map header; diff --git a/Core/Mouse.h b/Core/Mouse.h index 9f37d33d..3517f928 100644 --- a/Core/Mouse.h +++ b/Core/Mouse.h @@ -6,14 +6,14 @@ namespace SL { namespace Remote_Access_Library { namespace Input { - enum MouseEvents : unsigned char { + enum MouseEvents : unsigned int { LEFT, RIGHT, MIDDLE, SCROLL, NO_EVENTDATA }; - enum MousePress : unsigned char { + enum MousePress : unsigned int { UP, DOWN, NO_PRESS_DATA diff --git a/Core/Server.cpp b/Core/Server.cpp index 109f7809..1459aa14 100644 --- a/Core/Server.cpp +++ b/Core/Server.cpp @@ -76,6 +76,8 @@ namespace SL { } LastScreen = img;//swap } + + void OnMouseImg(std::shared_ptr img) { if (!LastMouse) { @@ -112,12 +114,10 @@ namespace SL { } } } - Server_Status get_Status() const { return Status; } }; - } } diff --git a/Core/ServerNetworkDriver.cpp b/Core/ServerNetworkDriver.cpp index 723ff610..b355ec3b 100644 --- a/Core/ServerNetworkDriver.cpp +++ b/Core/ServerNetworkDriver.cpp @@ -35,7 +35,7 @@ namespace SL { void MouseEvent(const std::shared_ptr& socket, std::shared_ptr& p) { - assert(p->Payload_Length == sizeof(Input::MouseEvent::EventData) + sizeof(Input::MouseEvent::Pos) + sizeof(Input::MouseEvent::ScrollDelta) + sizeof(Input::MouseEvent::PressData)); + assert(p->Payload_Length == sizeof(Input::MouseEvent)); _IServerDriver->OnMouse((Input::MouseEvent*) p->Payload); } public: @@ -53,7 +53,7 @@ namespace SL { virtual void OnClose(const std::shared_ptr& socket)override { _IServerDriver->OnClose(socket); std::lock_guard lock(_ClientsLock); - _Clients.erase(std::remove_if(begin(_Clients), end(_Clients), [socket](const std::shared_ptr& p) { return p == socket; }), _Clients.end()); + _Clients.erase(std::remove_if(begin(_Clients), end(_Clients), [&socket](const std::shared_ptr& p) { return p == socket; }), _Clients.end()); } virtual void OnReceive(const std::shared_ptr& socket, std::shared_ptr& p) override diff --git a/Core/SocketImpl.cpp b/Core/SocketImpl.cpp index feae1846..725c773f 100644 --- a/Core/SocketImpl.cpp +++ b/Core/SocketImpl.cpp @@ -39,8 +39,6 @@ SL::Remote_Access_Library::Network::SocketStats SL::Remote_Access_Library::Netwo { return _SocketStats; } - - char* SL::Remote_Access_Library::Network::SocketImpl::get_ReadBuffer() { _IncomingBuffer.reserve(ReadPacketHeader.Payload_Length); diff --git a/wwwroot/Core.js b/wwwroot/Core.js index e920218b..d49926a1 100644 --- a/wwwroot/Core.js +++ b/wwwroot/Core.js @@ -79,17 +79,14 @@ var SL; this.PressData = PressData; } MouseEvent.sizeof = function () { - return 1 + Utilities.Point.sizeof() + 4 + 1; + return 4 + Utilities.Point.sizeof() + 4 + 4; }; MouseEvent.prototype.Fill = function (d, offset) { var dt = new DataView(d, offset); - dt.setUint8(0, this.EventData); - - this.Pos.Fill(d, offset + 1); - - dt.setInt32(offset + 1 + Utilities.Point.sizeof(), this.ScrollDelta, true); - - dt.setUint8(offset + 1 + Utilities.Point.sizeof() + 4, this.PressData); + dt.setUint32(0, this.EventData, true); + this.Pos.Fill(d, offset + 4); + dt.setInt32(offset + 4 + Utilities.Point.sizeof(), this.ScrollDelta, true); + dt.setUint32(offset + 4 + Utilities.Point.sizeof() + 4, this.PressData, true); }; return MouseEvent; })(); @@ -160,7 +157,7 @@ var SL; _this._HTMLCanvasScreenImage.style.left = _this._HTMLCanvasScreenImage.style.top = _this._HTMLCanvasScreenImage.style.zIndex = '0'; _this._HTMLCanvasMouseImage = document.createElement('canvas'); - + _this._HTMLCanvasMouseImage.style.position = 'absolute'; _this._HTMLCanvasMouseImage.style.left = _this._HTMLCanvasMouseImage.style.top = '0'; _this._HTMLCanvasMouseImage.style.zIndex = '1'; @@ -446,7 +443,7 @@ var SL; _this._IClientDriver.OnReceive_ImageDif(_this._Socket, Utilities.Rect.FromArray(data), _this._arrayBufferToBase64(data, Utilities.Rect.sizeof())); }; this.MouseImage = function (data) { - _this._IClientDriver.OnReceive_MouseImage(_this._Socket, Utilities.Point.FromArray(data), new Uint8Array(data.buffer, Utilities.Point.sizeof())); + _this._IClientDriver.OnReceive_MouseImage(_this._Socket, Utilities.Point.FromArray(data), new Uint8Array(data.slice(Utilities.Point.sizeof()).buffer)); }; this.MousePos = function (data) { _this._IClientDriver.OnReceive_MousePos(_this._Socket, Utilities.Point.FromArray(data)); diff --git a/wwwroot/Core.ts b/wwwroot/Core.ts index 25aa2b4a..38e9e166 100644 --- a/wwwroot/Core.ts +++ b/wwwroot/Core.ts @@ -52,17 +52,13 @@ module SL { public ScrollDelta = 0, public PressData = MousePress.NO_PRESS_DATA) { } - static sizeof() { return 1 + Utilities.Point.sizeof() + 4 + 1; }//actual bytes used + static sizeof() { return 4 + Utilities.Point.sizeof() + 4 + 4; }//actual bytes used Fill(d: ArrayBuffer, offset: number): void { - var dt = new DataView(d, offset); - dt.setUint8(0, this.EventData); - - this.Pos.Fill(d, offset + 1); - - dt.setInt32(offset + 1 + Utilities.Point.sizeof(), this.ScrollDelta, true); - - dt.setUint8(offset + 1 + Utilities.Point.sizeof() + 4, this.PressData); + dt.setUint32(0, this.EventData, true); + this.Pos.Fill(d, offset + 4); + dt.setInt32(offset + 4 + Utilities.Point.sizeof(), this.ScrollDelta, true); + dt.setUint32(offset + 4 + Utilities.Point.sizeof() + 4, this.PressData, true); } }; @@ -137,7 +133,7 @@ module SL { this._HTMLCanvasScreenImage.style.left = this._HTMLCanvasScreenImage.style.top = this._HTMLCanvasScreenImage.style.zIndex = '0'; this._HTMLCanvasMouseImage = document.createElement('canvas'); - + this._HTMLCanvasMouseImage.style.position = 'absolute'; this._HTMLCanvasMouseImage.style.left = this._HTMLCanvasMouseImage.style.top = '0'; this._HTMLCanvasMouseImage.style.zIndex = '1'; @@ -436,8 +432,7 @@ module SL { this._arrayBufferToBase64(data, Utilities.Rect.sizeof())); } MouseImage = (data: Uint8Array): void => { - - this._IClientDriver.OnReceive_MouseImage(this._Socket, Utilities.Point.FromArray(data), new Uint8Array(data.buffer, Utilities.Point.sizeof())); + this._IClientDriver.OnReceive_MouseImage(this._Socket, Utilities.Point.FromArray(data), new Uint8Array(data.slice(Utilities.Point.sizeof()).buffer)); } MousePos = (data: Uint8Array): void => { this._IClientDriver.OnReceive_MousePos(this._Socket, Utilities.Point.FromArray(data));