raylib cameraview

selfdrive/ui/SConscript

@@ -1,6 +1,6 @@
 import os
 import json
-Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'transformations')
+Import('env', 'qt_env', 'arch', 'common', 'messaging', 'visionipc', 'transformations')
 
 base_libs = [common, messaging, visionipc, transformations,
              'm', 'OpenCL', 'ssl', 'crypto', 'pthread'] + qt_env["LIBS"]
@@ -109,4 +109,14 @@ if GetOption('extras') and arch != "Darwin":
 
 # build watch3
 if arch in ['x86_64', 'aarch64', 'Darwin'] or GetOption('extras'):
-  qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'msgq', 'visionipc'])
+  raylib_env = env.Clone()
+
+  raylib_frameworks = []
+  raylib_libs = [common, messaging, visionipc, transformations, 'common', 'msgq', 'visionipc', 'zmq', 'json11', 'pthread', 'raylib']
+  if arch == "Darwin":
+    raylib_frameworks += ['OpenCL', 'CoreVideo', 'Cocoa', 'GLUT', 'CoreFoundation', 'OpenGL', 'IOKit']
+  else:
+    raylib_libs.append('OpenCL')
+
+  raylib_env['LIBPATH'] += [f'#third_party/raylib/{arch}/']
+  raylib_env.Program("watch3", ['watch3.cc', 'raylib/cameraview.cc'], LIBS=raylib_libs, FRAMEWORKS=raylib_frameworks)

selfdrive/ui/raylib/cameraview.cc

@@ -0,0 +1,68 @@
+#include "selfdrive/ui/raylib/cameraview.h"
+
+#include "common/util.h"
+
+const char frame_fragment_shader[] = R"(
+  #version 330 core
+  in vec2 fragTexCoord;
+  uniform sampler2D texture0;  // Y plane
+  uniform sampler2D texture1;  // UV plane
+  out vec4 fragColor;
+  void main() {
+    float y = texture(texture0, fragTexCoord).r;
+    vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
+    float r = y + 1.402 * uv.y;
+    float g = y - 0.344 * uv.x - 0.714 * uv.y;
+    float b = y + 1.772 * uv.x;
+    fragColor = vec4(r, g, b, 1.0);
+  }
+)";
+
+CameraView::CameraView(const std::string &name, VisionStreamType type) {
+  client = std::make_unique<VisionIpcClient>(name, type, false);
+  shader = LoadShaderFromMemory(NULL, frame_fragment_shader);
+}
+
+CameraView::~CameraView() {
+  if (textureY.id) UnloadTexture(textureY);
+  if (textureUV.id) UnloadTexture(textureUV);
+  if (shader.id) UnloadShader(shader);
+}
+
+void CameraView::draw(const Rectangle &rec) {
+  if (!ensureConnection()) return;
+
+  auto buffer = client->recv(nullptr, 20);
+  frame = buffer ? buffer : frame;
+  if (!frame) return;
+
+  UpdateTexture(textureY, frame->y);
+  UpdateTexture(textureUV, frame->uv);
+
+  // Calculate scaling factors to maintain aspect ratio
+  float scale = std::min((float)rec.width / frame->width, (float)rec.height / frame->height);
+  float x_offset = rec.x + (rec.width - (frame->width * scale)) / 2;
+  float y_offset = rec.y + (rec.height - (frame->height * scale)) / 2;
+  Rectangle src_rect = {0, 0, (float)frame->width, (float)frame->height};
+  Rectangle dst_rect = {x_offset, y_offset, frame->width * scale, frame->height * scale};
+
+  BeginShaderMode(shader);
+  SetShaderValueTexture(shader, GetShaderLocation(shader, "texture1"), textureUV);
+  DrawTexturePro(textureY, src_rect, dst_rect, Vector2{0, 0}, 0.0, WHITE);
+  EndShaderMode();
+}
+
+bool CameraView::ensureConnection() {
+  if (!client->connected) {
+    frame = nullptr;
+    if (!client->connect(false)) return false;
+
+    if (textureY.id) UnloadTexture(textureY);
+    if (textureUV.id) UnloadTexture(textureUV);
+    // Create textures for Y and UV planes
+    const auto &buf = client->buffers[0];
+    textureY = LoadTextureFromImage(Image{nullptr, (int)buf.stride, (int)buf.height, 1, PIXELFORMAT_UNCOMPRESSED_GRAYSCALE});
+    textureUV = LoadTextureFromImage(Image{nullptr, (int)buf.stride / 2, (int)buf.height / 2, 1, PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA});
+  }
+  return true;
+}

selfdrive/ui/raylib/cameraview.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <memory>
+#include "msgq/visionipc/visionipc_client.h"
+#include "third_party/raylib/include/raylib.h"
+
+class CameraView {
+public:
+  CameraView(const std::string &name, VisionStreamType type);
+  virtual ~CameraView();
+  void draw(const Rectangle &rec);
+
+protected:
+  bool ensureConnection();
+
+  std::unique_ptr<VisionIpcClient> client;
+  Texture2D textureY = {};
+  Texture2D textureUV = {};
+  Shader shader = {};
+  VisionBuf *frame = nullptr;
+};

selfdrive/ui/watch3.cc

@@ -1,33 +1,33 @@
-#include <QApplication>
-#include <QtWidgets>
+#include "selfdrive/ui/raylib/cameraview.h"
 
-#include "selfdrive/ui/qt/qt_window.h"
-#include "selfdrive/ui/qt/util.h"
-#include "selfdrive/ui/qt/widgets/cameraview.h"
+void renderCameraViews() {
+  CameraView roadCamera("camerad", VISION_STREAM_ROAD);
+  CameraView wideRoadCamera("camerad", VISION_STREAM_WIDE_ROAD);
+  CameraView driverCamera("camerad", VISION_STREAM_DRIVER);
 
-int main(int argc, char *argv[]) {
-  initApp(argc, argv);
-
-  QApplication a(argc, argv);
-  QWidget w;
-  setMainWindow(&w);
+  while (!WindowShouldClose()) {
+    float w = GetScreenWidth(), h = GetScreenHeight();
+    Rectangle roadCameraRec = {0, 0, w, h / 2};
+    Rectangle wideRoadCameraRec = {0, h / 2, w / 2, h / 2};
+    Rectangle driverCameraRec = {w / 2, h / 2, w / 2, h / 2};
 
-  QVBoxLayout *layout = new QVBoxLayout(&w);
-  layout->setMargin(0);
-  layout->setSpacing(0);
-
-  {
-    QHBoxLayout *hlayout = new QHBoxLayout();
-    layout->addLayout(hlayout);
-    hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_ROAD));
+    BeginDrawing();
+      ClearBackground(BLACK);
+      roadCamera.draw(roadCameraRec);
+      wideRoadCamera.draw(wideRoadCameraRec);
+      driverCamera.draw(driverCameraRec);
+    EndDrawing();
   }
+}
 
-  {
-    QHBoxLayout *hlayout = new QHBoxLayout();
-    layout->addLayout(hlayout);
-    hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_DRIVER));
-    hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_WIDE_ROAD));
-  }
+int main(int argc, char *argv[]) {
+  SetTraceLogLevel(LOG_NONE);
+  SetConfigFlags(FLAG_WINDOW_RESIZABLE);
+  InitWindow(0, 0, "Watch 3 Cameras");
+  SetTargetFPS(20);
+
+  renderCameraViews();
 
-  return a.exec();
+  CloseWindow();
+  return 0;
 }