Skip to content

Commit

Permalink
improved parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
joreg committed Dec 20, 2023
1 parent 0b56dff commit f007e2d
Show file tree
Hide file tree
Showing 13 changed files with 1,521 additions and 738 deletions.
1,845 changes: 1,121 additions & 724 deletions VL.MediaPipe.vl

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion deployment/VL.MediaPipe.nuspec
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<package xmlns="http://schemas.microsoft.com/packaging/2011/10/nuspec.xsd">
<metadata>
<id>VL.MediaPipe</id>
<version>0.0.10-alpha</version>
<version>0.0.11-alpha</version>
<title>VL.MediaPipe</title>
<authors>Dom Scott, Torin Blankensmith, vvvv</authors>
<owners>vvvv</owners>
Expand Down
42 changes: 30 additions & 12 deletions help/HowTo Use MediaPipe.vl
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<Document xmlns:p="property" xmlns:r="reflection" Id="Jr3PSf8cf9JO0yxDYF4J1a" LanguageVersion="2023.5.3-0405-gb0165f3bf2" Version="0.128">
<NugetDependency Id="LlgpqYpbpIDN4Yh4YGtVJj" Location="VL.CoreLib" Version="2023.5.3-0405-gb0165f3bf2" />
<Document xmlns:p="property" xmlns:r="reflection" Id="Jr3PSf8cf9JO0yxDYF4J1a" LanguageVersion="2023.5.3-0413-g27f174d434" Version="0.128">
<NugetDependency Id="LlgpqYpbpIDN4Yh4YGtVJj" Location="VL.CoreLib" Version="2023.5.3-0413-g27f174d434" />
<Patch Id="Q76TiVeE82cNHpdVGLNjIM">
<Canvas Id="PHmrnxQ15guOqUWjJc4afe" DefaultCategory="Main" BordersChecked="false" CanvasType="FullCategory" />
<Canvas Id="PHmrnxQ15guOqUWjJc4afe" DefaultCategory="Main" CanvasType="FullCategory" />
<!--
************************ Application ************************
Expand All @@ -29,7 +29,7 @@
<Pin Id="EbczbNFE34RLCpZeYIRbCv" Name="Detect poses" Kind="InputPin" />
<Pin Id="Kz0tPenxZGEP5t5DQf20IO" Name="Detect objects" Kind="InputPin" />
<Pin Id="NS8f38KEgGMOJD9zQpou3O" Name="Classify images" Kind="InputPin" />
<Pin Id="Kftk1cUSf7KOK3ccgmcpEZ" Name="Image Segmentation" Kind="InputPin" />
<Pin Id="Kftk1cUSf7KOK3ccgmcpEZ" Name="Image Segmentation" Kind="InputPin" DefaultValue="False" />
<Pin Id="RzJlP9ULamlLwJQl4zpEEd" Name="Show overlays" Kind="InputPin" />
<Pin Id="U7uw9dxXxCTMdNAJoFxG84" Name="Width" Kind="InputPin" DefaultValue="640" />
<Pin Id="D8xWw7HRdV1Npcxtau7F7x" Name="Height" Kind="InputPin" DefaultValue="270" />
Expand All @@ -42,7 +42,7 @@
<Pin Id="DxARM8Wx9atLEjJsYCrZKI" Name="State Output" Kind="OutputPin" />
<Pin Id="SVGaxn1MF1QOmovm0LVVth" Name="Output" Kind="OutputPin" />
</Node>
<Node Bounds="402,382,95,19" Id="Qz02M0a265fMhPsfMeXCBR">
<Node Bounds="382,384,95,19" Id="Qz02M0a265fMhPsfMeXCBR">
<p:NodeReference LastCategoryFullName="CEF" LastDependency="VL.CEF.Stride.vl">
<Choice Kind="NodeFlag" Name="Node" Fixed="true" />
<Choice Kind="ProcessAppFlag" Name="ToStrideRenderer" />
Expand All @@ -51,7 +51,7 @@
<Pin Id="LzlPW6IcAYbOW0sKqzqQvs" Name="Browser" Kind="InputPin" />
<Pin Id="GQ9isDNWSFRNmCKwVaPJW1" Name="Output" Kind="OutputPin" />
</Node>
<Node Bounds="402,431,165,19" Id="QEn8pF0jXpxPvptaw8nyEc">
<Node Bounds="382,433,165,19" Id="QEn8pF0jXpxPvptaw8nyEc">
<p:NodeReference LastCategoryFullName="Stride" LastDependency="VL.Stride.vl">
<Choice Kind="NodeFlag" Name="Node" Fixed="true" />
<Choice Kind="ProcessAppFlag" Name="RenderWindow" />
Expand Down Expand Up @@ -90,7 +90,7 @@
<p:buttonmode p:Assembly="VL.UI.Forms" p:Type="VL.HDE.PatchEditor.Editors.ButtonModeEnum">Toggle</p:buttonmode>
</p:ValueBoxSettings>
</Pad>
<Pad Id="OhxUbFgoBiqPUIv9FeZ6pa" Comment="Detect facial landmarks" Bounds="124,173,35,35" ShowValueBox="true" isIOBox="true" Value="True">
<Pad Id="OhxUbFgoBiqPUIv9FeZ6pa" Comment="Detect facial landmarks" Bounds="124,173,35,35" ShowValueBox="true" isIOBox="true" Value="False">
<p:TypeAnnotation LastCategoryFullName="Primitive" LastDependency="VL.CoreLib.vl">
<Choice Kind="ImmutableTypeFlag" Name="Boolean" />
</p:TypeAnnotation>
Expand Down Expand Up @@ -150,12 +150,12 @@
<p:buttonmode p:Assembly="VL.UI.Forms" p:Type="VL.HDE.PatchEditor.Editors.ButtonModeEnum">Bang</p:buttonmode>
</p:ValueBoxSettings>
</Pad>
<Pad Id="Rl6H1fA5L95MB7LiXYxnuK" Comment="Camera" Bounds="344,228,182,15" ShowValueBox="true" isIOBox="true" Value="HD Pro Webcam C920 (046d:082d)">
<Pad Id="Rl6H1fA5L95MB7LiXYxnuK" Comment="Camera" Bounds="344,153,182,15" ShowValueBox="true" isIOBox="true" Value="HD Pro Webcam C920 (046d:082d)">
<p:TypeAnnotation LastCategoryFullName="MediaPipe" LastDependency="VL.MediaPipe.vl">
<Choice Kind="TypeFlag" Name="CamerasEnum" />
</p:TypeAnnotation>
</Pad>
<Pad Id="F5ApVdxsPh7LZXDOgTGvfX" Bounds="788,141,87,20" ShowValueBox="true" isIOBox="true" Value="Wait for it &gt;">
<Pad Id="F5ApVdxsPh7LZXDOgTGvfX" Bounds="789,88,87,20" ShowValueBox="true" isIOBox="true" Value="Wait for it &gt;">
<p:TypeAnnotation LastCategoryFullName="Primitive" LastDependency="VL.CoreLib.vl">
<Choice Kind="TypeFlag" Name="String" />
</p:TypeAnnotation>
Expand Down Expand Up @@ -251,15 +251,15 @@
<Pin Id="O7U3F8lZRqJLppWwwA38ga" Name="Min Tracking Confidence" Kind="InputPin" />
<Pin Id="GLPgNONyHzVNwvogqB9TGY" Name="Output" Kind="OutputPin" />
</Node>
<Pad Id="Ozd3k247z6HPMNyV6xa60k" Comment="Detect poses" Bounds="184,215,35,35" ShowValueBox="true" isIOBox="true" Value="False">
<Pad Id="Ozd3k247z6HPMNyV6xa60k" Comment="Detect poses" Bounds="184,215,35,35" ShowValueBox="true" isIOBox="true" Value="True">
<p:TypeAnnotation LastCategoryFullName="Primitive" LastDependency="VL.CoreLib.vl">
<Choice Kind="ImmutableTypeFlag" Name="Boolean" />
</p:TypeAnnotation>
<p:ValueBoxSettings>
<p:buttonmode p:Assembly="VL.UI.Forms" p:Type="VL.HDE.PatchEditor.Editors.ButtonModeEnum">Toggle</p:buttonmode>
</p:ValueBoxSettings>
</Pad>
<Pad Id="BphNfmMvyf7LzvBjVcY5WM" Bounds="575,386,248,152" ShowValueBox="true" isIOBox="true" Value="&lt; Red node? Check missing dependencies in the Document Menu!&#xD;&#xA;&#xD;&#xA;Explanation: This node is not required for the operation of MediaPipe, only to preview the results. Therefore the VL.CEF.Stride pack is not shipping with the VL.MediaPipe pack. ">
<Pad Id="BphNfmMvyf7LzvBjVcY5WM" Bounds="555,388,248,152" ShowValueBox="true" isIOBox="true" Value="&lt; Red node? Check missing dependencies in the Document Menu!&#xD;&#xA;&#xD;&#xA;Explanation: This node is not required for the operation of MediaPipe, only to preview the results. Therefore the VL.CEF.Stride pack is not shipping with the VL.MediaPipe pack. ">
<p:TypeAnnotation LastCategoryFullName="Primitive" LastDependency="VL.CoreLib.vl">
<Choice Kind="TypeFlag" Name="String" />
</p:TypeAnnotation>
Expand All @@ -282,6 +282,24 @@
</Node>
<Pad Id="UlGKLKhHjBiP1XqbrHxk0e" Comment="Detection Time" Bounds="159,447,35,15" ShowValueBox="true" isIOBox="true" />
<Pad Id="UEVYyt5d1DrPqZMTEg3gVs" Comment="Draw Time" Bounds="182,468,35,15" ShowValueBox="true" isIOBox="true" />
<Pad Id="FI16PEkCOnqOkmlcBGcZ59" Bounds="675,123,240,147" ShowValueBox="true" isIOBox="true" Value="No video showing after ~10s? &gt; &#xD;&#xA;&#xD;&#xA;In case you have 2 GPUs you have 2 options&#xD;&#xA;- Enable the optional &quot;Shared Texture..&quot; Input on the MediaPipe node&#xD;&#xA;- Make the unterlying WebRenderer run on the same GPU, see:&#xD;&#xA;">
<p:TypeAnnotation LastCategoryFullName="Primitive" LastDependency="VL.CoreLib.vl">
<Choice Kind="TypeFlag" Name="String" />
</p:TypeAnnotation>
<p:ValueBoxSettings>
<p:fontsize p:Type="Int32">9</p:fontsize>
<p:stringtype p:Assembly="VL.Core" p:Type="VL.Core.StringType">Comment</p:stringtype>
</p:ValueBoxSettings>
</Pad>
<Pad Id="Dcfe7mGBRoCPlTD0L8I5xc" Bounds="679,281,140,19" ShowValueBox="true" isIOBox="true" Value="https://github.com/vvvv/VL.CEF/blob/master/README.md#troubleshooting">
<p:TypeAnnotation LastCategoryFullName="Primitive" LastDependency="VL.CoreLib.vl">
<Choice Kind="TypeFlag" Name="String" />
</p:TypeAnnotation>
<p:ValueBoxSettings>
<p:fontsize p:Type="Int32">9</p:fontsize>
<p:stringtype p:Assembly="VL.Core" p:Type="VL.Core.StringType">Link</p:stringtype>
</p:ValueBoxSettings>
</Pad>
</Canvas>
<Patch Id="NnQxyM64wAjMFnS799ZJfW" Name="Create" />
<Patch Id="HlMT1WaS0ehPS9K2fCBqxG" Name="Update" />
Expand All @@ -308,6 +326,6 @@
</Node>
</Patch>
<NugetDependency Id="RxQUDZMcKbWNbdZgm6RaSj" Location="VL.CEF.Stride" Version="0.5.3" />
<NugetDependency Id="UomYyo1o89EL1tzcQv645B" Location="VL.Stride" Version="2023.5.3-0405-gb0165f3bf2" />
<NugetDependency Id="UomYyo1o89EL1tzcQv645B" Location="VL.Stride" Version="2023.5.3-0413-g27f174d434" />
<NugetDependency Id="LVi6AOgIWYkNdVDA5if0rq" Location="VL.MediaPipe" Version="0.0.0" />
</Document>
63 changes: 63 additions & 0 deletions src/FaceDetectorResults.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
using System.Text.Json;
using System.Text.Json.Serialization;

namespace MediaPipe.FaceDetectorResults
{
public class faceDetectorRoot
{
public Facedetectorresults faceDetectorResults { get; init; }
public Resolution resolution { get; }
}

public class Facedetectorresults
{
[JsonPropertyName("detections")]
public FaceDescription[] Faces { get; init; }
}

public class FaceDescription
{
[JsonPropertyName("categories")]
public Category[] Categories { get; init; }
[JsonPropertyName("keypoints")]
public Keypoint[] Keypoints { get; init; }

//with .NET8 we'll be able to set the below to private and then jsoninclude
//[JsonInclude]
public Boundingbox boundingBox { private get; init; }

public Rectangle BoundingRect { get { return new Rectangle(boundingBox.originX, boundingBox.originY, boundingBox.width, boundingBox.height); } }
}

public class Boundingbox
{
public int originX { get; set; }
public int originY { get; set; }
public int width { get; set; }
public int height { get; set; }
public int angle { get; set; }
}

public class Category
{
public float Score { get; set; }
public int Index { get; set; }
public string CategoryName { get; set; }
public string DisplayName { get; set; }
}

public class Keypoint
{
public float x { private get; init; }
public float y { private get; init; }
public Vector2 Position { get { return new Vector2(x, y); } }
public int Score { get; set; }
public string Label { get; set; }
}

public class Resolution
{
public int Width { get; set; }
public int Height { get; set; }
}
}
54 changes: 54 additions & 0 deletions src/FaceLandmarkResults.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
using System.Text.Json.Serialization;

namespace MediaPipe.FaceLandmarkResults
{
public class FaceLandmarkRoot
{
public Facelandmarkresults FaceLandmarkResults { get; set; }
public Resolution Resolution { get; set; }
}

public class Facelandmarkresults
{
public Facelandmark[][] FaceLandmarks { get; set; }
public Faceblendshape[] FaceBlendshapes { get; set; }
public Facialtransformationmatrix[] FacialTransformationMatrixes { get; set; }
}

public class Facelandmark
{
public float x { private get; init; }
public float y { private get; init; }
public float z { private get; init; }

public Vector3 Position { get { return new Vector3(x, y, z); } }
}

public class Faceblendshape
{
public Category[] Categories { get; set; }
public int HeadIndex { get; set; }
public string HeadName { get; set; }
}

public class Category
{
public int Index { get; set; }
public float Score { get; set; }
public string CategoryName { get; set; }
public string DisplayName { get; set; }
}

public class Facialtransformationmatrix
{
public int Rows { get; set; }
public int Columns { get; set; }
public float[] Data { get; set; }
}

public class Resolution
{
public int Width { get; set; }
public int Height { get; set; }
}
}
61 changes: 61 additions & 0 deletions src/GestureResults.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
namespace MediaPipe.GestureResults
{
public class Rootobject
{
public Gestureresults gestureResults { get; set; }
public Resolution resolution { get; set; }
}

public class Gestureresults
{
public Gesture[][] gestures { get; set; }
public Landmark[][] landmarks { get; set; }
public Worldlandmark[][] worldLandmarks { get; set; }
public Handedness[][] handedness { get; set; }
public Handedness1[][] handednesses { get; set; }
}

public class Gesture
{
public float score { get; set; }
public int index { get; set; }
public string categoryName { get; set; }
public string displayName { get; set; }
}

public class Landmark
{
public float x { get; set; }
public float y { get; set; }
public float z { get; set; }
}

public class Worldlandmark
{
public float x { get; set; }
public float y { get; set; }
public float z { get; set; }
}

public class Handedness
{
public float score { get; set; }
public int index { get; set; }
public string categoryName { get; set; }
public string displayName { get; set; }
}

public class Handedness1
{
public float score { get; set; }
public int index { get; set; }
public string categoryName { get; set; }
public string displayName { get; set; }
}

public class Resolution
{
public int width { get; set; }
public int height { get; set; }
}
}
52 changes: 52 additions & 0 deletions src/HandResults.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
namespace MediaPipe.HandResults
{
public class Rootobject
{
public Handresults handResults { get; set; }
public Resolution resolution { get; set; }
}

public class Handresults
{
public Landmark[][] landmarks { get; set; }
public Worldlandmark[][] worldLandmarks { get; set; }
public Handedness[][] handednesses { get; set; }
public Handedness1[][] handedness { get; set; }
}

public class Landmark
{
public float x { get; set; }
public float y { get; set; }
public float z { get; set; }
}

public class Worldlandmark
{
public float x { get; set; }
public float y { get; set; }
public float z { get; set; }
}

public class Handedness
{
public float score { get; set; }
public int index { get; set; }
public string categoryName { get; set; }
public string displayName { get; set; }
}

public class Handedness1
{
public float score { get; set; }
public int index { get; set; }
public string categoryName { get; set; }
public string displayName { get; set; }
}

public class Resolution
{
public int width { get; set; }
public int height { get; set; }
}
}
27 changes: 27 additions & 0 deletions src/ImageResults.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
namespace VL.MediaPipe.ImageResults
{
public class Rootobject
{
public Imageresults imageResults { get; set; }
public Resolution resolution { get; set; }
}

public class Imageresults
{
public Classification[] classifications { get; set; }
public int timestampMs { get; set; }
}

public class Classification
{
public object[] categories { get; set; }
public int headIndex { get; set; }
public string headName { get; set; }
}

public class Resolution
{
public int width { get; set; }
public int height { get; set; }
}
}
3 changes: 2 additions & 1 deletion src/ModelEnums.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
namespace MediaPipe;

public enum PoseModels { Lite, Full, Heavy };
public enum PoseModels { Lite, Full, Heavy };
public enum SegmentationModels { selfieSquare, selfieLandscape, hairSegmenter, selfieMulticlass, deepLabV3 };
Loading

0 comments on commit f007e2d

Please sign in to comment.