From 79d908a964c478e4396de78408aa92868c6718aa Mon Sep 17 00:00:00 2001 From: otmanon Date: Tue, 31 Oct 2023 11:12:29 -0400 Subject: [PATCH] added documentation --- demos/__init__.py | 0 src/face_capture.py | 13 ++ src/fast_cody/__init__.py | 2 +- src/fast_cody/apps/animate_rig.py | 55 ----- src/fast_cody/apps/cd_demo_pose_tracker.py | 191 ------------------ src/fast_cody/apps/face_mesh_mediapipe | 100 --------- .../apps/interactive_cd_affine_handle.py | 9 +- .../apps/interactive_cd_face_tracking.py | 6 +- src/fast_cody/apps/test_FAST_IK.py | 92 --------- src/fast_cody/apps/test_FAST_IK_CD_tex.py | 132 ------------ src/fast_cody/apps/test_FAST_IK_tex.py | 99 --------- .../complementary_constraint_matrix.py | 14 +- src/fast_cody/deformation_jacobian.py | 8 +- src/fast_cody/diffuse_weights.py | 14 +- src/fast_cody/laplacian_eigenmodes.py | 16 -- src/fast_cody/mediapipe_face_captor.py | 66 +++--- src/fast_cody/momentum_leaking_matrix.py | 15 +- src/fast_cody/one_euro_filter.py | 7 +- src/fast_cody/viewers/__init__.py | 2 +- .../interactive_handle_subspace_viewer.py | 23 ++- .../viewers/interactive_handle_viewer.py | 80 -------- 21 files changed, 82 insertions(+), 862 deletions(-) create mode 100644 demos/__init__.py create mode 100644 src/face_capture.py delete mode 100644 src/fast_cody/apps/animate_rig.py delete mode 100644 src/fast_cody/apps/cd_demo_pose_tracker.py delete mode 100644 src/fast_cody/apps/face_mesh_mediapipe delete mode 100644 src/fast_cody/apps/test_FAST_IK.py delete mode 100644 src/fast_cody/apps/test_FAST_IK_CD_tex.py delete mode 100644 src/fast_cody/apps/test_FAST_IK_tex.py delete mode 100644 src/fast_cody/viewers/interactive_handle_viewer.py diff --git a/demos/__init__.py b/demos/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/face_capture.py b/src/face_capture.py new file mode 100644 index 0000000..72fbb6d --- /dev/null +++ b/src/face_capture.py @@ -0,0 +1,13 @@ +# import fast_cody as fcd +# import time +# +# # captor = fcd.mediapipe_face_captor(draw_landmarks=True) +# # for i in range(100): +# # [R, info] = captor.query_rotation() +# # captor.imshow() +# # print(R) +# # captor.release() +# +# from fast_cody.apps.interactive_cd_face_tracking import interactive_cd_face_tracking +# +# interactive_cd_face_tracking() \ No newline at end of file diff --git a/src/fast_cody/__init__.py b/src/fast_cody/__init__.py index 41b0c7c..138763e 100644 --- a/src/fast_cody/__init__.py +++ b/src/fast_cody/__init__.py @@ -45,7 +45,7 @@ from .viewers.WeightsViewer import WeightsViewer from .viewers.ClustersViewer import ClustersViewer from .viewers.interactive_handle_subspace_viewer import interactive_handle_subspace_viewer -from .viewers.interactive_handle_viewer import interactive_handle_viewer +# from .viewers.interactive_handle_viewer import interactive_handle_viewer # set data path and shaders path import os diff --git a/src/fast_cody/apps/animate_rig.py b/src/fast_cody/apps/animate_rig.py deleted file mode 100644 index c95ebec..0000000 --- a/src/fast_cody/apps/animate_rig.py +++ /dev/null @@ -1,55 +0,0 @@ -import polyscope as ps -import numpy as np -import igl -import fast_cody as fcd -import fast_cd_pyb as fcdp -name = "stingray" - - -frame = 20 -num_modes = 3 - - -msh_file = "./data/stingray.msh" -rig_dir = "./data/rigs/skeleton_rig/" -rig_json = rig_dir + "./skeleton_rig.json" -anim_json = "./data/rigs/skeleton_rig/anim/flap.json" - - -[V, F, T] = fcd.readMSH(msh_file) -[V, so, to] = fcd.scale_and_center_geometry(V, 1, np.array([[0, 0, 0.]])) - - -[W, P0,pI, bl, Vs, Fs, rig_type ] = fcd.read_rig_from_json(rig_json) -A = np.identity(4)[:3, :4] - -if (rig_type == "surface"): - W = fcd.surface_to_volume_weights(W, Vs, V, T) - a = fcd.fit_rig_to_mesh_surface(V, T, Vs, P0) - fcd.write_volume_mesh = (W, P0, pI, bl, "volume", rig_dir + "./volume_skeleton_rig.json") -else: - [P0, A] = fcd.fit_rig_to_mesh(V, Vs, P0) -#read rig animations -anim_P = fcd.read_anim_from_json(anim_json) -anim_P = fcd.transform_rig_parameters_anim(anim_P, A) -p0 = np.reshape(P0, (P0.shape[0]*3, 1), order="F") -anim_P = fcd.world_to_rel_rig_anim(anim_P, p0) -J = fcd.lbs_jacobian(V, W) - -ps.init() - -F = igl.boundary_facets(T) -mesh = ps.register_volume_mesh(name="rest", vertices=V, tets=T) - -step = 0 -num_frames = anim_P.shape[1] -def callback(): - global step - u = J @ anim_P[:, step%num_frames] #world space rig parameters - U = np.reshape(u, (u.shape[0]//3, 3), order="F") - mesh.update_vertex_positions(U) - step += 1 - - -ps.set_user_callback(callback) -ps.show() diff --git a/src/fast_cody/apps/cd_demo_pose_tracker.py b/src/fast_cody/apps/cd_demo_pose_tracker.py deleted file mode 100644 index 61c2c5e..0000000 --- a/src/fast_cody/apps/cd_demo_pose_tracker.py +++ /dev/null @@ -1,191 +0,0 @@ -import cv2 -import mediapipe as mp -import numpy as np -import scipy as sp -import os -import igl -import fast_cd_pyb as fcd -import pose_landmarks_to_positions -import json - -#mediapipe confiig for pose sceleton -mp_drawing = mp.solutions.drawing_utils -mp_drawing_styles = mp.solutions.drawing_styles -mp_pose = mp.solutions.pose -headI = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -rightArmI = [12, 14, 16, 18, 20, 22] -leftArmI = [11, 13, 15, 17, 19, 21] -bodI = [11, 12, 24, 23] -leftLegI = [24, 26, 28, 30, 32] -rightLegI = [23, 25, 27, 29, 31] - -################## FAST CD CONFIG ############### - -name = "charizard" -mesh_file = "./data/" + name + ".msh" -cache_dir = "./cache/" + name + "/" -meta_file = cache_dir + "/meta.json" -[V, F, T] = fcd.readMSH(mesh_file) -[V, so, to] = fcd.scale_and_center_geometry(V, 1, np.array([[0, 0, 0.]])) -F = igl.boundary_facets(T); - - -############# FAST CD RIG ##################### -rightArmInd = 2 # indeces in weight matrix corresponding to right Arm -leftArmInd = 3 -rig_path = "./data/charizard_skeleton_rig_mp.json" -f = open(rig_path) -j = json.load(f) -W = np.array(j["W"]) -Vs = np.array(j["V"]) -P0 = np.array(j["p0"], order="F").reshape((6*4, 3)) -[W, P0] = fcd.fit_rig_to_mesh(W, P0, Vs, V, T); -#w = W[:, rightArmInd]; -#W = W[:, [rightArmInd, leftArmInd]] -J = fcd.lbs_jacobian(V, W); -##### FAST CD SIMULATION PARAMETERS -write_cache = True -read_cache = True -num_modes = 40 -num_clusters = 200 -num_clustering_features = 10 -mu = 100 -h = 1e-2 -lam = 0 -mode_type = 'skinning' -do_inertia = True - -cache_params = {"num_modes" : num_modes, -"num_clusters" : num_clusters, -"mu" : mu, "h" : h, -"lambda" : lam, -"mode_type" : mode_type, -"do_inertia" : do_inertia} -### Check cache for stored data -solver_params = fcd.cd_arap_local_global_solver_params(True, 10, 1e-3) -sim = fcd.fast_cd_arap_sim(); -well_read = False -if read_cache and os.path.isfile(meta_file) : - f = open(meta_file) - data = json.load(f) - if data == cache_params: - sim = fcd.fast_cd_arap_sim(cache_dir, solver_params) - B2 = sim.params.B.copy(); - labels2 = sim.params.labels.copy() - sim.params = fcd.fast_cd_sim_params(V, T, B2, labels2, J, mu, lam, h, do_inertia) - well_read = True -if not well_read: - # W2 = W[:, [leftArmInd, rightArmInd]]; - #J2 = fcd.lbs_jacobian(V, W2); - [B, Ws, L] = fcd.get_modes(V, T, W, J, mode_type, num_modes) - [labels, C] = fcd.compute_clusters(T, B, L, num_clusters, num_clustering_features) - sim_params = fcd.fast_cd_sim_params(V, T, B, labels, J, mu, lam, h, do_inertia) - sim = fcd.fast_cd_arap_sim(sim_params, solver_params) - if (write_cache): - sim.save(cache_dir); - with open(meta_file, 'w') as outfile: - json.dump(cache_params, outfile, indent=2) - - -# set sim state -z0 = np.zeros((num_modes*12, 1)) -P = np.zeros((6, 4, 3), order="F"); -P[:, :3, :3] = np.identity(3); -p = P.reshape(72, 1) -st = fcd.cd_sim_state(z0, z0, p, p) - -# momentum leaking matrix -M = fcd.massmatrix(V, T) -D = fcd.momentum_leaking_matrix(V, T) -md = np.tile((M@ D).diagonal(), (3)) -MD = sp.sparse.diags(md) -BMDJ = sim.params.B.T @ MD @ J - - -#init 3D libigl viewer -viewer = fcd.fast_cd_viewer() -viewer.set_mesh(V, F, 0) -viewer.invert_normals(True, 0) -color = np.array([144, 210, 236])/255.0 -viewer.set_color(color, 0) -viewer.set_camera_zoom(2); -viewer.set_camera_eye( [2, 1, 5 ]) - -cap = cv2.VideoCapture(0) - - -callibrated = False -Dref = None -def callback(): - global J, callibrated, Dref, sim, st - # For webcam input: - with mp_pose.Pose( - min_detection_confidence=0.5, - min_tracking_confidence=0.5, model_complexity=0, smooth_landmarks = True) as pose: - if cap.isOpened(): - success, image = cap.read() - if not success: - print("Ignoring empty camera frame.") - # If loading a video, use 'break' instead of 'continue'. - return - - # To improve performance, optionally mark the image as not writeable to - # pass by reference. - image.flags.writeable = False - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - results = pose.process(image) - if (results.pose_landmarks): - - D, vis = pose_landmarks_to_positions.pose_landmarks_to_numpy(results.pose_landmarks) - D[:, 2] = 0 - if (np.all(vis[rightArmI] > 0.8) and np.all(vis[leftArmI] > 0.8) and not (callibrated)): - Dref = D - callibrated = True - if (callibrated): - #fit right arm rotation - DrightArm = D[rightArmI] - DrefrightArm = Dref[rightArmI]; - K = DrightArm - DrightArm.mean(axis=0); - KX = DrefrightArm - DrefrightArm.mean(axis=0); - [Rr, S] = igl.polar_dec(K.T @ KX); - - #fit left arm rotation - DleftArm = D[leftArmI] - DrefleftArm = Dref[leftArmI]; - K = DleftArm - DleftArm.mean(axis=0); - KX = DrefleftArm - DrefleftArm.mean(axis=0); - [Rl, S] = igl.polar_dec(K.T @ KX); - - #flatten this appropriately and fill with rotation - P = np.zeros((6, 4, 3), order="F"); - P[:, :3, :3] = np.identity(3); #initialize all rig matrices to identity - P[rightArmInd, :3, :3] = Rr.transpose() - P[leftArmInd, :3, :3] = Rl.transpose() - Ps = P.reshape((24, 3)) - p = Ps.reshape((72, 1), order="F") - - - #get momentum leaking force/ boundary conditions - f_ext = sim.params.invh2 * BMDJ @ (2.0 * st.p_curr.reshape((72, 1)) - st.p_prev.reshape((72, 1)) - p) - bc = np.array([[]], dtype=np.float64).T - #initial guess... not sure if i shoulddelete the need of step to give as input z - z = st.z_curr - z = sim.step(z, p, st, f_ext, bc).reshape((12*num_modes, 1), order="F") - st.update(z, p); - - U = np.reshape(J@p+ sim.params.B@z, (int(J.shape[0]/3), 3), order="F") - viewer.set_vertices(U, 0 ) - # image.flags.writeable = True - image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - # mp_drawing.draw_landmarks( - # image, - # results.pose_landmarks, - # mp_pose.POSE_CONNECTIONS, - # landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style()) - cv2.imshow('MediaPipe Pose', cv2.flip(image, 1)) - - -viewer.set_pre_draw_callback(callback) -viewer.launch() - -cap.release() \ No newline at end of file diff --git a/src/fast_cody/apps/face_mesh_mediapipe b/src/fast_cody/apps/face_mesh_mediapipe deleted file mode 100644 index 983a2f8..0000000 --- a/src/fast_cody/apps/face_mesh_mediapipe +++ /dev/null @@ -1,100 +0,0 @@ -import cv2 -import mediapipe as mp -mp_drawing = mp.solutions.drawing_utils -mp_drawing_styles = mp.solutions.drawing_styles -mp_face_mesh = mp.solutions.face_mesh - -# For static images: -IMAGE_FILES = [] -drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1) -with mp_face_mesh.FaceMesh( - static_image_mode=True, - max_num_faces=1, - refine_landmarks=True, - min_detection_confidence=0.5) as face_mesh: - for idx, file in enumerate(IMAGE_FILES): - image = cv2.imread(file) - # Convert the BGR image to RGB before processing. - results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) - - # Print and draw face mesh landmarks on the image. - if not results.multi_face_landmarks: - continue - annotated_image = image.copy() - for face_landmarks in results.multi_face_landmarks: - print('face_landmarks:', face_landmarks) - mp_drawing.draw_landmarks( - image=annotated_image, - landmark_list=face_landmarks, - connections=mp_face_mesh.FACEMESH_TESSELATION, - landmark_drawing_spec=None, - connection_drawing_spec=mp_drawing_styles - .get_default_face_mesh_tesselation_style()) - mp_drawing.draw_landmarks( - image=annotated_image, - landmark_list=face_landmarks, - connections=mp_face_mesh.FACEMESH_CONTOURS, - landmark_drawing_spec=None, - connection_drawing_spec=mp_drawing_styles - .get_default_face_mesh_contours_style()) - mp_drawing.draw_landmarks( - image=annotated_image, - landmark_list=face_landmarks, - connections=mp_face_mesh.FACEMESH_IRISES, - landmark_drawing_spec=None, - connection_drawing_spec=mp_drawing_styles - .get_default_face_mesh_iris_connections_style()) - cv2.imwrite('/tmp/annotated_image' + str(idx) + '.png', annotated_image) - -# For webcam input: -drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1) -cap = cv2.VideoCapture(0) -with mp_face_mesh.FaceMesh( - max_num_faces=1, - refine_landmarks=True, - min_detection_confidence=0.5, - min_tracking_confidence=0.5) as face_mesh: - while cap.isOpened(): - success, image = cap.read() - if not success: - print("Ignoring empty camera frame.") - # If loading a video, use 'break' instead of 'continue'. - continue - - # To improve performance, optionally mark the image as not writeable to - # pass by reference. - image.flags.writeable = False - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - results = face_mesh.process(image) - - # Draw the face mesh annotations on the image. - image.flags.writeable = True - image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - if results.multi_face_landmarks: - for face_landmarks in results.multi_face_landmarks: - mp_drawing.draw_landmarks( - image=image, - landmark_list=face_landmarks, - connections=mp_face_mesh.FACEMESH_TESSELATION, - landmark_drawing_spec=None, - connection_drawing_spec=mp_drawing_styles - .get_default_face_mesh_tesselation_style()) - mp_drawing.draw_landmarks( - image=image, - landmark_list=face_landmarks, - connections=mp_face_mesh.FACEMESH_CONTOURS, - landmark_drawing_spec=None, - connection_drawing_spec=mp_drawing_styles - .get_default_face_mesh_contours_style()) - mp_drawing.draw_landmarks( - image=image, - landmark_list=face_landmarks, - connections=mp_face_mesh.FACEMESH_IRISES, - landmark_drawing_spec=None, - connection_drawing_spec=mp_drawing_styles - .get_default_face_mesh_iris_connections_style()) - # Flip the image horizontally for a selfie-view display. - cv2.imshow('MediaPipe Face Mesh', cv2.flip(image, 1)) - if cv2.waitKey(5) & 0xFF == 27: - break -cap.release() \ No newline at end of file diff --git a/src/fast_cody/apps/interactive_cd_affine_handle.py b/src/fast_cody/apps/interactive_cd_affine_handle.py index 3c9b6ef..dd7aff4 100644 --- a/src/fast_cody/apps/interactive_cd_affine_handle.py +++ b/src/fast_cody/apps/interactive_cd_affine_handle.py @@ -119,21 +119,14 @@ def interactive_cd_affine_handle(msh_file=None, Ws=None, l=None, mu=1e4, rho=1e3 def pre_draw_callback(): nonlocal J, B, T0, sim, st, step p = viewer.T0[0:3, :].reshape( (12, 1)) - z = sim.step( p, st) - st.update(z, p) - # U = np.reshape(J @ p + B @ z, (J.shape[0]//3, 3), order="F") # full positions - viewer.update_subspace_coefficients(z, p) - - step += 1 - viewer = fc.viewers.interactive_handle_subspace_viewer(V, T, Wp, Ws, pre_draw_callback,T0=T0, texture_png=texture_png, texture_obj=texture_obj, - t0=to, s0=so) + t0=to, s0=so, init_guizmo=True) viewer.launch() diff --git a/src/fast_cody/apps/interactive_cd_face_tracking.py b/src/fast_cody/apps/interactive_cd_face_tracking.py index dc83cff..769847d 100644 --- a/src/fast_cody/apps/interactive_cd_face_tracking.py +++ b/src/fast_cody/apps/interactive_cd_face_tracking.py @@ -39,7 +39,7 @@ def interactive_cd_face_tracking(msh_file=None, Wp=None, bI=0, Ws=None, l=None, constraint_enforcement="optimal", results_dir=None, read_cache=False, texture_png=None, texture_obj=None, - draw_landmarks=False): + draw_landmarks=True): """ Runs a standard interactive fast CD simulation, where the user can manipulate a single affine handle with a mediapipe face tracker in real time. @@ -127,7 +127,7 @@ def interactive_cd_face_tracking(msh_file=None, Wp=None, bI=0, Ws=None, l=None, T0 = np.tile(T0, (Wp.shape[1], 1)) p0 = np.reshape( T0, (Wp.shape[1]*12, 1), order="F") - face_captor = fc.mediapipe_face_captor(draw_landmarks=draw_landmarks) + st = fc.fast_cd_state(z0, p0) def user_callback(): @@ -146,6 +146,8 @@ def user_callback(): viewer = fc.viewers.interactive_handle_subspace_viewer(V, T, Wp, Ws, user_callback, texture_png=texture_png, texture_obj=texture_obj, t0=to, s0=so, init_guizmo=False, max_fps=100) + face_captor = fc.mediapipe_face_captor(draw_landmarks=draw_landmarks) + viewer.launch() face_captor.release() diff --git a/src/fast_cody/apps/test_FAST_IK.py b/src/fast_cody/apps/test_FAST_IK.py deleted file mode 100644 index 93eddad..0000000 --- a/src/fast_cody/apps/test_FAST_IK.py +++ /dev/null @@ -1,92 +0,0 @@ -import fast_cd_pyb as fcd -import igl -import numpy as np -import scipy as sp -import polyscope as ps -# msh_file = "./data/charizard/charizard.msh" -# rig_dir = "./data/charizard/skeleton_rig_arms_legs/" -# rig_file = rig_dir + "/skeleton_rig_arms_legs.json" - -# msh_file = "./data/king_ghidorah/king_g.msh" -# rig_dir = "./data/king_ghidorah/skeleton_rig/" -# rig_file = rig_dir + "/skeleton_rig.json" -# cache_ik = "./results/fast_ik_cache/king_ghidorah/ik/" -[V, F, T] = fcd.readMSH(msh_file) -[W, P0, pI, bl, Vs, Fs, rig_type] = fcd.read_rig_from_json(rig_file) -A = np.identity(4)[:3, :4] -if (rig_type == "surface"): - W = fcd.surface_to_volume_weights(W, Vs, V, T) - [P0, A] = fcd.fit_rig_to_mesh_surface(V, T, Vs, P0) - fcd.write_volume_mesh = (W, P0, pI, bl, "volume", rig_dir + "./volume_skeleton_rig.json") -else: - [P0, A] = fcd.fit_rig_to_mesh(V, Vs, P0) - -num_clusters = 10; -sub = fcd.fast_ik_subspace(num_clusters, False, "./debug/"); - - -sub.init_with_cache(V, T, W, True, True, - cache_ik, True); - - -bI = np.array([1]) -solver_params = fcd.cd_arap_local_global_solver_params(True, 10, 1e-3) -bI = np.array([]) -S = fcd.selection_matrix(bI, V.shape[0], V.shape[1]) -labels = np.copy(sub.labels) -sim = fcd.fast_ik_sim(V, T, W, labels, S, solver_params); - -z = np.zeros((W.shape[1] * 12, 1)) -z = np.tile(np.identity(4)[:3, :4], W.shape[1]).reshape(z.shape) -state = fcd.sim_state(z, z) - -print("Done!") - - - -# mesh = ps.register_volume_mesh("mesh", V, T) - -# B = fcd.lbs_jacobian(V, W) -# num_b = P0.shape[0]/4 -# p = np.zeros((0, 1)) -# z = np.zeros(B.shape[1]) -# state = fcd.sim_state(z, z) -# f_ext = np.zeros(z.shape); -# step = 0 -# d = [1] -f_ext = np.zeros(z.shape) -J = fcd.lbs_jacobian(V, W) -bc = np.array([]) - -v = fcd.fast_cd_viewer_vertex_selector() -F = igl.boundary_facets(T) - -v.set_mesh(V, F, 0) -v.set_show_lines(False, 0) -v.set_face_based(False, 0) -v.invert_normals(True, 0) - -X = V -def callback(): - global bc, X - [C, CI, new_handles] = v.query_new_handles_on_mesh(X, F) - if (new_handles): - S = fcd.selection_matrix(CI, V.shape[0], V.shape[1]) - sim.set_equality_constraint(S) - bc = np.reshape(C, (C.shape[0] * C.shape[1]), order="F") - # - # # global step - # # bc = step * np.array(d) - z_next = sim.step(z, state, f_ext, bc) - - X = (J @ z_next).reshape(V.shape, order="F") - - v.set_vertices(X, 0) - v.compute_normals(0) - - return - - -v.set_pre_draw_callback(callback) - -v.launch() \ No newline at end of file diff --git a/src/fast_cody/apps/test_FAST_IK_CD_tex.py b/src/fast_cody/apps/test_FAST_IK_CD_tex.py deleted file mode 100644 index 7539c62..0000000 --- a/src/fast_cody/apps/test_FAST_IK_CD_tex.py +++ /dev/null @@ -1,132 +0,0 @@ -import fast_cd_pyb as fcd -import igl -import numpy as np -import scipy as sp -import polyscope as ps -msh_file = "./data/charizard/charizard.msh" -rig_dir = "./data/charizard/skeleton_rig_arms_legs/" -rig_file = rig_dir + "/skeleton_rig_arms_legs.json" - -texture_obj = "./data/charizard/charizard_tex.obj" -texture_png = "./data/charizard/charizard_tex.png" - -read_cache = False -write_cache = True - - -######## SIM PARAMS ######### -num_clusters = 50 -num_skinning_modes =15 -mu = 20.0 -lam = 0 -h = 1e-2 -do_inertia = True -cache_cd = "./results/fast_cd_ik_cache/cd/" -cache_ik = "./results/fast_cd_ik_cache/ik/" -########## LOAD RIG ########## -[V, F, T] = fcd.readMSH(msh_file) -[W, P0, pI, bl, Vs, Fs, rig_type] = fcd.read_rig_from_json(rig_file) -A = np.identity(4)[:3, :4] -if (rig_type == "surface"): - W = fcd.surface_to_volume_weights(W, Vs, V, T) - [P0, A] = fcd.fit_rig_to_mesh_surface(V, T, Vs, P0) - fcd.write_volume_mesh = (W, P0, pI, bl, "volume", rig_dir + "./volume_skeleton_rig.json") -else: - [P0, A] = fcd.fit_rig_to_mesh(V, Vs, P0) -J = fcd.lbs_jacobian(V, W) # rig jacobian - -########## Load IK SIM ########## -num_clusters = 10; -sub = fcd.fast_ik_subspace(num_clusters, read_cache, "./debug/"); -sub.init_with_cache(V, T, W, read_cache, write_cache, - cache_ik, True); -bI = np.array([1]) -solver_params = fcd.local_global_solver_params(True, 10, 1e-3) -bI = np.array([]) -S = fcd.selection_matrix(bI, V.shape[0], V.shape[1]) -labels2 = np.copy(sub.labels) -sim_ik = fcd.fast_ik_sim(V, T, W, labels2, S, solver_params); - -p = np.zeros((W.shape[1] * 12, 1)) -p = np.tile(np.identity(4)[:3, :4], W.shape[1]).reshape(p.shape) -state_ik = fcd.sim_state(p, p) - -#repeat identity matrix for each column in W and flatten with order="F" -# p = np.tile(np.identity(4)[:3, :4], W.shape[1]).reshape((V.shape[0] * 12, 1), order="F") - - -sub_cd = fcd.fast_cd_subspace(num_skinning_modes, "cd_momentum_leak", - "skinning", num_clusters, num_skinning_modes, True) -sub_cd.init_with_cache(V, T, J, read_cache, write_cache, - cache_cd, cache_cd, True, True) -solver_params = fcd.local_global_solver_params(False, 100, 1e-4) -labels = np.copy(sub_cd.labels) # need to copy this, for some reason its not writeable otherwise -B = np.copy(sub_cd.B) -lam = 0.49 -sim_params = fcd.fast_cd_corot_sim_params(V, T, B, labels, J, mu, lam, h, do_inertia) -# sim_cd = fcd.fast_cd_arap_sim(cache_cd, sim_params, solver_params, False, write_cache) -sim_cd = fcd.fast_cd_corot_sim(sim_params, solver_params) - -z = np.zeros((sub_cd.W.shape[1] * 12, 1)); -# z = np.tile(np.identity(4)[:3, :4], sub_cd.W.shape[1]).reshape(z.shape) -state_cd = fcd.cd_sim_state(z, z, p, p) - -# mesh = ps.register_volume_mesh("mesh", V, T) - -# B = fcd.lbs_jacobian(V, W) -# num_b = P0.shape[0]/4 -# p = np.zeros((0, 1)) -# z = np.zeros(B.shape[1]) -# state = fcd.sim_state(z, z) -# f_ext = np.zeros(z.shape); -# step = 0 -# d = [1] -f_ext_cd = np.zeros(z.shape) -f_ext_ik = np.zeros(p.shape) - -bc_ik = np.array([]) -bc_cd = np.array([]) - -########## INIT VIEWER ########## -v = fcd.fast_cd_viewer_vertex_selector() -F = igl.boundary_facets(T) -[Vf, TC, N, Ff, FTC, FN] = fcd.readOBJ_tex(texture_obj); -P = sp.sparse.kron(sp.sparse.identity(3), fcd.prolongation(Vf, V, T)) -v.set_mesh(Vf, Ff, 0) -v.set_show_lines(False, 0) -# v.set_mesh(V, F, 0) -v.set_texture(texture_png, TC, FTC, 0) -# v.invert_normals(True, 0) -v.set_face_based(False, 0) -Vc = V -V_next = Vf -def callback(): - global bc_ik, V_next, Vc - [C, CI, new_handles] = v.query_new_handles_on_mesh(Vc, F) - if (new_handles): - S = fcd.selection_matrix(CI, V.shape[0], V.shape[1]) - sim_ik.set_equality_constraint(S) - bc_ik = np.reshape(C, (C.shape[0] * C.shape[1]), order="F") - # p_next_ik = p - p_next_ik = sim_ik.step(p, state_ik, f_ext_ik, bc_ik) - state_ik.update(p_next_ik) - - z_next_cd = sim_cd.step(z, p_next_ik, state_cd, f_ext_cd, bc_cd) - state_cd.update(z_next_cd, p_next_ik) - - x = np.reshape(V, (V.shape[0] * V.shape[1]), order="F") - Vc = (J @ p_next_ik + B@z_next_cd).reshape(V.shape, order="F") - # Vc = V + (B@z_next_cd).reshape(V.shape, order="F") - u = B@z_next_cd + J@p_next_ik - x - - uf = P@u; - V_next = uf.reshape((Vf.shape[0], 3), order="F") + Vf - v.set_vertices(V_next, 0) - v.compute_normals(0) - - return - - -v.set_pre_draw_callback(callback) - -v.launch() \ No newline at end of file diff --git a/src/fast_cody/apps/test_FAST_IK_tex.py b/src/fast_cody/apps/test_FAST_IK_tex.py deleted file mode 100644 index 8986b60..0000000 --- a/src/fast_cody/apps/test_FAST_IK_tex.py +++ /dev/null @@ -1,99 +0,0 @@ -import fast_cd_pyb as fcd -import igl -import numpy as np -import scipy as sp -import polyscope as ps -msh_file = "./data/charizard/charizard.msh" -rig_dir = "./data/charizard/skeleton_rig_arms_legs/" -rig_file = rig_dir + "/skeleton_rig_arms_legs.json" - -texture_obj = "./data/charizard/charizard_tex.obj" -texture_png = "./data/charizard/charizard_tex.png" - - -[V, F, T] = fcd.readMSH(msh_file) -[W, P0, pI, bl, Vs, Fs, rig_type] = fcd.read_rig_from_json(rig_file) -A = np.identity(4)[:3, :4] -if (rig_type == "surface"): - W = fcd.surface_to_volume_weights(W, Vs, V, T) - [P0, A] = fcd.fit_rig_to_mesh_surface(V, T, Vs, P0) - fcd.write_volume_mesh = (W, P0, pI, bl, "volume", rig_dir + "./volume_skeleton_rig.json") -else: - [P0, A] = fcd.fit_rig_to_mesh(V, Vs, P0) - -num_clusters = 10; -sub = fcd.fast_ik_subspace(num_clusters, False, "./debug/"); - - -sub.init_with_cache(V, T, W, True, True, - "./data/charizard/skeleton_rig_arms_legs/cache2/", True); - - -bI = np.array([1]) -solver_params = fcd.cd_arap_local_global_solver_params(True, 10, 1e-3) -bI = np.array([]) -S = fcd.selection_matrix(bI, V.shape[0], V.shape[1]) -labels = np.copy(sub.labels) -sim = fcd.fast_ik_sim(V, T, W, labels, S, solver_params); - -z = np.zeros((W.shape[1] * 12, 1)) -z = np.tile(np.identity(4)[:3, :4], W.shape[1]).reshape(z.shape) -state = fcd.sim_state(z, z) - -print("Done!") - - - -# mesh = ps.register_volume_mesh("mesh", V, T) - -# B = fcd.lbs_jacobian(V, W) -# num_b = P0.shape[0]/4 -# p = np.zeros((0, 1)) -# z = np.zeros(B.shape[1]) -# state = fcd.sim_state(z, z) -# f_ext = np.zeros(z.shape); -# step = 0 -# d = [1] -f_ext = np.zeros(z.shape) -J = fcd.lbs_jacobian(V, W) -bc = np.array([]) - -v = fcd.fast_cd_viewer_vertex_selector() -F = igl.boundary_facets(T) -[Vf, TC, N, Ff, FTC, FN] = fcd.readOBJ_tex(texture_obj); -P = sp.sparse.kron(sp.sparse.identity(3), fcd.prolongation(Vf, V, T)) -v.set_mesh(Vf, Ff, 0) -v.set_show_lines(False, 0) - -v.set_texture(texture_png, TC, FTC, 0) -# v.invert_normals(True, 0) -v.set_face_based(False, 0) -Vc = V -V_next = Vf -def callback(): - global bc, V_next, Vc - [C, CI, new_handles] = v.query_new_handles_on_mesh(Vc, F) - if (new_handles): - S = fcd.selection_matrix(CI, V.shape[0], V.shape[1]) - sim.set_equality_constraint(S) - bc = np.reshape(C, (C.shape[0] * C.shape[1]), order="F") - # - # # global step - # # bc = step * np.array(d) - z_next = sim.step(z, state, f_ext, bc) - x = np.reshape(V, (V.shape[0] * V.shape[1]), order="F") - - Vc = (J @ z_next).reshape(V.shape, order="F") - u = J@z_next - x - - uf = P@u; - V_next = uf.reshape((Vf.shape[0], 3), order="F") + Vf - v.set_vertices(V_next, 0) - v.compute_normals(0) - - return - - -v.set_pre_draw_callback(callback) - -v.launch() \ No newline at end of file diff --git a/src/fast_cody/complementary_constraint_matrix.py b/src/fast_cody/complementary_constraint_matrix.py index b50699e..3d39075 100644 --- a/src/fast_cody/complementary_constraint_matrix.py +++ b/src/fast_cody/complementary_constraint_matrix.py @@ -3,19 +3,7 @@ import fast_cody as fc -''' -Inputs: -V - n x 3 mesh geometry -T - t x 4 tet indices -J - 3n x 12m rig jacobian matrix - -Optional: -dt - timestep used for momentum leaking matrix, (default=1/l^2) - -Outputs: -C - 12m x 3n complementary dynamics constraint matrix - -''' + def complementary_constraint_matrix(V, T, J, dt=None): """ Computes the complementarity constraint matrix ``` diff --git a/src/fast_cody/deformation_jacobian.py b/src/fast_cody/deformation_jacobian.py index ed1e960..64fd5eb 100644 --- a/src/fast_cody/deformation_jacobian.py +++ b/src/fast_cody/deformation_jacobian.py @@ -24,9 +24,11 @@ def deformation_jacobian(V, T): -------- Obtain a stacked t x 3 x 3 list of deformation gradients from each tet, from n x 3 positions U ``` - J = deformation_jacobian(X, T) - f = J @ U.flatten(order='F') - F = f.reshape(-1, 3, 3) + >>>import fast_cody as fcd + >>> [V, F, T] = fcd.fcd.get_data('cd_fish.msh') + >>>J = fcd.deformation_jacobian(X, T) + >>>f = J @ U.flatten(order='F') + >>>F = f.reshape(-1, 3, 3) ``` diff --git a/src/fast_cody/diffuse_weights.py b/src/fast_cody/diffuse_weights.py index 2f46eaf..5e28b5f 100644 --- a/src/fast_cody/diffuse_weights.py +++ b/src/fast_cody/diffuse_weights.py @@ -6,19 +6,7 @@ import fast_cody from .laplacian import laplacian -''' -Diffuses phi quantities on tet mesh Vv, Tv at nodes bI fro time - -Inputs ------- -Vv: nx3 volume vertices -Tv: tx4 volume tetrahedra -phi: c x b quantity to diffuse -bI: c x b indices at diffusion points - -returns -W: n x b diffused quantities over entire mesh -''' + def diffuse_weights(Vv, Tv, phi, bI, dt=None, normalize=True): """ Performs a diffusion on the tet mesh Vv, Tv at nodes bI for time dt. diff --git a/src/fast_cody/laplacian_eigenmodes.py b/src/fast_cody/laplacian_eigenmodes.py index 2032fe9..9f58765 100644 --- a/src/fast_cody/laplacian_eigenmodes.py +++ b/src/fast_cody/laplacian_eigenmodes.py @@ -12,23 +12,7 @@ from .project_out_subspace import project_out_subspace from .orthonormalize import orthonormalize from .eigs import eigs -''' -Constructs a physics subspace corresponding with skinning eigenmodes and skinning clusters -Inputs: - X: V x 3 vertex positions - T: F x 4 tet indices - num_modes: number of modes to use -(Optional) - cache_dir: directory to cache results in (default None) - read_cache: whether to read from cache or not (default False) - C : c x n constarint matrix that acts on the skinning weights (default None) - constraint_enforcement : method of enforcing constraint. Either "project" or "optimal" - for python, default is "project", because optimal makes eigendecomposition take way too long. -Outputs: - B: 3n x 12m subspace matrix - W: n x m skinning weights -''' def laplacian_eigenmodes(V, T, m, read_cache=False, cache_dir=None, J=None, mu=None, constraint_enforcement="optimal"): """ Computes Laplacian Eigenmodes for a given mesh. diff --git a/src/fast_cody/mediapipe_face_captor.py b/src/fast_cody/mediapipe_face_captor.py index 9265cf2..07d6c94 100644 --- a/src/fast_cody/mediapipe_face_captor.py +++ b/src/fast_cody/mediapipe_face_captor.py @@ -34,15 +34,17 @@ class mediapipe_face_captor(): Example ------- ``` - captor = mediapipe_face_captor() - for i in range(100): + import fast_cody as fcd + import time + captor = fcd.mediapipe_face_captor() + for i in range(1000): [R, info] = captor.query_rotation() captor.imshow() print(R) captor.release() ``` """ - def __init__(self, R0=None, draw_landmarks=False): + def __init__(self, R0=None, draw_landmarks=True): """ Parameters ---------- @@ -127,32 +129,36 @@ def imshow(s): """ Displays the image on the screen. If self.draw_landmarks is True, also draws landmarks on the face. """ - if s.draw_landmarks: - if s.multi_face_landmarks: - for face_landmarks in s.multi_face_landmarks: - s.mp_drawing.draw_landmarks( - image=s.image, - landmark_list=face_landmarks, - connections=s.mp_face_mesh.FACEMESH_TESSELATION, - landmark_drawing_spec=None, - connection_drawing_spec=s.mp_drawing_styles - .get_default_face_mesh_tesselation_style()) - s.mp_drawing.draw_landmarks( - image=s.image, - landmark_list=face_landmarks, - connections=s.mp_face_mesh.FACEMESH_CONTOURS, - landmark_drawing_spec=None, - connection_drawing_spec=s.mp_drawing_styles - .get_default_face_mesh_contours_style()) - s.mp_drawing.draw_landmarks( - image=s.image, - landmark_list=face_landmarks, - connections=s.mp_face_mesh.FACEMESH_IRISES, - landmark_drawing_spec=None, - connection_drawing_spec=s.mp_drawing_styles - .get_default_face_mesh_iris_connections_style()) - - cv2.imshow('MediaPipe Face Detection', cv2.flip(s.image, 1)) + if s.cap.isOpened(): + if s.draw_landmarks: + if s.multi_face_landmarks: + for face_landmarks in s.multi_face_landmarks: + s.mp_drawing.draw_landmarks( + image=s.image, + landmark_list=face_landmarks, + connections=s.mp_face_mesh.FACEMESH_TESSELATION, + landmark_drawing_spec=None, + connection_drawing_spec=s.mp_drawing_styles + .get_default_face_mesh_tesselation_style()) + s.mp_drawing.draw_landmarks( + image=s.image, + landmark_list=face_landmarks, + connections=s.mp_face_mesh.FACEMESH_CONTOURS, + landmark_drawing_spec=None, + connection_drawing_spec=s.mp_drawing_styles + .get_default_face_mesh_contours_style()) + s.mp_drawing.draw_landmarks( + image=s.image, + landmark_list=face_landmarks, + connections=s.mp_face_mesh.FACEMESH_IRISES, + landmark_drawing_spec=None, + connection_drawing_spec=s.mp_drawing_styles + .get_default_face_mesh_iris_connections_style()) + + + cv2.imshow('MediaPipe Face Detection', cv2.flip(s.image, 1)) + cv2.waitKey(1) + # cv2.waitKey() # if (record): # cv2.imwrite(results_dir + "/camera_stream/" + str(step).zfill(4) + ".png", image) # @@ -161,4 +167,4 @@ def imshow(s): def release(self): """ Releases the camera. """ - self.cap.release() \ No newline at end of file + self.cap.release() diff --git a/src/fast_cody/momentum_leaking_matrix.py b/src/fast_cody/momentum_leaking_matrix.py index 0184294..97862a2 100644 --- a/src/fast_cody/momentum_leaking_matrix.py +++ b/src/fast_cody/momentum_leaking_matrix.py @@ -4,20 +4,7 @@ from .diffuse_weights import diffuse_weights -''' -Constructs the momentum leaking matrix, that fudges the CD constraint -in order to let momentum leak from the rig to the mesh - -Inputs: -V - n x 3 mesh geometry -T - t x 4 tet indices - -Optional: -dt - float, used in diffusion (default 1/l^2 where l is the mean edge lengths) -expand_dim - bool, if True returns a 3nx3n diagonal matrix, -Outputs: -D - n x n diagonal sparse matrix with entries varying from 0 (momentum-fully leaking) to 1 (momentum not leaking) for each vertex -''' + def momentum_leaking_matrix(V, T, dt=None, pow=1): """ Constructs the momentum leaking matrix, that fudges the CD constraint to allow momentum to leak from diff --git a/src/fast_cody/one_euro_filter.py b/src/fast_cody/one_euro_filter.py index d344f12..6eb4e5a 100644 --- a/src/fast_cody/one_euro_filter.py +++ b/src/fast_cody/one_euro_filter.py @@ -18,10 +18,13 @@ class OneEuroFilter: -------- ``` >>> from fast_cody import OneEuroFilter - >>> x0 = np.zeros((3, 1)) + >>> import numpy as np + >>> mu = np.ones((3, 1)) + >>> x0 = np.random.randn(3, 1) + mu >>> f = OneEuroFilter(x0) - >>> x = np.random.rand(3, 1) + >>> x = np.random.randn(3, 1) + mu >>> x_filtered = f(x) + >>> print(x_filtered) ``` """ def __init__(self, x0, dx0=0.0, min_cutoff=1.0, beta=0.0, diff --git a/src/fast_cody/viewers/__init__.py b/src/fast_cody/viewers/__init__.py index 78810e9..6a8f5dd 100644 --- a/src/fast_cody/viewers/__init__.py +++ b/src/fast_cody/viewers/__init__.py @@ -1,4 +1,4 @@ -from .interactive_handle_viewer import interactive_handle_viewer +# from .interactive_handle_viewer import interactive_handle_viewer from .interactive_handle_subspace_viewer import interactive_handle_subspace_viewer from .WeightsViewer import WeightsViewer from .ClustersViewer import ClustersViewer diff --git a/src/fast_cody/viewers/interactive_handle_subspace_viewer.py b/src/fast_cody/viewers/interactive_handle_subspace_viewer.py index 05b49cc..0706137 100644 --- a/src/fast_cody/viewers/interactive_handle_subspace_viewer.py +++ b/src/fast_cody/viewers/interactive_handle_subspace_viewer.py @@ -55,11 +55,11 @@ class interactive_handle_subspace_viewer(): max_fps : int maximum fps for the viewer - Examples - -------- + Examples -------- + Simple script for using the viewer in a CD App ``` import fast_cody as fcd import numpy as np @@ -73,8 +73,8 @@ class interactive_handle_subspace_viewer(): [B, l, Ws] =fcd.skinning_subspace(V, T, num_modes, num_clusters, C=Cd) sim = fcd.fast_cd_sim(V, T, B, l, J, mu=1e4) z0 = np.zeros((num_modes*12, 1)) - T0 = np.identity(4).astype( dtype=np.float32, order="F"); - p0 = T0[0:3, :].reshape((12, 1)) + T0 = np.identity(4); + p0 = T0[0:3, :].reshape((-1, 1)) st = fcd.fast_cd_state(z0, p0) step = 0 def pre_draw_callback(): @@ -158,12 +158,15 @@ def guizmo_callback(self, A): def callback_key_pressed(s, key, modifier): if (key == ord('g') or key == ord('G')): - if (s.transform == "translate"): - s.transform = "rotate" - elif (s.transform == "rotate"): - s.transform = "scale" - elif (s.transform == "scale"): - s.transform = "translate" + if (s.init_guizmo): + if (s.transform == "translate"): + s.transform = "rotate" + elif (s.transform == "rotate"): + s.transform = "scale" + elif (s.transform == "scale"): + s.transform = "translate" + else: + print("Guizmo not initialized, pass init_guizmo=True to the viewer constructor") if (key == ord('c') or key==ord('C') ): s.vis_cd = not s.vis_cd return False diff --git a/src/fast_cody/viewers/interactive_handle_viewer.py b/src/fast_cody/viewers/interactive_handle_viewer.py deleted file mode 100644 index 55ec7db..0000000 --- a/src/fast_cody/viewers/interactive_handle_viewer.py +++ /dev/null @@ -1,80 +0,0 @@ -import igl -import numpy as np -import scipy as sp -import igl as igl - -import fast_cd_pyb as fcd - -class interactive_handle_viewer(): - """ - Viewer for interactive affine handle app. This viewer updates the draw call by sending - the full space mesh and rig parameters to the GPU, and then does the full space projection - - Parameters - ---------- - V : (n, 3) float numpy array - vertex positions - T : (f, 3) int numpy array - triangle indices - T0 : (4, 4) float numpy array - initial rig transform in world space - guizmo_callback : function - callback function for guizmo widget - pre_draw_callback : function - callback function for pre-draw step - texture_png : str - path to texture png file - texture_obj : str - path to texture obj file - t0 : (3,) float numpy array - initial mesh translation (to align the textured mesh with) - s0 : float - initial mesh scale (to align the textured mesh with) - """ - def __init__(self, V, T, T0, guizmo_callback, pre_draw_callback, - texture_png=None, texture_obj=None, t0=None, s0=None): - viewer = fcd.fast_cd_viewer() - F = igl.boundary_facets(T) - vis_texture = False - if texture_png is not None and texture_obj is not None: - vis_texture = True - self.vis_texture = vis_texture - - if not vis_texture: - viewer.set_mesh(V, F, 0) - viewer.invert_normals(True, 0) - color = np.array([144, 210, 236]) / 255.0 - viewer.set_color(color, 0) - viewer.init_guizmo(True, T0, guizmo_callback, "translate") - viewer.set_pre_draw_callback(pre_draw_callback) - self.V = V - if vis_texture: - [Vf, TC, N, Ff, FTC, FN] = fcd.readOBJ_tex(texture_obj) - if s0 is not None: - Vf = Vf * s0 - if t0 is not None: - Vf = Vf - t0 - - P = sp.sparse.kron(sp.sparse.identity(3), fcd.prolongation(Vf, V, T)) - viewer.set_mesh(Vf, Ff, 0) - viewer.set_show_lines(False, 0) - viewer.set_texture(texture_png, TC, FTC, 0) - viewer.set_face_based(False, 0) - viewer.init_guizmo(True, T0, guizmo_callback, "translate") - viewer.set_pre_draw_callback(pre_draw_callback) - self.V = Vf - self.P = P - self.viewer = viewer - - def launch(self): - self.viewer.launch() - - def update_displacement(self, U): - if self.vis_texture: - U = self.P @ U.reshape((-1, 1), order='F') - U = U.reshape((-1, 3), order='F') - X = U + self.V - self.viewer.set_vertices(X, 0) - self.viewer.compute_normals(0) - -