diff --git a/common/vision/lasr_vision_clip/nodes/vqa b/common/vision/lasr_vision_clip/nodes/vqa index ff0b285f9..086dcf1d8 100644 --- a/common/vision/lasr_vision_clip/nodes/vqa +++ b/common/vision/lasr_vision_clip/nodes/vqa @@ -31,7 +31,7 @@ class VqaService: """ possible_answers = request.possible_answers answer, cos_score, annotated_img = query_image( - self._model, request.image_raw, possible_answers, annotate=True + request.image_raw, self._model, possible_answers, annotate=True ) self._debug_pub.publish(annotated_img) diff --git a/common/vision/lasr_vision_feature_extraction/nodes/service b/common/vision/lasr_vision_feature_extraction/nodes/service index dcc50a0a5..ede18a38d 100644 --- a/common/vision/lasr_vision_feature_extraction/nodes/service +++ b/common/vision/lasr_vision_feature_extraction/nodes/service @@ -48,9 +48,7 @@ def detect( head_frame, torso_frame, full_frame, - head_mask, - torso_mask, - request.image_raw, + image_raw=request.image_raw, cloth_predictor=cloth_predictor, ) response = TorchFaceFeatureDetectionDescriptionResponse() diff --git a/common/vision/lasr_vision_feature_extraction/src/lasr_vision_feature_extraction/__init__.py b/common/vision/lasr_vision_feature_extraction/src/lasr_vision_feature_extraction/__init__.py index 53640fb9b..1fc27b098 100644 --- a/common/vision/lasr_vision_feature_extraction/src/lasr_vision_feature_extraction/__init__.py +++ b/common/vision/lasr_vision_feature_extraction/src/lasr_vision_feature_extraction/__init__.py @@ -545,8 +545,6 @@ def predict_frame( head_frame, torso_frame, full_frame, - head_mask, - torso_mask, cloth_predictor, image_raw, clip_service: rospy.ServiceProxy = rospy.ServiceProxy( @@ -571,6 +569,7 @@ def predict_frame( rst_person = { "glasses": -0.5, "hat": -0.5, + "hair_shape": "short hair" } glasses_query = VqaRequest( @@ -583,14 +582,24 @@ def predict_frame( image_raw=image_raw, ) + hair_query = VqaRequest( + possible_answers=["A person with short hair", "A person with long hair"], + image_raw=image_raw, + ) + glasses_response = clip_service(glasses_query) - hat_response = clip_service(hat_query) if glasses_response.answer == "A person wearing glasses": rst_person["glasses"] = 0.5 + hat_response = clip_service(hat_query) if hat_response.answer == "A person wearing a hat": rst_person["hat"] = 0.5 + hair_response = clip_service(hair_query) + + if hair_response.answer == "A person with long hair": + rst_person["hair_shape"] = "long hair" + result = { **rst_person, **rst_cloth, diff --git a/tasks/receptionist/launch/setup.launch b/tasks/receptionist/launch/setup.launch index 73481cc4d..bd9dac8db 100644 --- a/tasks/receptionist/launch/setup.launch +++ b/tasks/receptionist/launch/setup.launch @@ -19,8 +19,7 @@ - - + diff --git a/tasks/receptionist/src/receptionist/states/introduce.py b/tasks/receptionist/src/receptionist/states/introduce.py index c9030c2df..91384fcbf 100644 --- a/tasks/receptionist/src/receptionist/states/introduce.py +++ b/tasks/receptionist/src/receptionist/states/introduce.py @@ -48,17 +48,20 @@ def stringify_guest_data( }, ) + + relevant_guest_data["attributes"]["has_hair"] = 0.5 + guest_str = f"{relevant_guest_data['name']}, their favourite drink is {relevant_guest_data['drink']}. " if not relevant_guest_data["detection"] or not describe_features: return guest_str filtered_attributes = {} - # filtered_attributes["hair"] = { - # "confidence": relevant_guest_data["attributes"]["has_hair"], - # "hair_shape": relevant_guest_data["attributes"]["hair_shape"], - # "hair_colour": relevant_guest_data["attributes"]["hair_colour"], - # } + filtered_attributes["hair"] = { + "confidence": relevant_guest_data["attributes"]["has_hair"], + "hair_shape": relevant_guest_data["attributes"]["hair_shape"], + # "hair_colour": relevant_guest_data["attributes"]["hair_colour"], + } most_confident_clothes = find_most_confident_clothes( relevant_guest_data, @@ -117,8 +120,8 @@ def stringify_guest_data( if attribute_name == "hair": hair_shape = attribute_value["hair_shape"] - hair_colour = attribute_value["hair_colour"] - guest_str += f"They have {hair_shape} and {hair_colour}. " + # hair_colour = attribute_value["hair_colour"] + guest_str += f"They have {hair_shape}. " elif attribute_name == "facial_hair": if confidence < 0: guest_str += "They don't have facial hair. "