merge: remote-tracking branch 'origin/main' into insert/feat/bodypix

.github/workflows/documentation.yml

@@ -0,0 +1,56 @@
+name: Build & Deploy Documentation
+
+on:
+  push:
+  pull_request:
+    branches: [main]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@v3
+        with:
+          path: "src/lasr-base"
+
+      - name: Install ROS Noetic
+        run: |
+          sudo add-apt-repository universe
+          sudo add-apt-repository restricted
+          sudo add-apt-repository multiverse
+
+          sudo apt update
+
+          sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list'
+
+          sudo apt install curl # if you haven't already installed curl
+          curl -s https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc | sudo apt-key add -
+
+          sudo apt update
+          sudo apt install ros-noetic-ros-base python3-catkin-tools
+
+      - name: Build documentation package
+        run: |
+          source /opt/ros/noetic/setup.bash
+          catkin build documentation
+
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: Build documentation
+        run: |
+          source /opt/ros/noetic/setup.bash
+          source devel/setup.bash
+          rosrun documentation build.py
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        if: github.event_name != 'pull_request' && github.ref_name == 'main'
+        with:
+          deploy_key: ${{ secrets.DOCS_DEPLOY_KEY }}
+          publish_dir: ./src/lasr-base/documentation/web/build
+          external_repository: lasr-at-home/docs

.gitignore

@@ -132,4 +132,9 @@ dmypy.json
 common/Perception/face_detection/output/
 common/Perception/lasr_object_detection_yolo/models
 .vscode/
-common/navigation/choosing_wait_position/src/choosing_wait_position/final_lift_key_point/models
+common/navigation/choosing_wait_position/src/choosing_wait_position/final_lift_key_point/models
+legacy/choosing_wait_position/src/choosing_wait_position/final_lift_key_point/models/keypointsrcnn_weights.pth
+
+# Python extension setup files
+.pylintrc
+mypy.ini

common/speech/lasr_rasa/assistants/receptionist/config.yml

@@ -0,0 +1,51 @@
+# The config recipe.
+# https://rasa.com/docs/rasa/model-configuration/
+recipe: default.v1
+
+# The assistant project unique identifier
+# This default value must be replaced with a unique assistant name within your deployment
+assistant_id: receptionist
+
+# Configuration for Rasa NLU.
+# https://rasa.com/docs/rasa/nlu/components/
+language: en
+
+pipeline:
+# # No configuration for the NLU pipeline was provided. The following default pipeline was used to train your model.
+# # If you'd like to customize it, uncomment and adjust the pipeline.
+# # See https://rasa.com/docs/rasa/tuning-your-model for more information.
+#   - name: WhitespaceTokenizer
+#   - name: RegexFeaturizer
+#   - name: LexicalSyntacticFeaturizer
+#   - name: CountVectorsFeaturizer
+#   - name: CountVectorsFeaturizer
+#     analyzer: char_wb
+#     min_ngram: 1
+#     max_ngram: 4
+#   - name: DIETClassifier
+#     epochs: 100
+#     constrain_similarities: true
+#   - name: EntitySynonymMapper
+#   - name: ResponseSelector
+#     epochs: 100
+#     constrain_similarities: true
+#   - name: FallbackClassifier
+#     threshold: 0.3
+#     ambiguity_threshold: 0.1
+
+
+# Configuration for Rasa Core.
+# https://rasa.com/docs/rasa/core/policies/
+policies: null
+# # No configuration for policies was provided. The following default policies were used to train your model.
+# # If you'd like to customize them, uncomment and adjust the policies.
+# # See https://rasa.com/docs/rasa/policies for more information.
+#   - name: MemoizationPolicy
+#   - name: RulePolicy
+#   - name: UnexpecTEDIntentPolicy
+#     max_history: 5
+#     epochs: 100
+#   - name: TEDPolicy
+#     max_history: 5
+#     epochs: 100
+#     constrain_similarities: true

common/speech/lasr_rasa/assistants/receptionist/credentials.yml

@@ -0,0 +1,33 @@
+# This file contains the credentials for the voice & chat platforms
+# which your bot is using.
+# https://rasa.com/docs/rasa/messaging-and-voice-channels
+
+rest:
+#  # you don't need to provide anything here - this channel doesn't
+#  # require any credentials
+
+
+#facebook:
+#  verify: "<verify>"
+#  secret: "<your secret>"
+#  page-access-token: "<your page access token>"
+
+#slack:
+#  slack_token: "<your slack token>"
+#  slack_channel: "<the slack channel>"
+#  slack_signing_secret: "<your slack signing secret>"
+
+#socketio:
+#  user_message_evt: <event name for user message>
+#  bot_message_evt: <event name for bot messages>
+#  session_persistence: <true/false>
+
+#mattermost:
+#  url: "https://<mattermost instance>/api/v4"
+#  token: "<bot token>"
+#  webhook_url: "<callback URL>"
+
+# This entry is needed if you are using Rasa Enterprise. The entry represents credentials
+# for the Rasa Enterprise "channel", i.e. Talk to your bot and Share with guest testers.
+rasa:
+  url: "http://localhost:5002/api"

common/speech/lasr_rasa/assistants/receptionist/data/nlu.yml

@@ -0,0 +1,36 @@
+version: "3.1"
+
+nlu:
+- intent: fav_drink
+  examples: |
+    - [Orange juice](drink)
+    - [orange juice](drink)
+    - [cola](drink)
+    - [Cola](drink)
+    - [Iced tea](drink)
+    - [iced tea](drink)
+    - [Milk](drink)
+    - [milk](drink)
+    - [Red wine](drink)
+    - [red wine](drink)
+    - [Tropical juice](drink)
+    - [tropical juice](drink)
+    - My favourite drink is [cola](drink)
+    - I like to drink [orange juice](drink)
+
+- intent: name
+  examples: |
+    - [Adel](name)
+    - [Angel](name)
+    - [Axel](name)
+    - [Charlie](name)
+    - [Jane](name)
+    - [Jules](name)
+    - [Morgan](name)
+    - [Paris](name)
+    - [Robin](name)
+    - [Simone](name)
+    - My name is [Adel](name)
+    - It's [Jane](name)
+    - I am [Paris](name)
+    - I'm [Robin](name)

common/speech/lasr_rasa/assistants/receptionist/domain.yml

@@ -0,0 +1,11 @@
+version: "3.1"
+
+intents:
+  - fav_drink
+  - name
+entities:
+  - drink
+  - name
+session_config:
+  session_expiration_time: 60
+  carry_over_slots_to_new_session: true

common/speech/lasr_rasa/assistants/receptionist/endpoints.yml

@@ -0,0 +1,42 @@
+# This file contains the different endpoints your bot can use.
+
+# Server where the models are pulled from.
+# https://rasa.com/docs/rasa/model-storage#fetching-models-from-a-server
+
+#models:
+#  url: http://my-server.com/models/default_core@latest
+#  wait_time_between_pulls:  10   # [optional](default: 100)
+
+# Server which runs your custom actions.
+# https://rasa.com/docs/rasa/custom-actions
+
+#action_endpoint:
+#  url: "http://localhost:5055/webhook"
+
+# Tracker store which is used to store the conversations.
+# By default the conversations are stored in memory.
+# https://rasa.com/docs/rasa/tracker-stores
+
+#tracker_store:
+#    type: redis
+#    url: <host of the redis instance, e.g. localhost>
+#    port: <port of your redis instance, usually 6379>
+#    db: <number of your database within redis, e.g. 0>
+#    password: <password used for authentication>
+#    use_ssl: <whether or not the communication is encrypted, default false>
+
+#tracker_store:
+#    type: mongod
+#    url: <url to your mongo instance, e.g. mongodb://localhost:27017>
+#    db: <name of the db within your mongo instance, e.g. rasa>
+#    username: <username used for authentication>
+#    password: <password used for authentication>
+
+# Event broker which all conversation events should be streamed to.
+# https://rasa.com/docs/rasa/event-brokers
+
+#event_broker:
+#  url: localhost
+#  username: username
+#  password: password
+#  queue: queue

common/speech/lasr_rasa/requirements.in

@@ -1,3 +1,3 @@
 requests==2.31.0
 rasa==3.6.5
-spacy==3.6.1
+spacy==3.6.1

common/speech/lasr_speech/launch/speech.launch

@@ -2,7 +2,7 @@
     <arg name="matcher" default="by-index" />
     <arg name="device_param" default="" />
     <arg name="rasa_model" default=""/>
-    <node pkg="lasr_speech_recognition_whisper" type="transcribe_microphone" name="whisper_service" output="screen" args="$(arg matcher) $(arg device_param)"/>
+    <node pkg="lasr_speech_recognition_whisper" type="simple_transcribe_microphone" name="whisper_service" output="screen" args="$(arg matcher) $(arg device_param)"/>
     <include file = "$(find lasr_rasa)/launch/rasa.launch">
         <arg name="model" value="$(arg rasa_model)"/>
     </include>

common/speech/lasr_speech/nodes/service

@@ -1,9 +1,8 @@
 #!/usr/bin/env python3
 import rospy
-from std_srvs.srv import Empty
 from lasr_speech.srv import Speech, SpeechResponse
 from lasr_rasa.srv import Rasa
-from lasr_speech_recognition_msgs.msg import Transcription
+from lasr_speech_recognition_msgs.srv import TranscribeAudio
 import sounddevice
 import soundfile
 import pyaudio
@@ -17,36 +16,29 @@ class TranscribeAndParse:
     def __init__(self):
         rospy.wait_for_service("/lasr_rasa/parse")
         self.rasa = rospy.ServiceProxy("/lasr_rasa/parse", Rasa)
-        self.adjust_for_noise = rospy.ServiceProxy("/whisper/adjust_for_noise", Empty)
-        self.start_listening = rospy.ServiceProxy("/whisper/start_listening", Empty)
-        self.stop_listening = rospy.ServiceProxy("/whisper/stop_listening", Empty)
-        self.transcription_topic = "/transcription"
+        self.transcribe_audio = rospy.ServiceProxy("/whisper/transcribe_audio", TranscribeAudio) 
         self.sound_data = soundfile.read(os.path.join(rospkg.RosPack().get_path("lasr_speech"), "sounds", "beep.wav"))[0]
 
     def play_sound(self):
         def play():
             audio_interface = pyaudio.PyAudio()
             sample_rate = int(audio_interface.get_default_output_device_info()["defaultSampleRate"])
             sounddevice.play(self.sound_data, sample_rate,
-                             device=audio_interface.get_default_output_device_info()["index"])  # device=None is default for TIAGo. Modify to match your device.
+                             device=audio_interface.get_default_output_device_info()["index"])
             sounddevice.wait()
+            audio_interface.terminate()
         p = Process(target=play)
         p.start()
         sleep(0.5)
 
     def __call__(self, req):
-        self.adjust_for_noise()
         if req.play_sound:
             self.play_sound()
-        self.start_listening()
-        while True:
-            transcription = rospy.wait_for_message(self.transcription_topic, Transcription)
-            rospy.loginfo(transcription)
-            if transcription.finished:
-                self.stop_listening()
-                rasa_response = self.rasa(transcription.phrase)
-                rospy.loginfo(rasa_response)
-                return SpeechResponse(rasa_response.json_response, rasa_response.success)
+        transcription = self.transcribe_audio()
+        rospy.loginfo(transcription)
+        rasa_response = self.rasa(transcription.phrase)
+        rospy.loginfo(rasa_response)
+        return SpeechResponse(rasa_response.json_response, rasa_response.success)
 
 if __name__ == "__main__":
     rospy.init_node("lasr_speech")

common/speech/lasr_speech_recognition_msgs/CMakeLists.txt

@@ -49,11 +49,10 @@ add_message_files(
 )
 
 ## Generate services in the 'srv' folder
-# add_service_files(
-#   FILES
-#   Service1.srv
-#   Service2.srv
-# )
+add_service_files(
+  FILES
+  TranscribeAudio.srv
+)
 
 ## Generate actions in the 'action' folder
 # add_action_files(

common/speech/lasr_speech_recognition_msgs/srv/TranscribeAudio.srv

@@ -0,0 +1,2 @@
+---
+string phrase

common/speech/lasr_speech_recognition_whisper/CMakeLists.txt

@@ -160,8 +160,10 @@ include_directories(
 ## Mark executable scripts (Python etc.) for installation
 ## in contrast to setup.py, you can choose the destination
 catkin_install_python(PROGRAMS
+  nodes/simple_transcribe_microphone
   nodes/transcribe_microphone
   scripts/list_microphones.py
+  scripts/test_microphones.py
   DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
 )