diff --git a/README.md b/README.md index ad60aaa..edfccd5 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,14 @@ Use a swarm of AI agents to generate a marketing plan for your business. ai marketing-plan ``` +### 8. Posture Coach (`ai posture`) + +Use the webcam and a tiny vision model to analyze your posture and focus. + +```bash +ai posture +``` + ## Tool Use Tools (`tooluse` command) ### 1. Podcast RSS Reader (`tooluse`) diff --git a/poetry.lock b/poetry.lock index 0183606..3117622 100644 --- a/poetry.lock +++ b/poetry.lock @@ -563,13 +563,13 @@ files = [ [[package]] name = "httpcore" -version = "1.0.6" +version = "1.0.7" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"}, - {file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"}, + {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, + {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, ] [package.dependencies] @@ -699,6 +699,88 @@ files = [ [package.dependencies] ansicon = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "jiter" +version = "0.7.1" +description = "Fast iterable JSON parser." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jiter-0.7.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:262e96d06696b673fad6f257e6a0abb6e873dc22818ca0e0600f4a1189eb334f"}, + {file = "jiter-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be6de02939aac5be97eb437f45cfd279b1dc9de358b13ea6e040e63a3221c40d"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935f10b802bc1ce2b2f61843e498c7720aa7f4e4bb7797aa8121eab017293c3d"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9cd3cccccabf5064e4bb3099c87bf67db94f805c1e62d1aefd2b7476e90e0ee2"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4aa919ebfc5f7b027cc368fe3964c0015e1963b92e1db382419dadb098a05192"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ae2d01e82c94491ce4d6f461a837f63b6c4e6dd5bb082553a70c509034ff3d4"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f9568cd66dbbdab67ae1b4c99f3f7da1228c5682d65913e3f5f95586b3cb9a9"}, + {file = "jiter-0.7.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ecbf4e20ec2c26512736284dc1a3f8ed79b6ca7188e3b99032757ad48db97dc"}, + {file = "jiter-0.7.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b1a0508fddc70ce00b872e463b387d49308ef02b0787992ca471c8d4ba1c0fa1"}, + {file = "jiter-0.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f84c9996664c460f24213ff1e5881530abd8fafd82058d39af3682d5fd2d6316"}, + {file = "jiter-0.7.1-cp310-none-win32.whl", hash = "sha256:c915e1a1960976ba4dfe06551ea87063b2d5b4d30759012210099e712a414d9f"}, + {file = "jiter-0.7.1-cp310-none-win_amd64.whl", hash = "sha256:75bf3b7fdc5c0faa6ffffcf8028a1f974d126bac86d96490d1b51b3210aa0f3f"}, + {file = "jiter-0.7.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ad04a23a91f3d10d69d6c87a5f4471b61c2c5cd6e112e85136594a02043f462c"}, + {file = "jiter-0.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e47a554de88dff701226bb5722b7f1b6bccd0b98f1748459b7e56acac2707a5"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e44fff69c814a2e96a20b4ecee3e2365e9b15cf5fe4e00869d18396daa91dab"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:df0a1d05081541b45743c965436f8b5a1048d6fd726e4a030113a2699a6046ea"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f22cf8f236a645cb6d8ffe2a64edb5d2b66fb148bf7c75eea0cb36d17014a7bc"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da8589f50b728ea4bf22e0632eefa125c8aa9c38ed202a5ee6ca371f05eeb3ff"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f20de711224f2ca2dbb166a8d512f6ff48c9c38cc06b51f796520eb4722cc2ce"}, + {file = "jiter-0.7.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8a9803396032117b85ec8cbf008a54590644a062fedd0425cbdb95e4b2b60479"}, + {file = "jiter-0.7.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3d8bae77c82741032e9d89a4026479061aba6e646de3bf5f2fc1ae2bbd9d06e0"}, + {file = "jiter-0.7.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3dc9939e576bbc68c813fc82f6620353ed68c194c7bcf3d58dc822591ec12490"}, + {file = "jiter-0.7.1-cp311-none-win32.whl", hash = "sha256:f7605d24cd6fab156ec89e7924578e21604feee9c4f1e9da34d8b67f63e54892"}, + {file = "jiter-0.7.1-cp311-none-win_amd64.whl", hash = "sha256:f3ea649e7751a1a29ea5ecc03c4ada0a833846c59c6da75d747899f9b48b7282"}, + {file = "jiter-0.7.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ad36a1155cbd92e7a084a568f7dc6023497df781adf2390c345dd77a120905ca"}, + {file = "jiter-0.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7ba52e6aaed2dc5c81a3d9b5e4ab95b039c4592c66ac973879ba57c3506492bb"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b7de0b6f6728b678540c7927587e23f715284596724be203af952418acb8a2d"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9463b62bd53c2fb85529c700c6a3beb2ee54fde8bef714b150601616dcb184a6"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:627164ec01d28af56e1f549da84caf0fe06da3880ebc7b7ee1ca15df106ae172"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25d0e5bf64e368b0aa9e0a559c3ab2f9b67e35fe7269e8a0d81f48bbd10e8963"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c244261306f08f8008b3087059601997016549cb8bb23cf4317a4827f07b7d74"}, + {file = "jiter-0.7.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7ded4e4b75b68b843b7cea5cd7c55f738c20e1394c68c2cb10adb655526c5f1b"}, + {file = "jiter-0.7.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:80dae4f1889b9d09e5f4de6b58c490d9c8ce7730e35e0b8643ab62b1538f095c"}, + {file = "jiter-0.7.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5970cf8ec943b51bce7f4b98d2e1ed3ada170c2a789e2db3cb484486591a176a"}, + {file = "jiter-0.7.1-cp312-none-win32.whl", hash = "sha256:701d90220d6ecb3125d46853c8ca8a5bc158de8c49af60fd706475a49fee157e"}, + {file = "jiter-0.7.1-cp312-none-win_amd64.whl", hash = "sha256:7824c3ecf9ecf3321c37f4e4d4411aad49c666ee5bc2a937071bdd80917e4533"}, + {file = "jiter-0.7.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:097676a37778ba3c80cb53f34abd6943ceb0848263c21bf423ae98b090f6c6ba"}, + {file = "jiter-0.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3298af506d4271257c0a8f48668b0f47048d69351675dd8500f22420d4eec378"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12fd88cfe6067e2199964839c19bd2b422ca3fd792949b8f44bb8a4e7d21946a"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dacca921efcd21939123c8ea8883a54b9fa7f6545c8019ffcf4f762985b6d0c8"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de3674a5fe1f6713a746d25ad9c32cd32fadc824e64b9d6159b3b34fd9134143"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65df9dbae6d67e0788a05b4bad5706ad40f6f911e0137eb416b9eead6ba6f044"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ba9a358d59a0a55cccaa4957e6ae10b1a25ffdabda863c0343c51817610501d"}, + {file = "jiter-0.7.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:576eb0f0c6207e9ede2b11ec01d9c2182973986514f9c60bc3b3b5d5798c8f50"}, + {file = "jiter-0.7.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:e550e29cdf3577d2c970a18f3959e6b8646fd60ef1b0507e5947dc73703b5627"}, + {file = "jiter-0.7.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:81d968dbf3ce0db2e0e4dec6b0a0d5d94f846ee84caf779b07cab49f5325ae43"}, + {file = "jiter-0.7.1-cp313-none-win32.whl", hash = "sha256:f892e547e6e79a1506eb571a676cf2f480a4533675f834e9ae98de84f9b941ac"}, + {file = "jiter-0.7.1-cp313-none-win_amd64.whl", hash = "sha256:0302f0940b1455b2a7fb0409b8d5b31183db70d2b07fd177906d83bf941385d1"}, + {file = "jiter-0.7.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:c65a3ce72b679958b79d556473f192a4dfc5895e8cc1030c9f4e434690906076"}, + {file = "jiter-0.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e80052d3db39f9bb8eb86d207a1be3d9ecee5e05fdec31380817f9609ad38e60"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70a497859c4f3f7acd71c8bd89a6f9cf753ebacacf5e3e799138b8e1843084e3"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c1288bc22b9e36854a0536ba83666c3b1fb066b811019d7b682c9cf0269cdf9f"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b096ca72dd38ef35675e1d3b01785874315182243ef7aea9752cb62266ad516f"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8dbbd52c50b605af13dbee1a08373c520e6fcc6b5d32f17738875847fea4e2cd"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af29c5c6eb2517e71ffa15c7ae9509fa5e833ec2a99319ac88cc271eca865519"}, + {file = "jiter-0.7.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f114a4df1e40c03c0efbf974b376ed57756a1141eb27d04baee0680c5af3d424"}, + {file = "jiter-0.7.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:191fbaee7cf46a9dd9b817547bf556facde50f83199d07fc48ebeff4082f9df4"}, + {file = "jiter-0.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e2b445e5ee627fb4ee6bbceeb486251e60a0c881a8e12398dfdff47c56f0723"}, + {file = "jiter-0.7.1-cp38-none-win32.whl", hash = "sha256:47ac4c3cf8135c83e64755b7276339b26cd3c7ddadf9e67306ace4832b283edf"}, + {file = "jiter-0.7.1-cp38-none-win_amd64.whl", hash = "sha256:60b49c245cd90cde4794f5c30f123ee06ccf42fb8730a019a2870cd005653ebd"}, + {file = "jiter-0.7.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8f212eeacc7203256f526f550d105d8efa24605828382cd7d296b703181ff11d"}, + {file = "jiter-0.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d9e247079d88c00e75e297e6cb3a18a039ebcd79fefc43be9ba4eb7fb43eb726"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0aacaa56360139c53dcf352992b0331f4057a0373bbffd43f64ba0c32d2d155"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bc1b55314ca97dbb6c48d9144323896e9c1a25d41c65bcb9550b3e0c270ca560"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f281aae41b47e90deb70e7386558e877a8e62e1693e0086f37d015fa1c102289"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:93c20d2730a84d43f7c0b6fb2579dc54335db742a59cf9776d0b80e99d587382"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e81ccccd8069110e150613496deafa10da2f6ff322a707cbec2b0d52a87b9671"}, + {file = "jiter-0.7.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0a7d5e85766eff4c9be481d77e2226b4c259999cb6862ccac5ef6621d3c8dcce"}, + {file = "jiter-0.7.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f52ce5799df5b6975439ecb16b1e879d7655e1685b6e3758c9b1b97696313bfb"}, + {file = "jiter-0.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e0c91a0304373fdf97d56f88356a010bba442e6d995eb7773cbe32885b71cdd8"}, + {file = "jiter-0.7.1-cp39-none-win32.whl", hash = "sha256:5c08adf93e41ce2755970e8aa95262298afe2bf58897fb9653c47cd93c3c6cdc"}, + {file = "jiter-0.7.1-cp39-none-win_amd64.whl", hash = "sha256:6592f4067c74176e5f369228fb2995ed01400c9e8e1225fb73417183a5e635f0"}, + {file = "jiter-0.7.1.tar.gz", hash = "sha256:448cf4f74f7363c34cdef26214da527e8eeffd88ba06d0b80b485ad0667baf5d"}, +] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -764,6 +846,30 @@ files = [ [package.dependencies] httpx = ">=0.27.0,<0.28.0" +[[package]] +name = "openai" +version = "1.54.4" +description = "The official Python library for the openai API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "openai-1.54.4-py3-none-any.whl", hash = "sha256:0d95cef99346bf9b6d7fbf57faf61a673924c3e34fa8af84c9ffe04660673a7e"}, + {file = "openai-1.54.4.tar.gz", hash = "sha256:50f3656e45401c54e973fa05dc29f3f0b0d19348d685b2f7ddb4d92bf7b1b6bf"}, +] + +[package.dependencies] +anyio = ">=3.5.0,<5" +distro = ">=1.7.0,<2" +httpx = ">=0.23.0,<1" +jiter = ">=0.4.0,<1" +pydantic = ">=1.9.0,<3" +sniffio = "*" +tqdm = ">4" +typing-extensions = ">=4.11,<5" + +[package.extras] +datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] + [[package]] name = "packaging" version = "24.2" @@ -1656,4 +1762,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "81910a386f81af2f51761d35d03774ff0f2cd4d1374cc5a5a0cd3e9e3fd48b42" +content-hash = "ca37820d18082bb80476989cde3ed8f0d29fbc63e65b7af0b159b25c92cfe730" diff --git a/pyproject.toml b/pyproject.toml index db1b6ec..9d9b578 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tool-use-ai" -version = "1.1.6" +version = "1.1.10" description = "Tools to simplify life with AI" authors = ["Ty Fiero ", "Mike Bird "] readme = "README.md" diff --git a/src/tool_use/cli.py b/src/tool_use/cli.py index 4648ecf..aea8946 100644 --- a/src/tool_use/cli.py +++ b/src/tool_use/cli.py @@ -1,9 +1,8 @@ import sys import argparse import subprocess -import pkg_resources +from importlib.metadata import version, PackageNotFoundError from .scripts._script_dependencies import SCRIPT_DEPENDENCIES -from .scripts import ai_cli, cal, obsidian_plugin, convert, transcribe, prioritize, activity_tracker, marketing_agency from .utils.config_wizard import setup_wizard, SCRIPT_INFO @@ -13,8 +12,8 @@ def ensure_dependencies(script_name): for package in SCRIPT_DEPENDENCIES[script_name]: try: - pkg_resources.require(package) - except (pkg_resources.DistributionNotFound, pkg_resources.VersionConflict): + version(package) + except PackageNotFoundError: print(f"Installing required dependency: {package}") subprocess.check_call( [sys.executable, "-m", "pip", "install", "--upgrade", package] @@ -44,6 +43,7 @@ def main(): "prioritize": "Brain dump and prioritize tasks", "log": "Track your activities", "marketing-plan": "Use a marketing agency of AI agents to create a marketing plan", + "posture": "Use the webcam and a tiny vision model to analyze your posture and focus", } for name, help_text in all_scripts.items(): @@ -68,22 +68,25 @@ def main(): # Try to import dependencies, install if missing ensure_dependencies(script_name) - - # Map script names to their modules + + # Map script names to their module paths script_modules = { - "do": ai_cli, - "make-obsidian-plugin": obsidian_plugin, - "cal": cal, - "convert": convert, - "transcribe": transcribe, - "prioritize": prioritize, - "log": activity_tracker, - "marketing-plan": marketing_agency, + "do": "ai_cli", + "make-obsidian-plugin": "obsidian_plugin", + "cal": "cal", + "convert": "convert", + "transcribe": "transcribe", + "prioritize": "prioritize", + "log": "activity_tracker", + "marketing-plan": "marketing_agency", + "posture": "posture", } - # Run the appropriate script + # Dynamic import of only the needed module if script_name in script_modules: - script_modules[script_name].main(script_args) + module_name = script_modules[script_name] + module = __import__(f"tool_use.scripts.{module_name}", fromlist=["main"]) + module.main(script_args) else: print(f"Unknown script: {script_name}") sys.exit(1) diff --git a/src/tool_use/scripts/_script_dependencies.py b/src/tool_use/scripts/_script_dependencies.py index ebfeb98..de83627 100644 --- a/src/tool_use/scripts/_script_dependencies.py +++ b/src/tool_use/scripts/_script_dependencies.py @@ -31,4 +31,17 @@ ], "log": [], "marketing-plan": ["rich", "openai", "swarm"], + "posture": [ + "moondream", + "rich", + "opencv-python", + "pillow", + "pydantic", + "llama-index", + "numpy>=1.22.4,<1.29.0", + "scipy>=1.10.0", + "transformers", + "llama-index", + "llama-index-llms-ollama", + ], } diff --git a/src/tool_use/scripts/posture.py b/src/tool_use/scripts/posture.py new file mode 100644 index 0000000..7a79b47 --- /dev/null +++ b/src/tool_use/scripts/posture.py @@ -0,0 +1,442 @@ +import os +import time +import sys +import json +import cv2 +import numpy as np +import subprocess +from rich.console import Console +from rich.panel import Panel +from rich.layout import Layout +from rich.live import Live +from rich.table import Table +from rich.text import Text +from datetime import datetime +from collections import deque +import statistics +from PIL import Image +from pydantic import BaseModel +from enum import Enum +from typing import List +from llama_index.core.bridge.pydantic import BaseModel +from llama_index.core.tools import FunctionTool +from llama_index.llms.ollama import Ollama +from llama_index.core.llms import ChatMessage +from transformers import AutoModelForCausalLM, AutoTokenizer +from ..config_manager import config_manager + +console = Console() + + + +class Moondream: + MODEL_ID = "vikhyatk/moondream2" + REVISION = "2024-08-26" + + def __init__(self): + print("Loading Moondream2 model...") + self.model = AutoModelForCausalLM.from_pretrained( + self.MODEL_ID, + trust_remote_code=True, + revision=self.REVISION + ) + self.tokenizer = AutoTokenizer.from_pretrained( + self.MODEL_ID, + revision=self.REVISION + ) + print("Moondream2 model loaded successfully!") + + def analyze_posture(self, image): + start_total = time.time() + + # Time image encoding + start_encode = time.time() + encoded_image = self.model.encode_image(image) + encode_time = time.time() - start_encode + print(f"Image encoding took {encode_time:.2f} seconds") + + # Check for focus + start_focus = time.time() + focus_answer = self.model.answer_question( + encoded_image, + "Look at this webcam image. Please provide several sentences that describe if there is a person there, and if so, if they are focused/distracted, or if they are slouching.", + self.tokenizer + ) + focus_time = time.time() - start_focus + print(f"Focus analysis took {focus_time:.2f} seconds") + + # # Check for slouching + # start_posture = time.time() + # posture_answer = self.model.answer_question( + # encoded_image, + # "Is the person slouching or showing poor posture? Answer with just Yes or No.", + # self.tokenizer + # ) + + total_time = time.time() - start_total + print(f"\nTotal analysis time: {total_time:.2f} seconds") + print(f"Results: Focus: {focus_answer}\n") + + return focus_answer.lower().strip() + +class AnalysisResults(BaseModel): + """Results of analyzing the description of a webcam image""" + present: bool + focused: bool + slouching: bool + distracted: bool + +llm = Ollama(model="llama3.1:latest", request_timeout=120.0) +sllm = llm.as_structured_llm(AnalysisResults) + +def extractDataFromMoondream(text: str) -> str: + prompt = f""" + Determine the following information from the provided text: + 1. Human is present + 2. Human is focused on work + 3. Human is not slouching + 4. Human is not distracted + + {text} + """ + try: + response = sllm.chat([ChatMessage(role="user", content=prompt)]) + + # Create a dictionary with the analysis results + analysis = { + "present": False, + "focused": False, + "slouching": False, + "distracted": False + } + + # Parse the response content + if hasattr(response.message, 'content'): + content = response.message.content + if isinstance(content, str): + try: + # Try to parse it as JSON first + analysis = json.loads(content) + except json.JSONDecodeError: + # If it's not JSON, try to parse the text response + analysis["present"] = "present: true" in content.lower() + analysis["focused"] = "focused: true" in content.lower() + analysis["slouching"] = "slouching: true" in content.lower() + analysis["distracted"] = "distracted: true" in content.lower() + elif isinstance(content, dict): + analysis = content + + # Convert the analysis to a JSON string + return json.dumps(analysis) + + except Exception as e: + print(f"Error in extractDataFromMoondream: {e}") + # Return a default response if there's an error + return json.dumps({ + "present": False, + "focused": False, + "slouching": False, + "distracted": False + }) + + +class WebcamHandler: + def __init__( + self, + capture_interval=10, + save_images=False, + save_directory="captured_images", + resize_factor=0.5, + jpeg_quality=85 + ): + self.capture_interval = capture_interval + self.save_images = save_images + self.save_directory = save_directory + self.resize_factor = resize_factor + self.jpeg_quality = jpeg_quality + + if save_images: + os.makedirs(save_directory, exist_ok=True) + + self.camera = None + self.initialize_camera() + + def initialize_camera(self): + """Initialize the webcam with proper error handling""" + # Try to open the camera + self.camera = cv2.VideoCapture(0) + + if not self.camera.isOpened(): + if sys.platform == "darwin": # macOS + print("Camera access denied. Attempting to fix...") + print("Please grant camera permissions in System Preferences -> Security & Privacy -> Camera") + print("You might need to run: 'tccutil reset Camera' in Terminal with admin privileges") + + try: + subprocess.run(['tccutil', 'reset', 'Camera'], check=True) + print("Camera permissions reset. Please run the program again.") + except subprocess.CalledProcessError: + print("Failed to reset camera permissions automatically.") + except FileNotFoundError: + print("Could not find tccutil. Please reset permissions manually.") + + else: # Other platforms + print("Could not open webcam. Please check if:") + print("1. Your webcam is connected") + print("2. It's not being used by another application") + print("3. You have granted necessary permissions") + + raise RuntimeError("Could not open webcam. Please check permissions and try again.") + + # Test capture to ensure camera is working + ret, _ = self.camera.read() + if not ret: + self.camera.release() + raise RuntimeError("Camera opened but failed to capture. Please check if it's being used by another application.") + + def capture_frame(self): + """Capture a frame from webcam and return as PIL Image""" + if self.camera is None or not self.camera.isOpened(): + self.initialize_camera() + + ret, frame = self.camera.read() + if not ret: + print("Failed to capture frame. Attempting to reinitialize camera...") + if self.camera is not None: + self.camera.release() + self.initialize_camera() + ret, frame = self.camera.read() + if not ret: + raise RuntimeError("Failed to capture frame even after reinitializing camera") + + # Convert BGR to RGB + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + # Convert to PIL Image + image = Image.fromarray(rgb_frame) + + # Resize based on resize_factor + width, height = image.size + new_width = int(width * self.resize_factor) + new_height = int(height * self.resize_factor) + image = image.resize((new_width, new_height), Image.Resampling.LANCZOS) + + if self.save_images: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + save_path = os.path.join(self.save_directory, f"frame_{timestamp}.jpg") + image.save(save_path, "JPEG", quality=self.jpeg_quality) + return image, save_path + + return image, None + + def process_saved_image(self, image_path: str): + """Process a saved image (e.g., move it to an archive folder or delete it)""" + if os.path.exists(image_path): + # For now, we'll just delete the temporary image + # You could modify this to move it to an archive folder instead + os.remove(image_path) + + def __del__(self): + if self.camera is not None: + self.camera.release() + + +class ProductivityCoach: + def __init__( + self, + capture_interval=10, + save_images=False, + resize_factor=0.5, + jpeg_quality=85, + ): + # Initialize with a fancy loading display + with console.status("[bold green]Initializing Productivity Coach...", spinner="dots"): + self.moondream = Moondream() + self.webcam = WebcamHandler( + capture_interval=capture_interval, + save_images=save_images, + resize_factor=resize_factor, + jpeg_quality=jpeg_quality + ) + self.capture_interval = capture_interval + + # Stats tracking + self.stats = { + "total_frames": 0, + "focused_frames": 0, + "slouching_frames": 0, + "distracted_frames": 0, + "start_time": datetime.now(), + "focus_history": deque(maxlen=50), # Keep last 50 readings + } + + def generate_layout(self) -> Layout: + layout = Layout() + + layout.split( + Layout(name="header", size=3), + Layout(name="main"), + Layout(name="footer", size=3) + ) + + layout["main"].split_row( + Layout(name="stats"), + Layout(name="current_status"), + ) + + return layout + + def make_stats_panel(self) -> Panel: + duration = datetime.now() - self.stats["start_time"] + hours = duration.seconds // 3600 + minutes = (duration.seconds % 3600) // 60 + + # Calculate percentages + total = max(1, self.stats["total_frames"]) + focus_rate = (self.stats["focused_frames"] / total) * 100 + slouch_rate = (self.stats["slouching_frames"] / total) * 100 + distraction_rate = (self.stats["distracted_frames"] / total) * 100 + + # Calculate recent focus trend + recent_focus = list(self.stats["focus_history"]) + trend = "β†’" + if len(recent_focus) > 5: + recent_avg = statistics.mean(recent_focus[-5:]) + older_avg = statistics.mean(recent_focus[-10:-5]) + if recent_avg > older_avg + 0.1: + trend = "↑" + elif recent_avg < older_avg - 0.1: + trend = "↓" + + stats_table = Table(show_header=False, box=None) + stats_table.add_row("Session Duration:", f"{hours:02d}:{minutes:02d}") + stats_table.add_row("Total Frames:", str(self.stats["total_frames"])) + stats_table.add_row("Focus Rate:", f"{focus_rate:.1f}% {trend}") + stats_table.add_row("Slouching Rate:", f"{slouch_rate:.1f}%") + stats_table.add_row("Distraction Rate:", f"{distraction_rate:.1f}%") + + return Panel(stats_table, title="[bold]Session Statistics", border_style="blue") + + def make_status_panel(self, status_data) -> Panel: + status_text = Text() + + # Focus Status + if status_data["focused"]: + status_text.append("🎯 Focused\n", style="bold green") + else: + status_text.append("πŸ˜Άβ€πŸŒ«οΈ Distracted\n", style="bold red") + + # Posture Status + if status_data["slouching"]: + status_text.append("πŸͺ‘ Poor Posture\n", style="bold red") + else: + status_text.append("πŸ‘ Good Posture\n", style="bold green") + + # Presence Status + if status_data["present"]: + status_text.append("πŸ‘€ Present\n", style="bold green") + else: + status_text.append("❓ Not Detected\n", style="bold yellow") + + return Panel(status_text, title="[bold]Current Status", border_style="green") + + def update_stats(self, status_str: str): + # Parse the JSON string into a dictionary + try: + status = json.loads(status_str) + except json.JSONDecodeError: + console.print("[bold red]Error parsing status data[/]") + return + + self.stats["total_frames"] += 1 + if status.get("focused", False): + self.stats["focused_frames"] += 1 + if status.get("slouching", False): + self.stats["slouching_frames"] += 1 + if status.get("distracted", False): + self.stats["distracted_frames"] += 1 + + # Update focus history (1 for focused, 0 for not) + self.stats["focus_history"].append(1 if status.get("focused", False) else 0) + + def run(self): + layout = self.generate_layout() + + # Header and footer + layout["header"].update(Panel( + "[bold blue]Productivity Coach[/] - Press Ctrl+C to exit", + style="bold white on blue" + )) + layout["footer"].update(Panel( + f"Capturing every {self.capture_interval}s β€’ Image Size: {self.webcam.resize_factor*100:.0f}% β€’ JPEG Quality: {self.webcam.jpeg_quality}", + style="bold white on blue" + )) + + # Initialize the panels with default content + layout["stats"].update(self.make_stats_panel()) + layout["current_status"].update(self.make_status_panel({ + "present": False, + "focused": False, + "slouching": False, + "distracted": False + })) + + with Live(layout, refresh_per_second=1, screen=True): + try: + while True: + # Capture and analyze frame + image, temp_path = self.webcam.capture_frame() + results = self.moondream.analyze_posture(image) + status_str = extractDataFromMoondream(results) + + try: + status_data = json.loads(status_str) + # Update stats + self.update_stats(status_str) + + # Update panels + layout["stats"].update(self.make_stats_panel()) + layout["current_status"].update(self.make_status_panel(status_data)) + except json.JSONDecodeError: + console.print("[bold red]Error parsing status data[/]") + + # Process saved image + if temp_path: + self.webcam.process_saved_image(temp_path) + + time.sleep(self.capture_interval) + + except KeyboardInterrupt: + console.print("\n[bold red]Stopping Productivity Coach...[/]") + +def main(args=None): + tool_config = config_manager.get_tool_config("posture") + max_retries = 3 + retry_delay = 5 + capture_interval = tool_config.get("capture_interval", 10) + save_images = tool_config.get("save_images", False) + + for attempt in range(max_retries): + try: + print(f"Attempt {attempt + 1} of {max_retries} to start Productivity Coach...") + coach = ProductivityCoach(capture_interval=capture_interval, save_images=save_images) + coach.run() + break + except RuntimeError as e: + print(f"\nError: {e}") + if attempt < max_retries - 1: + print(f"Retrying in {retry_delay} seconds...") + time.sleep(retry_delay) + else: + print("\nFailed to initialize after multiple attempts.") + print("Please ensure your webcam is connected and permissions are granted.") + if sys.platform == "darwin": + print("\nOn macOS, try these steps:") + print("1. Open System Preferences") + print("2. Go to Security & Privacy -> Privacy -> Camera") + print("3. Ensure your terminal/IDE has camera permissions") + print("4. Run 'tccutil reset Camera' in Terminal with admin privileges") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/tool_use/utils/config_wizard.py b/src/tool_use/utils/config_wizard.py index 024ba82..cecd830 100644 --- a/src/tool_use/utils/config_wizard.py +++ b/src/tool_use/utils/config_wizard.py @@ -34,6 +34,20 @@ "default": "tiny.en", } } + }, + "posture": { + "name": "Posture Coach", + "description": "Monitor posture and focus using AI and webcam", + "config_keys": { + "capture_interval": { + "description": "Interval between captures in seconds", + "default": 10, + }, + "save_images": { + "description": "Save captured images to disk", + "default": False, + } + } } }