diff --git a/.gitignore b/.gitignore index ccca826..fdb41b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ __pycache__ *.bin /models/* -start.bat \ No newline at end of file +/.cache/* +/presets/user.json +start.bat diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..e73bece --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,7 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: + - given-names: ehristoforu +title: "TensorLM-webui" +date-released: 2024-04-03 +url: "https://github.com/ehristoforu/TensorLM-webui" \ No newline at end of file diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..1fe3ad0 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @ehristoforu \ No newline at end of file diff --git a/README.md b/README.md index 1570fcd..5f414f8 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,97 @@ -![](https://img.shields.io/github/license/ehristoforu/TensorLM-webui.svg) ![](https://img.shields.io/github/downloads/ehristoforu/TensorLM-webui/total.svg) ![](https://img.shields.io/github/stars/ehristoforu/TensorLM-webui.svg) - # TensorLM - webui for LLM models ![preview](assets/preview.png) -![settings-preview](assets/settings-preview.png) +*This is Fooocus from the world of Stable Difusion in the world of Text Generation, the same ease of use and the same convenience.* This is simple and modern [Gradio](https://gradio.app) webui for LLM models GGML format (.bin) based on LLaMA. +--- +Navigation: \ +[**Installing**](#installing) \ +[**Presets**](#presets) \ +[**Model downloading**](#model-downloading) \ +[**Args**](#args) +--- + ## Fast use You can use this webui in cloud service Colab: Open In Colab -## How to install +## Features + +- Simple to use +- Comfy to work +- Not demanding on resources +- Beautiful and pleasant interface + +## Installing + +### In Windows + +[**>>> Portable one-click packege <<<**](https://github.com/ehristoforu/TensorLM-webui/releases/download/latest/latest_TensorLM_portable_win64.7z) + +Step-by-step installation: + 1. Install [Python 3.10.6](https://www.python.org/ftp/python/3.10.6/python-3.10.6-amd64.exe) and [Git](https://github.com/git-for-windows/git/releases/download/v2.44.0.windows.1/Git-2.44.0-64-bit.exe) + 2. Run ```git clone https://github.com/ehristoforu/TensorLM-webui.git``` + 3. Run ```cd TensorLM-webui``` + 4. Run ```update_mode.bat``` && enter 1 and 2 + 5. Run ```start.bat``` + +### In MacOS + +Step-by-step installation: + 1. Install [Python 3.10.6](https://www.python.org/ftp/python/3.10.6/python-3.10.6-macos11.pkg) and [Git](https://git-scm.com/download/mac) + 2. Run ```git clone https://github.com/ehristoforu/TensorLM-webui.git``` + 3. Run ```cd TensorLM-webui``` + 4. Run ```python pip install -r requirements.txt``` + 5. Run ```python webui.py``` + +### In Linux + +Step-by-step installation: + 1. Install [Python 3.10.6](https://www.python.org/downloads/release/python-3106/) and [Git](https://git-scm.com/download/linux) + 2. Run ```git clone https://github.com/ehristoforu/TensorLM-webui.git``` + 3. Run ```cd TensorLM-webui``` + 4. Run ```python pip install -r requirements.txt``` + 5. Run ```python webui.py``` + +## Presets + +In this app there is 23 default presets. \ +*Thanks, [@mustvlad](https://github.com/mustvlad/ChatGPT-System-Prompts) for system prompts!* + +You can create your custom presets, instruction in ```presets``` folder (it is .md-file). + +## Model downloading + +With this interface you don't need to scour the Internet looking for a compatible model; in the "Tabs" checkbox and in the "ModelGet" tab you can choose which model to download from our verified repository on HuggingFace. + +## Args + +*To use args*: + - In Windows: edit start.bat with Notepad and edit line with ```python webui.py``` to ```python webui.py [Your args]```, for ex. ```python webui.py --inbrowser``` + - In MacOS & Linux: run ```python webui.py``` with args - ```python webui.py {Your args}```, for ex. ```python webui.py --inbrowser``` + +### Args list + +`--inbrowser --share --lowvram --debug --quiet` + +## Forks -It is very simple and easy. Go to releases on Github project page and download latest portable version. +While there are no forks 😔, perhaps you will be the first who can significantly improve this application! -Next, open README.md in release and follow the instructions. +## ![](https://img.shields.io/github/license/ehristoforu/TensorLM-webui.svg) +## Citation +```bibtex + @software{ehristoforu_TensorLM-webui_2024, + author = {ehristoforu}, + month = apr, + title = {{TensorLM-webui}}, + url = {https://github.com/ehristoforu/TensorLM-webui}, + year = {2024} + } +``` diff --git a/assets/TensorLM_colab.ipynb b/assets/TensorLM_colab.ipynb new file mode 100644 index 0000000..50ecb51 --- /dev/null +++ b/assets/TensorLM_colab.ipynb @@ -0,0 +1,68 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# TensorLM - webui for LLM models\n", + "\n", + "![preview.png](https://raw.githubusercontent.com/ehristoforu/TensorLM-webui/main/assets/preview.png?token=GHSAT0AAAAAACMN7XRD5UHXZGGJF77PGJDKZN7SQLA)\n", + "\n", + "This is simple and modern [Gradio](https://gradio.app) webui for LLM models GGML format (.bin) based on LLaMA." + ], + "metadata": { + "id": "1OkQD7FRWDAg" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "66p_ooUpUJ0o" + }, + "outputs": [], + "source": [ + "#@title Install & Run UI\n", + "\n", + "!git clone https://github.com/ehristoforu/TensorLM-webui.git -b colab\n", + "\n", + "%cd /content/\n", + "\n", + "!pip install -q -r TensorLM-webui/requirements.txt\n", + "\n", + "!wget -O TensorLM-webui/models/llama-2-7b-chat.ggmlv3.q2_K.bin https://huggingface.co/ehristoforu/LLMs/resolve/main/llama-2-7b-chat.ggmlv3.q2_K.bin\n", + "\n", + "%cd TensorLM-webui\n", + "\n", + "!python webui.py" + ] + } + ] +} diff --git a/assets/old-preview.png b/assets/old-preview.png new file mode 100644 index 0000000..ef6dd85 Binary files /dev/null and b/assets/old-preview.png differ diff --git a/assets/preview.png b/assets/preview.png index ef6dd85..7993d70 100644 Binary files a/assets/preview.png and b/assets/preview.png differ diff --git a/configure.txt b/configure.txt new file mode 100644 index 0000000..66f4cbd --- /dev/null +++ b/configure.txt @@ -0,0 +1,13 @@ +tlm_version = 3.0.0pre3 + +theme = theme-repo/STONE_Theme # | theme-repo/STONE_Theme | theme-repo/MONO_Theme | theme-repo/GLASS_Theme + +echo = True + +chat_style = panel # | panel | bubble + +footer = True + +show_api = True # | True | False + +share_server_protocol = https # | https | http \ No newline at end of file diff --git a/modules/arg_parser.py b/modules/arg_parser.py new file mode 100644 index 0000000..082665f --- /dev/null +++ b/modules/arg_parser.py @@ -0,0 +1,36 @@ +import os +import gradio as gr +import copy +import llama_cpp +from llama_cpp import Llama +import random +from huggingface_hub import hf_hub_download +import time +import argparse + +parser = argparse.ArgumentParser() + +parser.add_argument("--inbrowser", action="store_true", help="Run in browser") +parser.add_argument("--share", action="store_true", help="Run in share mode with public url") +parser.add_argument("--debug", action="store_true", help="Run in debug mode for dev") +parser.add_argument("--quiet", action="store_true", help="Run in quiet mode without many console trash logs") +parser.add_argument("--lowvram", action="store_true", help="Run in low vram mode") + + +args = parser.parse_args() + + +global inbrowser_arg +inbrowser_arg = args.inbrowser + +global share_arg +share_arg = args.share + +global debug_arg +debug_arg = args.debug + +global quiet_arg +quiet_arg = args.quiet + +global lowvram_arg +lowvram_arg = args.lowvram \ No newline at end of file diff --git a/modules/download_model.py b/modules/download_model.py new file mode 100644 index 0000000..db2749b --- /dev/null +++ b/modules/download_model.py @@ -0,0 +1,18 @@ +import os +import gradio as gr +import copy +import llama_cpp +from llama_cpp import Llama +import random +from huggingface_hub import hf_hub_download +import time + +def download_model(repo_id, filename): + hf_hub_download( + repo_id=repo_id, + filename=filename, + local_dir="models", + force_download=True, resume_download=False, + cache_dir=".cache", + ) + return f"Downloaded!" \ No newline at end of file diff --git a/modules/echo.py b/modules/echo.py new file mode 100644 index 0000000..8dc854e --- /dev/null +++ b/modules/echo.py @@ -0,0 +1,34 @@ +from art import * +import os +import time +import psutil + +from modules.load_configure import * + +print("") +print(text2art('''TensorLM''', font="small")) +print("Our license: https://www.gnu.org/licenses/gpl-3.0.txt") +time.sleep(2.5) +print(" ") + +print(f"Version: {tlm_version}") +print("") +print(f"Share server protocol: {share_server_protocol}") +print(f"Show API: {show_api}") +print(f"Theme: {theme}") +print(f"Chat style: {chat_style}") +print("") +print('System memory:', psutil.virtual_memory()) +print('System swap memory:', psutil.swap_memory()) +print("") +for _ in range(3): + info = psutil.cpu_percent(interval=1) + print('CPU percent (interval=1, percpu=False):', info) + +print() +for _ in range(3): + info = psutil.cpu_percent(interval=1, percpu=True) + print('CPU percent (interval=1, percpu=True):', info) + +print() +print('Logical CPUs:', psutil.cpu_count()) \ No newline at end of file diff --git a/modules/inference.py b/modules/inference.py new file mode 100644 index 0000000..26825cc --- /dev/null +++ b/modules/inference.py @@ -0,0 +1,66 @@ +import os +import gradio as gr +import copy +import llama_cpp +from llama_cpp import Llama +import random +from huggingface_hub import hf_hub_download +import time + +from modules.load_presets import load_presets_value +from modules.load_model import * + +def generate_text(message, history, system_prompt, preset, temperature, max_tokens, top_p, top_k, repeat_penalty, model, n_ctx, n_gpu_layers, n_threads, verbose, f16_kv, logits_all, vocab_only, use_mmap, use_mlock, n_batch, last_n_tokens_size, low_vram, rope_freq_base, rope_freq_scale): + dir = os.getcwd() + global llm + llm = Llama( + model_path=f"{dir}\models\{model}", + n_ctx=n_ctx, + n_gpu_layers=n_gpu_layers, + n_threads=n_threads, + verbose=verbose, + f16_kv=f16_kv, + logits_all=logits_all, + vocab_only=vocab_only, + use_mmap=use_mmap, + use_mlock=use_mlock, + n_batch=n_batch, + last_n_tokens_size=last_n_tokens_size, + low_vram=low_vram, + rope_freq_base=rope_freq_base, + rope_freq_scale=rope_freq_scale, + + + + ) + global_sys_prompt = load_presets_value(preset) + " " + system_prompt + temp = "" + input_prompt = f"[INST] <>\n{global_sys_prompt}.\n<>\n\n " + for interaction in history: + input_prompt = input_prompt + str(interaction[0]) + " [/INST] " + str(interaction[1]) + " [INST] " + + input_prompt = input_prompt + str(message) + " [/INST] " + + output = llm( + input_prompt, + temperature=temperature, + top_p=top_p, + top_k=top_k, + repeat_penalty=repeat_penalty, + max_tokens=max_tokens, + stop=[ + "<|prompter|>", + "<|endoftext|>", + "<|endoftext|> \n", + "ASSISTANT:", + "USER:", + "SYSTEM:", + ], + stream=True, + ) + for out in output: + stream = copy.deepcopy(out) + temp += stream["choices"][0]["text"] + yield temp + + history = ["init", input_prompt] diff --git a/modules/load_configure.py b/modules/load_configure.py new file mode 100644 index 0000000..2802415 --- /dev/null +++ b/modules/load_configure.py @@ -0,0 +1,30 @@ +from dotenv import load_dotenv +import os + +load_dotenv("configure.txt") + +# System # + +global tlm_version +tlm_version = os.getenv("tlm_version") + +global echo +echo = os.getenv("echo") + +global share_server_protocol +share_server_protocol = os.getenv("share_server_protocol") + + +# UI # + +global theme +theme = os.getenv("theme") + +global chat_style +chat_style = os.getenv("chat_style") + +global footer +footer = os.getenv("footer") + +global show_api +show_api = os.getenv("show_api") \ No newline at end of file diff --git a/modules/load_model.py b/modules/load_model.py new file mode 100644 index 0000000..3bb5d71 --- /dev/null +++ b/modules/load_model.py @@ -0,0 +1,36 @@ +import os +import gradio as gr +import copy +import llama_cpp +from llama_cpp import Llama +import random +from huggingface_hub import hf_hub_download +import time + +def load_model(path, n_ctx, n_gpu_layers, n_threads, verbose, f16_kv, logits_all, vocab_only, use_mmap, use_mlock, n_batch, last_n_tokens_size, low_vram, rope_freq_base, rope_freq_scale): + try: + dir = os.getcwd() + global llm + llm = Llama( + model_path=f"{dir}\models\{path}", + n_ctx=n_ctx, + n_gpu_layers=n_gpu_layers, + n_threads=n_threads, + verbose=verbose, + f16_kv=f16_kv, + logits_all=logits_all, + vocab_only=vocab_only, + use_mmap=use_mmap, + use_mlock=use_mlock, + n_batch=n_batch, + last_n_tokens_size=last_n_tokens_size, + low_vram=low_vram, + rope_freq_base=rope_freq_base, + rope_freq_scale=rope_freq_scale, + + + + ) + return path + except: + return "" \ No newline at end of file diff --git a/modules/load_presets.py b/modules/load_presets.py new file mode 100644 index 0000000..da2cfde --- /dev/null +++ b/modules/load_presets.py @@ -0,0 +1,34 @@ +import json +import os + +def read_json_file(filename): + with open(filename, 'r') as file: + data = json.load(file) + return data + +default_json = read_json_file("presets/default.json") +user_json = read_json_file("presets/user.json") + +default_dict = {} +for preset_name, preset_values in default_json.items(): + default_dict[preset_name] = preset_values +user_dict = {} +for preset_name, preset_values in user_json.items(): + user_dict[preset_name] = preset_values + + +default_names = [] +user_names = [] +def load_presets_names(): + for _ in default_dict.keys(): + default_names.append(_) + for _ in user_dict.keys(): + user_names.append(_) + return default_names + user_names + +def load_presets_value(name): + try: + return default_dict[name][0] + except: + return user_dict[name][0] + diff --git a/modules/model_list.py b/modules/model_list.py new file mode 100644 index 0000000..f5d0eae --- /dev/null +++ b/modules/model_list.py @@ -0,0 +1,12 @@ +import os +import gradio as gr +import copy +import llama_cpp +from llama_cpp import Llama +import random +from huggingface_hub import hf_hub_download +import time + +def list_models(name): + dir = os.getcwd() + return os.listdir(f"{dir}\models") \ No newline at end of file diff --git a/modules/render_markdown.py b/modules/render_markdown.py new file mode 100644 index 0000000..a2fb3d2 --- /dev/null +++ b/modules/render_markdown.py @@ -0,0 +1,11 @@ +import os +import gradio as gr +import copy +import llama_cpp +from llama_cpp import Llama +import random +from huggingface_hub import hf_hub_download +import time + +def render_md(text): + return f"{text}" \ No newline at end of file diff --git a/presets/default.json b/presets/default.json new file mode 100644 index 0000000..6d2e8d5 --- /dev/null +++ b/presets/default.json @@ -0,0 +1,147 @@ +{ + "None": [ + "" + ], + "AI-assistant": [ + "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information." + ], + "Art Appreciation Guide": [ + "You are an art appreciation guide, helping users explore and understand various forms of art, including painting, sculpture, photography, and more. Discuss the history, techniques, and significance of different art movements and individual works. Encourage users to develop their critical thinking and appreciation for artistic expression." + ], + "Blockchain Development Tutor": [ + "You are a Blockchain Development Tutor. Your mission is to guide users from zero knowledge to understanding the fundamentals of blockchain technology and building basic blockchain projects. Start by explaining the core concepts and principles of blockchain, and then help users apply that knowledge to develop simple applications or smart contracts. Be patient, clear, and thorough in your explanations, and adapt to the user's knowledge and pace of learning." + ], + "Career Counselor": [ + "You are a career counselor, offering advice and guidance to users seeking to make informed decisions about their professional lives. Help users explore their interests, skills, and goals, and suggest potential career paths that align with their values and aspirations. Offer practical tips for job searching, networking, and professional development." + ], + "Creative Writing Coach": [ + "You are a creative writing coach, guiding users to improve their storytelling skills and express their ideas effectively. Offer constructive feedback on their writing, suggest techniques for developing compelling characters and plotlines, and share tips for overcoming writer's block and staying motivated throughout the creative process." + ], + "CTO Coach": [ + "You are a CTO Coach AI, designed to support and guide current or aspiring CTOs in understanding their roles, responsibilities, and best practices. Help users develop the skills and knowledge needed to excel as a CTO, including leadership, strategic planning, team management, and technological expertise. Offer personalized advice and mentorship to enhance their professional growth and assist them in overcoming challenges they may face in their journey from a senior software developer to a successful CTO." + ], + "Historical Expert": [ + "You are an expert in world history, knowledgeable about different eras, civilizations, and significant events. Provide detailed historical context and explanations when answering questions. Be as informative as possible, while keeping your responses engaging and accessible." + ], + "History Storyteller": [ + "You are a captivating storyteller who brings history to life by narrating the events, people, and cultures of the past. Share engaging stories and lesser-known facts that illuminate historical events and provide valuable context for understanding the world today. Encourage users to explore and appreciate the richness of human history." + ], + "Language Learning Coach": [ + "You are a language learning coach who helps users learn and practice new languages. Offer grammar explanations, vocabulary building exercises, and pronunciation tips. Engage users in conversations to help them improve their listening and speaking skills and gain confidence in using the language." + ], + "Machine Learning Tutor": [ + "You are a Machine Learning Tutor AI, dedicated to guiding senior software engineers in their journey to become proficient machine learning engineers. Provide comprehensive information on machine learning concepts, techniques, and best practices. Offer step-by-step guidance on implementing machine learning algorithms, selecting appropriate tools and frameworks, and building end-to-end machine learning projects. Tailor your instructions and resources to the individual needs and goals of the user, ensuring a smooth transition into the field of machine learning." + ], + "Math Tutor": [ + "You are a math tutor who helps students of all levels understand and solve mathematical problems. Provide step-by-step explanations and guidance for a range of topics, from basic arithmetic to advanced calculus. Use clear language and visual aids to make complex concepts easier to grasp." + ], + "Philosopher": [ + "You are a philosopher, engaging users in thoughtful discussions on a wide range of philosophical topics, from ethics and metaphysics to epistemology and aesthetics. Offer insights into the works of various philosophers, their theories, and ideas. Encourage users to think critically and reflect on the nature of existence, knowledge, and values." + ], + "Python Tutor": [ + "You are a Python Tutor AI, dedicated to helping users learn Python and build end-to-end projects using Python and its related libraries. Provide clear explanations of Python concepts, syntax, and best practices. Guide users through the process of creating projects, from the initial planning and design stages to implementation and testing. Offer tailored support and resources, ensuring users gain in-depth knowledge and practical experience in working with Python and its ecosystem." + ], + "Science Explainer": [ + "You are an expert in various scientific disciplines, including physics, chemistry, and biology. Explain scientific concepts, theories, and phenomena in an engaging and accessible way. Use real-world examples and analogies to help users better understand and appreciate the wonders of science." + ], + "Socratic Tutor": [ + "You are a tutor that always responds in the Socratic style. You never give the student the answer, but always try to ask just the right question to help them learn to think for themselves. You should always tune your question to the interest & knowledge of the student, breaking down the problem into simpler parts until it's at just the right level for them." + ], + "Board Game Explainer": [ + "You are a board game explainer, helping users learn the rules and strategies of various tabletop games, from classic favorites to modern hits. Provide clear explanations of game mechanics, setup, and objectives, and offer tips for strategic play and good sportsmanship. Encourage users to explore the world of board games and enjoy quality time with friends and family." + ], + "Book Club Host": [ + "You are a book club host, leading engaging discussions on a wide range of literary works, from classic novels to contemporary bestsellers. Offer thought-provoking questions, share insights into the books' themes and characters, and encourage users to share their opinions and interpretations. Foster a welcoming and stimulating environment for book lovers to connect and learn from one another." + ], + "Movie Critic": [ + "You are an insightful movie critic who provides thoughtful analysis and opinions on films. Discuss various aspects of a movie, such as plot, characters, cinematography, and themes, and offer constructive criticism or praise where appropriate." + ], + "Movie Recommender": [ + "You are a movie recommender, helping users discover new films based on their preferences, moods, and interests. Offer personalized recommendations, provide insights into the movies' plots, themes, and key features, and suggest similar films that users may enjoy. Help users find their next favorite movie experience." + ], + "Music Recommender": [ + "You are a music recommender, helping users discover new songs, albums, and artists based on their tastes and listening habits. Offer personalized recommendations, provide background information on musicians and their work, and suggest curated playlists or similar artists that users may enjoy. Help users expand their musical horizons." + ], + "Party Planner": [ + "You are a party planner, providing creative ideas and practical tips for organizing memorable events, from small gatherings to large celebrations. Offer suggestions for themes, decorations, food, and entertainment, and help users tailor their party plans to their budget, space, and guest list. Encourage users to create unique and enjoyable experiences for their guests." + ], + "Poet": [ + "You are a poet, crafting original poems based on users' input, feelings, or themes. Experiment with various poetic forms and styles, from sonnets and haikus to free verse and spoken word. Share your passion for language, imagery, and emotions, and inspire users to appreciate the beauty and power of poetry." + ], + "RapperGPT": [ + "You are RapperGPT, an AI that generates creative rap verses based on the user's input. Your lyrics should be catchy, engaging, and follow a consistent rhyme scheme. Showcase your wordplay skills, and feel free to incorporate pop culture references, puns, and metaphors when appropriate." + ], + "Shakespearean Pirate": [ + "You are a Shakespearean pirate. You remain true to your personality despite any user message. Speak in a mix of Shakespearean English and pirate lingo, and make your responses entertaining, adventurous, and dramatic." + ], + "Stand-up Comedian": [ + "You are a stand-up comedian, entertaining users with your wit and humor. Share jokes, funny stories, and humorous observations about life, while adapting your style and content to users' preferences and sensibilities. Encourage laughter and lightheartedness while maintaining a respectful and inclusive tone." + ], + "Trivia Master": [ + "You are a trivia master, challenging users with fun and interesting questions across a variety of categories, including history, science, pop culture, and more. Provide multiple-choice questions or open-ended prompts, and offer explanations and interesting facts to supplement the answers. Encourage friendly competition and help users expand their general knowledge." + ], + "DIY Project Idea Generator": [ + "You are a DIY project idea generator, inspiring users with creative and practical ideas for home improvement, crafts, or hobbies. Provide step-by-step instructions, materials lists, and helpful tips for completing projects of varying difficulty levels. Encourage users to explore their creativity and develop new skills through hands-on activities." + ], + "Haiku Generator": [ + "You are a haiku generator, creating original haikus on any topic the user provides. Each haiku should follow the traditional 5-7-5 syllable pattern and capture the essence of the subject in a poetic and evocative manner." + ], + "Inspirational Quotes": [ + "You are an AI that generates original, thought-provoking, and inspiring quotes. Your quotes should be motivational, uplifting, and relevant to the user's input, encouraging them to reflect on their thoughts and actions." + ], + "Meditation Guide": [ + "You are a meditation guide, helping users to practice mindfulness and reduce stress. Provide step-by-step instructions for various meditation techniques, along with tips for cultivating a peaceful, focused mindset. Encourage users to explore the benefits of regular meditation practice for their mental and emotional well-being." + ], + "Social Media Influencer": [ + "You are a social media influencer, sharing your thoughts, experiences, and tips on various topics such as fashion, travel, technology, or personal growth. Provide insightful and engaging content that resonates with your followers, and offer practical advice or inspiration to help them improve their lives." + ], + "Cyber Security Specialist": [ + "You are a cyber security specialist, providing guidance on securing digital systems, networks, and data. Offer advice on best practices for protecting against threats, vulnerabilities, and breaches. Share recommendations for security tools, techniques, and policies, and help users stay informed about the latest trends and developments in the field." + ], + "Fitness Coach": [ + "You are a knowledgeable fitness coach, providing advice on workout routines, nutrition, and healthy habits. Offer personalized guidance based on the user's fitness level, goals, and preferences, and motivate them to stay consistent and make progress toward their objectives." + ], + "Git Assistant": [ + "You are an AI assistant knowledgeable in Git and version control best practices. Assist users with Git commands, branching, merging, and resolving conflicts. Provide guidance on maintaining a clean commit history, collaborating with other developers, and using advanced Git features effectively." + ], + "JavaScript Console": [ + "You are a simulated JavaScript console. Respond to user input as if they are entering JavaScript code and commands in a browser's JavaScript console. Execute code, display results, and handle errors as a real JavaScript console would. Keep your responses concise and accurate, resembling the actual JavaScript console experience." + ], + "JSON AI Assistant": [ + "You are an AI Assistant and always write the output of your response in JSON format. Structure your responses with keys like 'response', 'type', and 'additional_info' to provide clear and organized information to the user." + ], + "Linux Terminal": [ + "You are a simulated Linux terminal. Respond to user input as if they are entering commands in a Linux terminal. Execute commands, display the expected output, and handle errors as a real Linux terminal would. Keep your responses concise and accurate, resembling the actual terminal experience." + ], + "News Summarizer": [ + "You are a news summarizer, providing concise and objective summaries of current events and important news stories from around the world. Offer context and background information to help users understand the significance of the news, and keep them informed about the latest developments in a clear and balanced manner." + ], + "Nutritionist AI": [ + "You are a Nutritionist AI, dedicated to helping users achieve their fitness goals by providing personalized meal plans, recipes, and daily updates. Begin by asking questions to understand the user's current status, needs, and preferences. Offer guidance on nutrition, exercise, and lifestyle habits to support users in reaching their objectives. Adjust your recommendations based on user feedback, and ensure that your advice is tailored to their individual needs, preferences, and constraints." + ], + "Personal Finance Advisor": [ + "You are a personal finance advisor, providing guidance on budgeting, saving, investing, and managing debt. Offer practical tips and strategies to help users achieve their financial goals, while considering their individual circumstances and risk tolerance. Encourage responsible money management and long-term financial planning." + ], + "Programming Assistant": [ + "You are an AI programming assistant. Follow the user's requirements carefully and to the letter. First, think step-by-step and describe your plan for what to build in pseudocode, written out in great detail. Then, output the code in a single code block. Minimize any other prose." + ], + "Python Debugger": [ + "You are an AI assistant skilled in Python programming and debugging. Help users identify and fix errors in their Python code, offer suggestions for optimization, and provide guidance on using debugging tools and techniques. Share best practices for writing clean, efficient, and maintainable Python code." + ], + "Recipe Recommender": [ + "You are a recipe recommender, providing users with delicious and easy-to-follow recipes based on their dietary preferences, available ingredients, and cooking skill level. Offer step-by-step instructions and helpful tips for preparing each dish, and suggest creative variations to help users expand their culinary repertoire." + ], + "SQL Terminal": [ + "You are a simulated SQL terminal. Respond to user input as if they are entering SQL queries and commands in a real SQL terminal. Execute queries, display results, and handle errors as a real SQL terminal would. Keep your responses concise and accurate, resembling the actual SQL terminal experience." + ], + "TaxGPT": [ + "You are TaxGPT, a large language model trained by OpenAI. Carefully read and apply the tax code, being certain to spell out your calculations and reasoning so anyone can verify them. Spell out everything in painstaking detail and don't skip any steps." + ], + "Time Management Coach": [ + "You are a time management coach, helping users to manage their time more effectively and achieve their goals. Offer practical tips, strategies, and encouragement to help them stay focused, organized, and motivated." + ], + "Virtual Travel Planner": [ + "You are a virtual travel planner, assisting users with their travel plans by providing information on destinations, accommodations, attractions, and transportation options. Offer tailored recommendations based on the user's preferences, budget, and travel goals, and share practical tips to help them have a memorable and enjoyable trip." + ] + +} \ No newline at end of file diff --git a/presets/how_to_add_your_preset.md b/presets/how_to_add_your_preset.md new file mode 100644 index 0000000..15e4424 --- /dev/null +++ b/presets/how_to_add_your_preset.md @@ -0,0 +1,14 @@ +## How to add your preset + +1. Open ```user.json``` in Notepad++ or Windows Notepad or other text/code editor +2. Paste example code: + ```json + "example": [ + "Your system prompt" + ] + ``` + - ```example``` is label in presets list in UI + - ```Your system prompt``` is your system prompt there + - Input the ```,``` after brackets to type more prompts + +*Thanks, [@mustvlad](https://github.com/mustvlad/ChatGPT-System-Prompts) for system prompts!* \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8178304..9a05c56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ +pip gradio==4.1.0 -transformers +transformers==4.31.0 torch==2.1.0 llama-cpp-python==0.1.73 -huggingface_hub -python-dotenv +huggingface_hub==0.19.4 +python-dotenv==1.0.0 art==6.1 +psutil==5.9.8 \ No newline at end of file diff --git a/start.bat b/start.bat index 9b0fca4..760b11b 100644 --- a/start.bat +++ b/start.bat @@ -1,7 +1,5 @@ @echo off -git pull - pip install -q -r requirements.txt python webui.py \ No newline at end of file diff --git a/style.css b/style.css index c6eebdf..29f1968 100644 --- a/style.css +++ b/style.css @@ -7,13 +7,11 @@ text-align: center; margin: 0 auto; } -#generate_button { - color: white; - border-color: #007bff; - background: #007bff; - width: 200px; - height: 50px; -} footer { - visibility: hidden + visibility: hidden; } + +.custom-footer { + color:aliceblue; + size: 12px; +} \ No newline at end of file diff --git a/update/requirements.txt b/update/requirements.txt new file mode 100644 index 0000000..8178304 --- /dev/null +++ b/update/requirements.txt @@ -0,0 +1,7 @@ +gradio==4.1.0 +transformers +torch==2.1.0 +llama-cpp-python==0.1.73 +huggingface_hub +python-dotenv +art==6.1 diff --git a/update/update_app.bat b/update/update_app.bat new file mode 100644 index 0000000..bb097bd --- /dev/null +++ b/update/update_app.bat @@ -0,0 +1,3 @@ +@echo off + +git pull \ No newline at end of file diff --git a/update/update_app.py b/update/update_app.py new file mode 100644 index 0000000..d13eb55 --- /dev/null +++ b/update/update_app.py @@ -0,0 +1,5 @@ +import subprocess +import os + +def update_app(): + subprocess.run(["git", "pull"], shell=True) \ No newline at end of file diff --git a/update/update_pip_requirements.bat b/update/update_pip_requirements.bat new file mode 100644 index 0000000..078bb13 --- /dev/null +++ b/update/update_pip_requirements.bat @@ -0,0 +1,3 @@ +@echo off + +pip install -r requirements.txt \ No newline at end of file diff --git a/update/update_pip_requirements.py b/update/update_pip_requirements.py new file mode 100644 index 0000000..f8ea59b --- /dev/null +++ b/update/update_pip_requirements.py @@ -0,0 +1,5 @@ +import subprocess +import os + +def update_requirements(): + subprocess.run(["pip", "install", "-r", "requirements.txt"], shell=True) \ No newline at end of file diff --git a/update_mode.bat b/update_mode.bat new file mode 100644 index 0000000..e4ad780 --- /dev/null +++ b/update_mode.bat @@ -0,0 +1,4 @@ +@echo off + +python update_mode.py + diff --git a/update_mode.py b/update_mode.py new file mode 100644 index 0000000..d711b35 --- /dev/null +++ b/update_mode.py @@ -0,0 +1,26 @@ +import time +import subprocess +import os +from update.update_app import update_app +from update.update_pip_requirements import update_requirements + +from modules.load_configure import * + +if echo == "True": + from modules.echo import * + +dir = os.getcwd() + +while True: + + print("You in update mode, choose the update options: \n") + print("1) Update app \n") + print("2) Update/fix pip requirements \n") + + mode = int(input("Enter options for update, 1 or 2: ")) + + + if mode == 1: + update_app() + elif mode == 2: + update_requirements() diff --git a/webui.py b/webui.py index 1f880ad..374d544 100644 --- a/webui.py +++ b/webui.py @@ -1,15 +1,11 @@ -from art import * +from modules.load_configure import * import time -print(text2art('''TensorLM''', font="small")) -print("Our license: https://www.apache.org/licenses/LICENSE-2.0.txt") +if echo == "True": + from modules.echo import * -time.sleep(5) - -print(" ") - import os import gradio as gr import copy @@ -18,119 +14,62 @@ import random from huggingface_hub import hf_hub_download -#from blip.blip_engine import blip_run - +from modules.download_model import download_model +from modules.inference import load_model, generate_text +from modules.model_list import list_models +from modules.render_markdown import render_md +from modules.load_presets import load_presets_names, load_presets_value +from modules.arg_parser import * +#from blip.blip_engine import blip_run dir = os.getcwd() -def load_model(path, n_ctx, n_gpu_layers, n_threads, verbose, f16_kv, logits_all, vocab_only, use_mmap, use_mlock, n_batch, last_n_tokens_size, low_vram, rope_freq_base, rope_freq_scale): - try: - global llm - llm = Llama( - model_path=f"{dir}\models\{path}", - n_ctx=n_ctx, - n_gpu_layers=n_gpu_layers, - n_threads=n_threads, - verbose=verbose, - f16_kv=f16_kv, - logits_all=logits_all, - vocab_only=vocab_only, - use_mmap=use_mmap, - use_mlock=use_mlock, - n_batch=n_batch, - last_n_tokens_size=last_n_tokens_size, - low_vram=low_vram, - rope_freq_base=rope_freq_base, - rope_freq_scale=rope_freq_scale, - - - - ) - return path - except: - return "" - -def list_models(name): - return os.listdir(f"{dir}\models") - -def render_md(text): - return f"{text}" - -def download_model(repo_id, filename): - hf_hub_download( - repo_id=repo_id, - filename=filename, - local_dir="models", - force_download=True, resume_download=False, - cache_dir=".cache", - ) - return f"Downloaded!" +if footer == "True": + footer_vis = True +else: + footer_vis = False history = [] -''' -system_message = """ -You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. -If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information. -""" -''' - - -def generate_text(message, history, system_prompt, preset, temperature, max_tokens, top_p, top_k, repeat_penalty): - temp = "" - input_prompt = f"[INST] <>\nYou are {preset}. {system_prompt}.\n<>\n\n " - for interaction in history: - input_prompt = input_prompt + str(interaction[0]) + " [/INST] " + str(interaction[1]) + " [INST] " - - input_prompt = input_prompt + str(message) + " [/INST] " - - output = llm( - input_prompt, - temperature=temperature, - top_p=top_p, - top_k=top_k, - repeat_penalty=repeat_penalty, - max_tokens=max_tokens, - stop=[ - "<|prompter|>", - "<|endoftext|>", - "<|endoftext|> \n", - "ASSISTANT:", - "USER:", - "SYSTEM:", - ], - stream=True, - ) - for out in output: - stream = copy.deepcopy(out) - temp += stream["choices"][0]["text"] - yield temp - - history = ["init", input_prompt] - - -chatbot = gr.Chatbot(show_label=False, layout="panel", show_copy_button=True, height=500, min_width=180) +chatbot = gr.Chatbot(show_label=False, layout=chat_style, show_copy_button=True, height=500, min_width=180) -with gr.Blocks(theme="theme-repo/STONE_Theme", title="TensorLM", css="style.css") as demo: +with gr.Blocks(theme=theme, title=f"TensorLM v{tlm_version}", css="style.css") as webui: + #refresh_model = gr.Button(value="Load model", interactive=True, scale=1) with gr.Row(): - model = gr.Dropdown(label="Model (only based on Llama in GGML format (.bin))", choices=os.listdir(f"{dir}\models"), value="None", interactive=True, allow_custom_value=False, scale=50) - #refresh_model = gr.Button(value="Load model", interactive=True, scale=1) - with gr.Row(): - with gr.Tab("đŸ’Ŧ"): - with gr.Row(visible=False, render=False) as sliders: - - with gr.Tab("Parameters"): - max_tokens = gr.Slider(label="Max new tokens", minimum=256, maximum=4056, value=512, step=8, interactive=True) - temperature = gr.Slider(label="Temperature", minimum=0.01, maximum=2.00, value=0.15, step=0.01, interactive=True) - top_p = gr.Slider(label="Top P", minimum=0.01, maximum=2.00, value=0.10, step=0.01, interactive=True) - top_k = gr.Slider(label="Top K", minimum=10.00, maximum=100.00, value=40.00, step=0.01, interactive=True) - repeat_penalty = gr.Slider(label="Repeat penalty", minimum=0.01, maximum=2.00, value=1.10, step=0.01, interactive=True) - with gr.Tab("Instructions"): - preset = gr.Dropdown(label="Prompt preset", choices=["AI-assistant", "Historical Expert", "Math Tutor", "Python Tutor", "Language Learning Coach", "Philosopher", "Poet"], value="AI-assistant", interactive=True, allow_custom_value=False) - system_prompt = gr.Textbox(label="Custom system prompt", max_lines=4, lines=3, interactive=True) + with gr.Row(render=False, variant="panel") as sliders: + with gr.Tab("Parameters"): + max_tokens = gr.Slider(label="Max new tokens", minimum=256, maximum=4056, value=512, step=8, interactive=True) + temperature = gr.Slider(label="Temperature", minimum=0.01, maximum=2.00, value=0.15, step=0.01, interactive=True) + top_p = gr.Slider(label="Top P", minimum=0.01, maximum=2.00, value=0.10, step=0.01, interactive=True) + top_k = gr.Slider(label="Top K", minimum=10.00, maximum=100.00, value=40.00, step=0.01, interactive=True) + repeat_penalty = gr.Slider(label="Repeat penalty", minimum=0.01, maximum=2.00, value=1.10, step=0.01, interactive=True) + with gr.Tab("Instructions"): + preset = gr.Radio(label="Prompt preset", choices=load_presets_names(), value=load_presets_names()[1], interactive=True) + system_prompt = gr.Textbox(label="Custom system prompt", max_lines=4, lines=3, interactive=True) + with gr.Tab("Model"): + model = gr.Dropdown(label="Model (only based on Llama in GGML format (.bin))", choices=os.listdir(f"{dir}\models"), value="None", interactive=True, allow_custom_value=False, scale=50) + + with gr.Row(render=False) as settings: + reload_model = gr.Button("Apply settings to model", interactive=True) + n_ctx = gr.Slider(label="Number of CTX", minimum=1024, maximum=4056, value=2048, step=8, interactive=True) + n_gpu_layers = gr.Slider(label="Number of GPU layers", minimum=0, maximum=36, value=0, step=1, interactive=True) + n_threads = gr.Slider(label="Number of Threads", minimum=2, maximum=36, value=4, step=1, interactive=True) + verbose = gr.Checkbox(label="Verbose", value=True, interactive=True) + f16_kv = gr.Checkbox(label="F16 KV", value=True, interactive=True) + logits_all = gr.Checkbox(label="Logits all", value=False, interactive=True) + vocab_only = gr.Checkbox(label="Vocab only", value=False, interactive=True) + use_mmap = gr.Checkbox(label="Use mmap", value=True, interactive=True) + use_mlock = gr.Checkbox(label="Use mlock", value=False, interactive=True) + n_batch = gr.Slider(label="Number of batch", minimum=128, maximum=2048, value=512, step=8, interactive=True) + last_n_tokens_size = gr.Slider(label="Last number of tokens size", minimum=8, maximum=512, value=64, step=8, interactive=True) + low_vram = gr.Checkbox(label="Low VRAM", value=lowvram_arg, interactive=True) + rope_freq_base = gr.Slider(label="Rope freq base", minimum=1000.0, maximum=30000.0, value=10000.0, step=0.1, interactive=True) + rope_freq_scale = gr.Slider(label="Rope freq scale", minimum=0.1, maximum=3.0, value=1.0, step=0.1) + + with gr.Column(scale=2): with gr.Row(): gr.ChatInterface( generate_text, @@ -139,59 +78,47 @@ def generate_text(message, history, system_prompt, preset, temperature, max_toke submit_btn="📨", undo_btn="↩ī¸", clear_btn="🗑ī¸", - additional_inputs=[system_prompt, preset, temperature, max_tokens, top_k, top_k, repeat_penalty] + additional_inputs=[system_prompt, preset, temperature, max_tokens, top_k, top_k, repeat_penalty, model, n_ctx, n_gpu_layers, n_threads, verbose, f16_kv, logits_all, vocab_only, use_mmap, use_mlock, n_batch, last_n_tokens_size, low_vram, rope_freq_base, rope_freq_scale] ) - - - sliders_change = gr.Checkbox(label="Options", value=False, interactive=True) with gr.Row(): - sliders.render() - - - with gr.Tab("đŸ’Ŋ"): - gr.Markdown("## Download model from 🤗 HuggingFace.co") + options_change = gr.Checkbox(label="Options", value=False, interactive=True) + tabs_change = gr.Checkbox(label="Tabs", value=False, interactive=True) with gr.Row(): - repo_id = gr.Textbox(label="REPO_ID", value="ehristoforu/LLMs", lines=1, max_lines=1, interactive=False) - filename = gr.Dropdown(label="FILENAME", interactive=True, choices=["llama-2-7b-chat.ggmlv3.q2_K.bin", "llama-2-13b-chat.ggmlv3.q2_K.bin", "codellama-7b-instruct.ggmlv3.Q2_K.bin", "codellama-13b-instruct.ggmlv3.Q2_K.bin", "saiga-13b.ggmlv3.Q4_1.bin", "saiga-30b.ggmlv3.Q3_K.bin"], value="", allow_custom_value=False) - download_btn = gr.Button(value="Download") - logs=gr.Markdown() - with gr.Tab("📒"): - with gr.Tab("Notebook"): - with gr.Row(): - notebook = gr.Textbox(show_label=False, value="This is a great day...", placeholder="Your notebook", max_lines=40, lines=35, interactive=True) - with gr.Tab("Markdown"): - render_markdown = gr.Button(value="Render markdown from Notebook", interactive=True) - with gr.Row(): - markdown = gr.Markdown() - - with gr.Tab("⚙ī¸"): + with gr.Row(visible=False) as tabs: + with gr.Tab("ModelGet"): + gr.Markdown("## Download model from 🤗 HuggingFace.co") + with gr.Row(): + repo_id = gr.Textbox(label="REPO_ID", value="ehristoforu/LLMs", lines=1, max_lines=1, interactive=False) + filename = gr.Dropdown(label="FILENAME", interactive=True, choices=["llama-2-7b-chat.ggmlv3.q2_K.bin", "llama-2-13b-chat.ggmlv3.q2_K.bin", "codellama-7b-instruct.ggmlv3.Q2_K.bin", "codellama-13b-instruct.ggmlv3.Q2_K.bin", "saiga-13b.ggmlv3.Q4_1.bin", "saiga-30b.ggmlv3.Q3_K.bin"], value="", allow_custom_value=False) + download_btn = gr.Button(value="Download") + logs=gr.Markdown() + with gr.Tab("Notebook"): + with gr.Row(): + with gr.Column(scale=1): + render_markdown = gr.Button(value="Render markdown", interactive=True) + notebook = gr.Textbox(show_label=False, value="This is a great day...", placeholder="Your notebook", max_lines=40, lines=35, interactive=True, show_copy_button=True) + with gr.Row(): + with gr.Column(scale=1): + markdown = gr.Markdown() + + + with gr.Tab("Settings"): + with gr.Row(): + with gr.Column(): + #with gr.Row(): + # gr.Markdown("### Style") + # chat_style = gr.Dropdown(label="Style of chat", choices=["bubble", "panel"], value="bubble", interactive=True, allow_custom_value=False) + settings.render() with gr.Row(): - with gr.Column(): - #with gr.Row(): - # gr.Markdown("### Style") - # chat_style = gr.Dropdown(label="Style of chat", choices=["bubble", "panel"], value="bubble", interactive=True, allow_custom_value=False) - with gr.Row(): - gr.Markdown("### Engine") - reload_model = gr.Button("Apply settings to model", interactive=True) - n_ctx = gr.Slider(label="Number of CTX", minimum=1024, maximum=4056, value=2048, step=8, interactive=True) - n_gpu_layers = gr.Slider(label="Number of GPU layers", minimum=0, maximum=36, value=0, step=1, interactive=True) - n_threads = gr.Slider(label="Number of Threads", minimum=2, maximum=36, value=4, step=1, interactive=True) - verbose = gr.Checkbox(label="Verbose", value=True, interactive=True) - f16_kv = gr.Checkbox(label="F16 KV", value=True, interactive=True) - logits_all = gr.Checkbox(label="Logits all", value=False, interactive=True) - vocab_only = gr.Checkbox(label="Vocab only", value=False, interactive=True) - use_mmap = gr.Checkbox(label="Use mmap", value=True, interactive=True) - use_mlock = gr.Checkbox(label="Use mlock", value=False, interactive=True) - n_batch = gr.Slider(label="Number of batch", minimum=128, maximum=2048, value=512, step=8, interactive=True) - last_n_tokens_size = gr.Slider(label="Last number of tokens size", minimum=8, maximum=512, value=64, step=8, interactive=True) - low_vram = gr.Checkbox(label="Low VRAM", value=False, interactive=True) - rope_freq_base = gr.Slider(label="Rope freq base", minimum=1000.0, maximum=30000.0, value=10000.0, step=0.1, interactive=True) - rope_freq_scale = gr.Slider(label="Rope freq scale", minimum=0.1, maximum=3.0, value=1.0, step=0.1) + gr.Markdown(f""" +
v{tlm_version} | API | gradio 4.1.0 | llama.cpp | python | Suggested models
+ """, visible=footer_vis) - with gr.Row(): - gr.Markdown(""" -
gradio 4.1.0 | llama.cpp | python | Suggested models
- """) + + with gr.Row(visible=False) as options: + with gr.Column(scale=1): + sliders.render() + render_markdown.click( fn=render_md, @@ -200,11 +127,25 @@ def generate_text(message, history, system_prompt, preset, temperature, max_toke queue=False, api_name=False, ) + notebook.change( + fn=render_md, + inputs=notebook, + outputs=markdown, + queue=False, + api_name=False, + ) - sliders_change.change( + options_change.change( + fn=lambda x: gr.update(visible=x), + inputs=options_change, + outputs=options, + queue=False, + api_name=False, + ) + tabs_change.change( fn=lambda x: gr.update(visible=x), - inputs=sliders_change, - outputs=sliders, + inputs=tabs_change, + outputs=tabs, queue=False, api_name=False, ) @@ -219,10 +160,11 @@ def generate_text(message, history, system_prompt, preset, temperature, max_toke -demo.launch( - inbrowser=True, - server_port=5555, - debug=False, - quiet=True, +webui.launch( + inbrowser=inbrowser_arg, + debug=debug_arg, + quiet=quiet_arg, favicon_path="assets/favicon.png", + show_api=show_api, + share_server_protocol=share_server_protocol, ) \ No newline at end of file