diff --git a/.github/workflows/ruff_linter.yml b/.github/workflows/ruff_linter.yml new file mode 100644 index 00000000..056f3a40 --- /dev/null +++ b/.github/workflows/ruff_linter.yml @@ -0,0 +1,26 @@ +name: Ruff + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install Ruff + run: | + python -m pip install --upgrade pip + pip install ruff==0.5.1 + + - name: Run Ruff Check + run: | + ruff check + diff --git a/README.md b/README.md index 7cb05448..afe072c9 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,12 @@ Open [http://0.0.0.0:8000/docs](http://0.0.0.0:8000/docs) for the API docs. ## Tests -1) Ruff (TODO) +1) Ruff: Auto-formatter and checker for python + +``` +pip install ruff +ruff format && ruff check +``` 2) Mypy: A static type checker for python diff --git a/evaluation/human_evaluation/main.py b/evaluation/human_evaluation/main.py index 023be58e..18091e6b 100644 --- a/evaluation/human_evaluation/main.py +++ b/evaluation/human_evaluation/main.py @@ -11,6 +11,7 @@ read_question_and_description, ) + def main() -> None: load_dotenv() diff --git a/evaluation/human_evaluation/setup.py b/evaluation/human_evaluation/setup.py index 44e66be4..63d038ca 100644 --- a/evaluation/human_evaluation/setup.py +++ b/evaluation/human_evaluation/setup.py @@ -108,13 +108,35 @@ def update_env_file(updates: dict[str, str]) -> None: with open(env_file, "w") as file: file.writelines(new_lines) + def main() -> None: - parser = argparse.ArgumentParser(description="Create Google Form and/or Google Sheet, and update .env file.") - parser.add_argument('--create-form', action='store_true', help='Create a Google Form') - parser.add_argument('--create-sheet', action='store_true', help='Create a Google Sheet') - parser.add_argument('--user-email', type=str, required=True, help="Email address to share the created resources with") - parser.add_argument('--form-title', type=str, default='OR Assistant Feedback Form', help='Title for the Google Form') - parser.add_argument('--sheet-title', type=str, default='OR Assistant Evaluation Sheet', help='Title for the Google Sheet') + parser = argparse.ArgumentParser( + description="Create Google Form and/or Google Sheet, and update .env file." + ) + parser.add_argument( + "--create-form", action="store_true", help="Create a Google Form" + ) + parser.add_argument( + "--create-sheet", action="store_true", help="Create a Google Sheet" + ) + parser.add_argument( + "--user-email", + type=str, + required=True, + help="Email address to share the created resources with", + ) + parser.add_argument( + "--form-title", + type=str, + default="OR Assistant Feedback Form", + help="Title for the Google Form", + ) + parser.add_argument( + "--sheet-title", + type=str, + default="OR Assistant Evaluation Sheet", + help="Title for the Google Sheet", + ) args = parser.parse_args() @@ -122,17 +144,22 @@ def main() -> None: if args.create_form: form_id = create_google_form(args.form_title, args.user_email) - print(f"Form created successfully. View form at: https://docs.google.com/forms/d/{form_id}/edit") + print( + f"Form created successfully. View form at: https://docs.google.com/forms/d/{form_id}/edit" + ) updates["GOOGLE_FORM_ID"] = form_id if args.create_sheet: sheet_id = create_google_sheet(args.sheet_title, args.user_email) - print(f"Google Sheet created successfully. View sheet at: https://docs.google.com/spreadsheets/d/{sheet_id}/edit") + print( + f"Google Sheet created successfully. View sheet at: https://docs.google.com/spreadsheets/d/{sheet_id}/edit" + ) updates["GOOGLE_SHEET_ID"] = sheet_id if updates: update_env_file(updates) print("The .env file has been updated with the new IDs.") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/evaluation/human_evaluation/utils/sheets.py b/evaluation/human_evaluation/utils/sheets.py index f3e5e4f0..979ba011 100644 --- a/evaluation/human_evaluation/utils/sheets.py +++ b/evaluation/human_evaluation/utils/sheets.py @@ -114,4 +114,4 @@ def write_responses(responses: list[str], row_numbers: list[int]) -> int: except HttpError as error: st.error("Failed to write responses to the Google Sheet.") st.error(f"An error occurred: {error}") - return 0 \ No newline at end of file + return 0 diff --git a/evaluation/human_evaluation/utils/utils.py b/evaluation/human_evaluation/utils/utils.py index 12a27def..1d373f55 100644 --- a/evaluation/human_evaluation/utils/utils.py +++ b/evaluation/human_evaluation/utils/utils.py @@ -142,7 +142,7 @@ def update_gform(questions_descriptions: list[dict[str, str]]) -> None: }, } requests.append(update_request) - else: #If update is not required, create a new question and description + else: # If update is not required, create a new question and description create_request = { "createItem": { "item": { @@ -177,4 +177,4 @@ def update_gform(questions_descriptions: list[dict[str, str]]) -> None: except HttpError as error: st.error(f"An error occurred while updating the form: {error}") except Exception as e: - st.error(f"An unexpected error occurred: {e}") \ No newline at end of file + st.error(f"An unexpected error occurred: {e}") diff --git a/frontend/streamlit_app.py b/frontend/streamlit_app.py index cd91da2d..13c70291 100644 --- a/frontend/streamlit_app.py +++ b/frontend/streamlit_app.py @@ -10,6 +10,7 @@ from dotenv import load_dotenv from typing import Callable, Any + def measure_response_time(func: Callable[..., Any]) -> Callable[..., tuple[Any, float]]: def wrapper(*args: Any, **kwargs: Any) -> tuple[Any, float]: start_time = time.time() @@ -17,8 +18,10 @@ def wrapper(*args: Any, **kwargs: Any) -> tuple[Any, float]: end_time = time.time() response_time = (end_time - start_time) * 1000 # In milliseconds return result, response_time + return wrapper + @measure_response_time def response_generator(user_input: str) -> tuple[tuple[str, str]]: """ @@ -61,6 +64,7 @@ def response_generator(user_input: str) -> tuple[tuple[str, str]]: st.error(f"Request failed: {e}") return None, None + def fetch_endpoints() -> tuple[str, list]: base_url = os.getenv("CHAT_ENDPOINT", "http://localhost:8000") url = f"{base_url}/chains/listAll" @@ -73,6 +77,7 @@ def fetch_endpoints() -> tuple[str, list]: st.error(f"Failed to fetch endpoints: {e}") return base_url, [] + def main() -> None: load_dotenv() @@ -85,20 +90,20 @@ def main() -> None: st.title("OR Assistant") base_url, endpoints = fetch_endpoints() - + selected_endpoint = st.selectbox( "Select preferred architecture", options=endpoints, index=0, - format_func=lambda x: x.split('/')[-1].capitalize() + format_func=lambda x: x.split("/")[-1].capitalize(), ) - if 'selected_endpoint' not in st.session_state: + if "selected_endpoint" not in st.session_state: st.session_state.selected_endpoint = selected_endpoint else: st.session_state.selected_endpoint = selected_endpoint - - if 'base_url' not in st.session_state: + + if "base_url" not in st.session_state: st.session_state.base_url = base_url if not os.getenv("CHAT_ENDPOINT"): @@ -130,9 +135,13 @@ def main() -> None: st.markdown(user_input) response_tuple, response_time = response_generator(user_input) - + # Validate the response tuple - if response_tuple and isinstance(response_tuple, tuple) and len(response_tuple) == 2: + if ( + response_tuple + and isinstance(response_tuple, tuple) + and len(response_tuple) == 2 + ): response, sources = response_tuple if response is not None: response_buffer = "" @@ -141,26 +150,32 @@ def main() -> None: message_placeholder = st.empty() response_buffer = "" - for chunk in response.split(' '): - response_buffer += chunk + ' ' - if chunk.endswith('\n'): - response_buffer += ' ' + for chunk in response.split(" "): + response_buffer += chunk + " " + if chunk.endswith("\n"): + response_buffer += " " message_placeholder.markdown(response_buffer) time.sleep(0.05) message_placeholder.markdown(response_buffer) - - response_time_text = f"Response Time: {response_time / 1000:.2f} seconds" - response_time_colored = f":{'green' if response_time < 5000 else 'orange' if response_time < 10000 else 'red'}[{response_time_text}]" + + response_time_text = ( + f"Response Time: {response_time / 1000:.2f} seconds" + ) + response_time_colored = f":{"green" if response_time < 5000 else "orange" if response_time < 10000 else "red"}[{response_time_text}]" st.markdown(response_time_colored) - st.session_state.chat_history.append( - {"content": response_buffer, "role": "ai"}) + st.session_state.chat_history.append({ + "content": response_buffer, + "role": "ai", + }) if sources: with st.expander("Sources:"): try: if isinstance(sources, str): - cleaned_sources = sources.replace("{", "[").replace("}", "]") + cleaned_sources = sources.replace("{", "[").replace( + "}", "]" + ) parsed_sources = ast.literal_eval(cleaned_sources) else: parsed_sources = sources @@ -193,10 +208,15 @@ def update_state() -> None: """ st.session_state.feedback_button = True - if st.button("Feedback", on_click=update_state) or st.session_state.feedback_button: + if ( + st.button("Feedback", on_click=update_state) + or st.session_state.feedback_button + ): try: show_feedback_form( - question_dict, st.session_state.metadata, st.session_state.chat_history + question_dict, + st.session_state.metadata, + st.session_state.chat_history, ) except Exception as e: st.error(f"Failed to load feedback form: {e}") diff --git a/frontend/utils/feedback.py b/frontend/utils/feedback.py index e77a98ac..5dafcd9c 100644 --- a/frontend/utils/feedback.py +++ b/frontend/utils/feedback.py @@ -8,6 +8,7 @@ load_dotenv() + def get_sheet_title_by_gid(spreadsheet_metadata: dict, gid: int) -> Optional[str]: """ Get the sheet title by Sheet GID @@ -25,6 +26,7 @@ def get_sheet_title_by_gid(spreadsheet_metadata: dict, gid: int) -> Optional[str return sheet["properties"]["title"] return None + def format_sources(sources: list[str]) -> str: """ Format the sources into a string suitable for Google Sheets. @@ -39,6 +41,7 @@ def format_sources(sources: list[str]) -> str: return "\n".join(sources) return str(sources) + def format_context(context: list[str]) -> str: """ Format the context into a string suitable for Google Sheets. @@ -53,13 +56,9 @@ def format_context(context: list[str]) -> str: return "\n".join(context) return str(context) + def submit_feedback_to_google_sheet( - question: str, - answer: str, - sources: str, - context: str, - issue: str, - version: str + question: str, answer: str, sources: str, context: str, issue: str, version: str ) -> None: """ Submit feedback to a specific Google Sheet. @@ -81,18 +80,20 @@ def submit_feedback_to_google_sheet( ) if not os.getenv("FEEDBACK_SHEET_ID"): - raise ValueError("The FEEDBACK_SHEET_ID environment variable is not set or is empty.") + raise ValueError( + "The FEEDBACK_SHEET_ID environment variable is not set or is empty." + ) if not os.getenv("RAG_VERSION"): raise ValueError("The RAG_VERSION environment variable is not set or is empty.") - SERVICE_ACCOUNT_FILE = os.getenv("GOOGLE_CREDENTIALS_JSON") - SCOPE = [ + service_account_file = os.getenv("GOOGLE_CREDENTIALS_JSON") + scope = [ "https://spreadsheets.google.com/feeds", "https://www.googleapis.com/auth/drive", ] - creds = Credentials.from_service_account_file(SERVICE_ACCOUNT_FILE, scopes=SCOPE) + creds = Credentials.from_service_account_file(service_account_file, scopes=scope) client = gspread.authorize(creds) sheet_id = os.getenv("FEEDBACK_SHEET_ID") @@ -107,9 +108,17 @@ def submit_feedback_to_google_sheet( if sheet_title: sheet = spreadsheet.worksheet(sheet_title) timestamp = datetime.now(timezone.utc).isoformat() - formatted_sources = format_sources(sources) - formatted_context = format_context(context) - data_to_append = [question, answer, formatted_sources, formatted_context, issue, timestamp, version] + formatted_sources = format_sources(sources) + formatted_context = format_context(context) + data_to_append = [ + question, + answer, + formatted_sources, + formatted_context, + issue, + timestamp, + version, + ] if not sheet.row_values(1): sheet.format("A1:G1", {"textFormat": {"bold": True}}) @@ -131,7 +140,12 @@ def submit_feedback_to_google_sheet( else: st.sidebar.error(f"Sheet with GID {target_gid} not found.") -def show_feedback_form(questions: dict[str, int], metadata: dict[str, dict[str, str]], interactions: list[dict[str, str]]) -> None: + +def show_feedback_form( + questions: dict[str, int], + metadata: dict[str, dict[str, str]], + interactions: list[dict[str, str]], +) -> None: """ Display feedback form in the sidebar. @@ -174,4 +188,4 @@ def show_feedback_form(questions: dict[str, int], metadata: dict[str, dict[str, st.session_state.submitted = True if st.session_state.submitted: - st.sidebar.success("Thank you for your feedback!") \ No newline at end of file + st.sidebar.success("Thank you for your feedback!") diff --git a/frontend/utils/mock_endpoint.py b/frontend/utils/mock_endpoint.py index 294c454a..6313afd8 100644 --- a/frontend/utils/mock_endpoint.py +++ b/frontend/utils/mock_endpoint.py @@ -1,8 +1,9 @@ from flask import Flask, request, jsonify, Response -from typing import Any +from typing import Any app = Flask(__name__) + @app.route("/chains/listAll") def list_all_chains() -> Response: """ @@ -13,6 +14,7 @@ def list_all_chains() -> Response: """ return jsonify(["/chains/mock"]) + @app.route("/chains/mock", methods=["POST"]) def chat_app() -> Response: """ @@ -31,9 +33,11 @@ def chat_app() -> Response: "sources": [ "https://mocksource1.com", "https://mocksource2.com", - "https://mocksource3.com" - ] if list_sources else [], - "context": ["This is Mock Context"] if list_context else [] + "https://mocksource3.com", + ] + if list_sources + else [], + "context": ["This is Mock Context"] if list_context else [], } return jsonify(response) diff --git a/pyproject.toml b/ruff.toml similarity index 62% rename from pyproject.toml rename to ruff.toml index ceaf2c9b..a57a63f1 100644 --- a/pyproject.toml +++ b/ruff.toml @@ -1,4 +1,3 @@ -[tool.ruff] exclude = [ ".bzr", ".direnv", @@ -33,22 +32,37 @@ indent-width = 4 target-version = "py312" -[tool.ruff.lint] +[lint] select = ["E4", "E7", "E9","E301","E304","E305","E401","E223","E224","E242", "F","N"] extend-select = ["D203", "D204"] ignore = [] preview = true +# Allow fix for all enabled rules (when `--fix` is provided). fixable = ["ALL"] unfixable = [] +# Allow unused variables when underscore-prefixed. dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" -[tool.ruff.format] +[format] +# Like Black, use double quotes for strings. quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. indent-style = "space" + +# Like Black, respect magic trailing commas. skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. line-ending = "auto" -docstring-code-format = true + +# Enable auto-formatting of code examples in docstrings. +docstring-code-format = false + +# Dynamic line length limit for formatting code snippets in docstrings. docstring-code-line-length = "dynamic" + +# Enable preview features preview = true