attempt buffer again

rplugin/python3/copilot.py

@@ -126,12 +126,12 @@ def ask(
                     response.status_code, f"Unknown error: {response.status_code}"
                 )
             )
-        response.encoding = "utf-8"
         for line in response.iter_lines():
-            line = line.decode("utf-8").replace("data: ", "").strip()
-            if line.startswith("[DONE]"):
+            line: bytes = line
+            line = line.replace(b"data: ", b"")
+            if line.startswith(b"[DONE]"):
                 break
-            elif line == "":
+            elif line == b"":
                 continue
             try:
                 line = json.loads(line)

rplugin/python3/handlers/chat_handler.py

@@ -200,7 +200,8 @@ def _add_chat_messages(
             if self.copilot.github_token is None:
                 req = self.copilot.request_auth()
                 self.nvim.out_write(
-                    f"Please visit {req['verification_uri']} and enter the code {req['user_code']}\n"
+                    f"Please visit {req['verification_uri']
+                                    } and enter the code {req['user_code']}\n"
                 )
                 current_time = time.time()
                 wait_until = current_time + req["expires_in"]
@@ -212,30 +213,52 @@ def _add_chat_messages(
                         return
                 self.nvim.out_write("Successfully authenticated with Copilot\n")
             self.copilot.authenticate()
-        buffer = b""
-        for token in self.copilot.ask(
+
+        incomplete_token = ""  # Buffer for accumulating incomplete tokens
+
+        for token_bytes in self.copilot.ask(
             system_prompt, prompt, code, language=cast(str, file_type), model=model
         ):
-            buffer += token.encode("utf-8")
             try:
-                buffer.decode("utf-8")
+                token = token_bytes.decode('utf-8')
             except UnicodeDecodeError:
+                # Handle partial token; append to incomplete_token and continue to next iteration
+                incomplete_token += token_bytes
                 continue
-            token = buffer.decode("utf-8")
-            self.nvim.exec_lua(
-                'require("CopilotChat.utils").log_info(...)', f"Token: {token}"
-            )
+
+            # Check if there's an incomplete token buffered
+            if incomplete_token:
+                # Try to complete the token with the current chunk
+                try:
+                    completed_token = (incomplete_token + token).decode('utf-8')
+                    token = completed_token  # Update token to the completed one
+                    incomplete_token = ""  # Reset incomplete token buffer
+                except UnicodeDecodeError:
+                    # If still incomplete, append current chunk to buffer and continue
+                    incomplete_token += token_bytes
+                    continue
+
+            # Split the token into lines
+            token_lines = token.split("\n")
+
+            # Process each line in token_lines to update the Neovim buffer
             buffer_lines = cast(list[str], self.buffer.lines())
             last_line_row = len(buffer_lines) - 1
             last_line_col = len(buffer_lines[-1])
 
+            # Update the buffer with the token lines
             self.nvim.api.buf_set_text(
                 self.buffer.number,
                 last_line_row,
                 last_line_col,
                 last_line_row,
                 last_line_col,
-                token.split("\n"),
+                token_lines,
+            )
+
+            # Log each token for debugging
+            self.nvim.exec_lua(
+                'require("CopilotChat.utils").log_info(...)', f"Token: {token}"
             )
 
     def _add_end_separator(self, model: str, no_annoyance: bool = False):
@@ -252,3 +275,17 @@ def _add_end_separator(self, model: str, no_annoyance: bool = False):
             end_message = "\n" + current_datetime + "\n---\n"
 
         self.buffer.append(end_message.split("\n"))
+
+
+def is_complete_utf8_data(data):
+    """
+    Check if the given byte string ends with a complete UTF-8 character.
+    UTF-8 characters may be more than one byte long. A complete UTF-8 character
+    ends with a byte that does not start with the bits 10, which are continuation bytes.
+    """
+    if not data:
+        return False
+    # Check if the last byte is a starting byte (0xxxxxxx or 11xxxxxx)
+    return (data[-1] & 0xC0) != 0x80
+
+