Automatically sanitize input sent to copilot API

Removes all non-standard characters to avoid issues with copilot API
(especially claude).

Possibly related: #464

Signed-off-by: Tomas Slusny <[email protected]>

lua/CopilotChat/copilot.lua

@@ -74,6 +74,16 @@ local function machine_id()
   return hex
 end
 
+local function sanitize_text(text)
+  if not text then
+    return ''
+  end
+
+  -- Keep basic ASCII, numbers, common punctuation and whitespace
+  -- Remove everything else (e.g. special characters, emojis)
+  return text:gsub('[^%w%p%s\n]', '')
+end
+
 local function find_config_path()
   local config = vim.fn.expand('$XDG_CONFIG_HOME')
   if config and vim.fn.isdirectory(config) > 0 then
@@ -561,15 +571,17 @@ function Copilot:ask(prompt, opts)
 
         -- Generate the request
         local url = 'https://api.githubcopilot.com/chat/completions'
-        local body = vim.json.encode(
-          generate_ask_request(
-            self.history,
-            prompt,
-            embeddings_message,
-            selection_message,
-            system_prompt,
-            model,
-            temperature
+        local body = sanitize_text(
+          vim.json.encode(
+            generate_ask_request(
+              self.history,
+              prompt,
+              embeddings_message,
+              selection_message,
+              system_prompt,
+              model,
+              temperature
+            )
           )
         )
 
@@ -744,7 +756,7 @@ function Copilot:embed(inputs, opts)
 
   local jobs = {}
   for _, chunk in ipairs(chunks) do
-    local body = vim.json.encode(generate_embedding_request(chunk, model))
+    local body = sanitize_text(vim.json.encode(generate_embedding_request(chunk, model)))
 
     table.insert(jobs, function(resolve)
       local headers = generate_headers(self.token.token, self.sessionid, self.machineid)