use newlines as ultimate delimiter

rplugin/python3/handlers/chat_handler.py

@@ -200,7 +200,8 @@ def _add_chat_messages(
             if self.copilot.github_token is None:
                 req = self.copilot.request_auth()
                 self.nvim.out_write(
-                    f"Please visit {req['verification_uri']} and enter the code {req['user_code']}\n"
+                    f"Please visit {req['verification_uri']
+                                    } and enter the code {req['user_code']}\n"
                 )
                 current_time = time.time()
                 wait_until = current_time + req["expires_in"]
@@ -212,34 +213,22 @@ def _add_chat_messages(
                         return
                 self.nvim.out_write("Successfully authenticated with Copilot\n")
             self.copilot.authenticate()
-        buffer = b""
+        buffer = ""
         for token in self.copilot.ask(
             system_prompt, prompt, code, language=cast(str, file_type), model=model
         ):
-            buffer += token.encode("utf-8")
-            try:
-                buffer.decode("utf-8")
-            except UnicodeDecodeError:
+            buffer += token
+            # Look for the last newline in buffer
+            last_newline = buffer.rfind("\n")
+            # If there is no newline, then continue
+            if last_newline == -1:
                 continue
-            if "\n" in buffer.decode("utf-8"):
-                token = buffer.decode("utf-8").split("\n")[0]
-                buffer = buffer[len(token) + 1:]
-                self.nvim.exec_lua(
-                    'require("CopilotChat.utils").log_info(...)', f"Token: {token}"
-                )
-                buffer_lines = cast(list[str], self.buffer.lines())
-                last_line_row = len(buffer_lines) - 1
-                last_line_col = len(buffer_lines[-1])
-
-                self.nvim.api.buf_set_text(
-                    self.buffer.number,
-                    last_line_row,
-                    last_line_col,
-                    last_line_row,
-                    last_line_col,
-                    [token],
-                )
-        if buffer:
+            # If there is a newline, take everything in front of it and set that as the token
+            token = buffer[: last_newline + 1]
+
+            self.nvim.exec_lua(
+                'require("CopilotChat.utils").log_info(...)', f"Token: {token}"
+            )
             buffer_lines = cast(list[str], self.buffer.lines())
             last_line_row = len(buffer_lines) - 1
             last_line_col = len(buffer_lines[-1])
@@ -250,9 +239,21 @@ def _add_chat_messages(
                 last_line_col,
                 last_line_row,
                 last_line_col,
-                buffer.split(b"\n"),
+                [token],
             )
 
+        buffer_lines = cast(list[str], self.buffer.lines())
+        last_line_row = len(buffer_lines) - 1
+        last_line_col = len(buffer_lines[-1])
+        self.nvim.api.buf_set_text(
+            self.buffer.number,
+            last_line_row,
+            last_line_col,
+            last_line_row,
+            last_line_col,
+            buffer.split("\n"),
+        )
+
     def _add_end_separator(self, model: str, no_annoyance: bool = False):
         current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         model_info = f"\n#### Answer provided by Copilot (Model: `{model}`) on {current_datetime}."