add hyphen handling

marker/providers/pdf_parsing.py

@@ -96,6 +96,7 @@ def get_lines(spans):
             ]
 
         if span["text"].endswith("\r\n") or span["text"].endswith("\x02"):
+            span["text"] = span["text"].replace("\x02", "-")
             lines.append(current_line)
             current_line = None
 

marker/schema/text/line.py

@@ -40,9 +40,15 @@ def assemble_html(self, document, child_blocks, parent_structure):
         for c in child_blocks:
             template += c.html
 
+        raw_text = remove_tags(template).strip()
         structure_idx = parent_structure.index(self.id)
-        if not structure_idx < len(parent_structure) - 1:
-            template = template.removesuffix(' ')  # strip any trailing whitespace from the last line
+        if structure_idx < len(parent_structure) - 1:
+            next_block_id = parent_structure[structure_idx + 1]
+            next_line = document.get_block(next_block_id)
+            next_line_raw_text = next_line.raw_text(document)
+            template = strip_trailing_hyphens(raw_text, next_line_raw_text, template)
+        else:
+            template = template.strip(' ')  # strip any trailing whitespace from the last line
 
         return template