diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..5ace460
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,6 @@
+version: 2
+updates:
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "weekly"
diff --git a/.gitignore b/.gitignore
index 82f9275..7f0de2b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+.vscode
+
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
@@ -160,3 +162,5 @@ cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
+src/.DS_Store
+.DS_Store
diff --git a/README.md b/README.md
index 1ce8eda..0fae1e6 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,11 @@
+> [!IMPORTANT]
+> (12/19/24) Hello! MarkItDown team members will be resting and recharging with family and friends over the holiday period. Activity/responses on the project may be delayed during the period of Dec 21-Jan 06. We will be excited to engage with you in the new year!
+
# MarkItDown
[](https://pypi.org/project/markitdown/)
+
+
MarkItDown is a utility for converting various files to Markdown (e.g., for indexing, text analysis, etc).
It supports:
@@ -66,7 +71,43 @@ print(result.text_content)
docker build -t markitdown:latest .
docker run --rm -i markitdown:latest < ~/your-file.pdf > output.md
```
+
+
+Batch Processing Multiple Files
+
+This example shows how to convert multiple files to markdown format in a single run. The script processes all supported files in a directory and creates corresponding markdown files.
+
+
+```python convert.py
+from markitdown import MarkItDown
+from openai import OpenAI
+import os
+client = OpenAI(api_key="your-api-key-here")
+md = MarkItDown(llm_client=client, llm_model="gpt-4o-2024-11-20")
+supported_extensions = ('.pptx', '.docx', '.pdf', '.jpg', '.jpeg', '.png')
+files_to_convert = [f for f in os.listdir('.') if f.lower().endswith(supported_extensions)]
+for file in files_to_convert:
+ print(f"\nConverting {file}...")
+ try:
+ md_file = os.path.splitext(file)[0] + '.md'
+ result = md.convert(file)
+ with open(md_file, 'w') as f:
+ f.write(result.text_content)
+
+ print(f"Successfully converted {file} to {md_file}")
+ except Exception as e:
+ print(f"Error converting {file}: {str(e)}")
+
+print("\nAll conversions completed!")
+```
+2. Place the script in the same directory as your files
+3. Install required packages: like openai
+4. Run script ```bash python convert.py ```
+Note that original files will remain unchanged and new markdown files are created with the same base name.
+
+
+
## Contributing
This project welcomes contributions and suggestions. Most contributions require you to agree to a
@@ -81,6 +122,20 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+### How to Contribute
+
+You can help by looking at issues or helping review PRs. Any issue or PR is welcome, but we have also marked some as 'open for contribution' and 'open for reviewing' to help faciliate community contributions. These are ofcourse just suggestions and you are welcome to contribute in any way you like.
+
+
+
+
+| | All | Especially Needs Help from Community |
+|-----------------------|------------------------------------------|------------------------------------------------------------------------------------------|
+| **Issues** | [All Issues](https://github.com/microsoft/markitdown/issues) | [Issues open for contribution](https://github.com/microsoft/markitdown/issues?q=is%3Aissue+is%3Aopen+label%3A%22open+for+contribution%22) |
+| **PRs** | [All PRs](https://github.com/microsoft/markitdown/pulls) | [PRs open for reviewing](https://github.com/microsoft/markitdown/pulls?q=is%3Apr+is%3Aopen+label%3A%22open+for+reviewing%22) |
+
+
+
### Running Tests and Checks
- Install `hatch` in your environment and run tests:
diff --git a/pyproject.toml b/pyproject.toml
index c5bd58b..3e14cec 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
[project]
name = "markitdown"
dynamic = ["version"]
-description = ''
+description = 'Utility tool for converting various files to Markdown'
readme = "README.md"
requires-python = ">=3.10"
license = "MIT"
diff --git a/src/markitdown/__main__.py b/src/markitdown/__main__.py
index 3f61561..3193ae7 100644
--- a/src/markitdown/__main__.py
+++ b/src/markitdown/__main__.py
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: MIT
import sys
import argparse
+from textwrap import dedent
from ._markitdown import MarkItDown, DocumentConverterResult
@@ -10,28 +11,34 @@ def main():
parser = argparse.ArgumentParser(
description="Convert various file formats to markdown.",
formatter_class=argparse.RawDescriptionHelpFormatter,
- usage="""
-SYNTAX:
+ usage=dedent(
+ """
+ SYNTAX:
+
+ markitdown
+ If FILENAME is empty, markitdown reads from stdin.
+
+ EXAMPLE:
+
+ markitdown example.pdf
+
+ OR
+
+ cat example.pdf | markitdown
+
+ OR
+
+ markitdown < example.pdf
+
+ OR to save to a file use
- markitdown
- If FILENAME is empty, markitdown reads from stdin.
-
-EXAMPLE:
-
- markitdown example.pdf
-
- OR
-
- cat example.pdf | markitdown
-
- OR
-
- markitdown < example.pdf
-
- OR
-
- markitdown example.pdf -o example.md
-""".strip(),
+ markitdown example.pdf -o example.md
+
+ OR
+
+ markitdown example.pdf > example.md
+ """
+ ).strip(),
)
parser.add_argument("filename", nargs="?")
diff --git a/src/markitdown/_markitdown.py b/src/markitdown/_markitdown.py
index 2e7e5ff..789c1e5 100644
--- a/src/markitdown/_markitdown.py
+++ b/src/markitdown/_markitdown.py
@@ -15,6 +15,7 @@
import zipfile
from xml.dom import minidom
from typing import Any, Dict, List, Optional, Union
+from pathlib import Path
from urllib.parse import parse_qs, quote, unquote, urlparse, urlunparse
from warnings import warn, resetwarnings, catch_warnings
@@ -1133,27 +1134,33 @@ def convert(
extracted_zip_folder_name = (
f"extracted_{os.path.basename(local_path).replace('.zip', '_zip')}"
)
- new_folder = os.path.normpath(
+ extraction_dir = os.path.normpath(
os.path.join(os.path.dirname(local_path), extracted_zip_folder_name)
)
md_content = f"Content from the zip file `{os.path.basename(local_path)}`:\n\n"
- # Safety check for path traversal
- if not new_folder.startswith(os.path.dirname(local_path)):
- return DocumentConverterResult(
- title=None, text_content=f"[ERROR] Invalid zip file path: {local_path}"
- )
-
try:
- # Extract the zip file
+ # Extract the zip file safely
with zipfile.ZipFile(local_path, "r") as zipObj:
- zipObj.extractall(path=new_folder)
+ # Safeguard against path traversal
+ for member in zipObj.namelist():
+ member_path = os.path.normpath(os.path.join(extraction_dir, member))
+ if (
+ not os.path.commonprefix([extraction_dir, member_path])
+ == extraction_dir
+ ):
+ raise ValueError(
+ f"Path traversal detected in zip file: {member}"
+ )
+
+ # Extract all files safely
+ zipObj.extractall(path=extraction_dir)
# Process each extracted file
- for root, dirs, files in os.walk(new_folder):
+ for root, dirs, files in os.walk(extraction_dir):
for name in files:
file_path = os.path.join(root, name)
- relative_path = os.path.relpath(file_path, new_folder)
+ relative_path = os.path.relpath(file_path, extraction_dir)
# Get file extension
_, file_extension = os.path.splitext(name)
@@ -1177,7 +1184,7 @@ def convert(
# Clean up extracted files if specified
if kwargs.get("cleanup_extracted", True):
- shutil.rmtree(new_folder)
+ shutil.rmtree(extraction_dir)
return DocumentConverterResult(title=None, text_content=md_content.strip())
@@ -1186,6 +1193,11 @@ def convert(
title=None,
text_content=f"[ERROR] Invalid or corrupted zip file: {local_path}",
)
+ except ValueError as ve:
+ return DocumentConverterResult(
+ title=None,
+ text_content=f"[ERROR] Security error in zip file {local_path}: {str(ve)}",
+ )
except Exception as e:
return DocumentConverterResult(
title=None,
@@ -1275,11 +1287,11 @@ def __init__(
self.register_page_converter(ZipConverter())
def convert(
- self, source: Union[str, requests.Response], **kwargs: Any
+ self, source: Union[str, requests.Response, Path], **kwargs: Any
) -> DocumentConverterResult: # TODO: deal with kwargs
"""
Args:
- - source: can be a string representing a path or url, or a requests.response object
+ - source: can be a string representing a path either as string pathlib path object or url, or a requests.response object
- extension: specifies the file extension to use when interpreting the file. If None, infer from source (path, uri, content-type, etc.)
"""
@@ -1296,10 +1308,14 @@ def convert(
# Request response
elif isinstance(source, requests.Response):
return self.convert_response(source, **kwargs)
+ elif isinstance(source, Path):
+ return self.convert_local(source, **kwargs)
def convert_local(
- self, path: str, **kwargs: Any
+ self, path: Union[str, Path], **kwargs: Any
) -> DocumentConverterResult: # TODO: deal with kwargs
+ if isinstance(path, Path):
+ path = str(path)
# Prepare a list of extensions to try (in order of priority)
ext = kwargs.get("file_extension")
extensions = [ext] if ext is not None else []
diff --git a/src/markitdown/py.typed b/src/markitdown/py.typed
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_markitdown.py b/tests/test_markitdown.py
index 316e670..4a981bd 100644
--- a/tests/test_markitdown.py
+++ b/tests/test_markitdown.py
@@ -131,6 +131,17 @@
]
+# --- Helper Functions ---
+def validate_strings(result, expected_strings, exclude_strings=None):
+ """Validate presence or absence of specific strings."""
+ text_content = result.text_content.replace("\\", "")
+ for string in expected_strings:
+ assert string in text_content
+ if exclude_strings:
+ for string in exclude_strings:
+ assert string not in text_content
+
+
@pytest.mark.skipif(
skip_remote,
reason="do not run tests that query external urls",
@@ -163,73 +174,53 @@ def test_markitdown_local() -> None:
# Test XLSX processing
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.xlsx"))
- for test_string in XLSX_TEST_STRINGS:
- text_content = result.text_content.replace("\\", "")
- assert test_string in text_content
+ validate_strings(result, XLSX_TEST_STRINGS)
# Test DOCX processing
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.docx"))
- for test_string in DOCX_TEST_STRINGS:
- text_content = result.text_content.replace("\\", "")
- assert test_string in text_content
+ validate_strings(result, DOCX_TEST_STRINGS)
# Test DOCX processing, with comments
result = markitdown.convert(
os.path.join(TEST_FILES_DIR, "test_with_comment.docx"),
style_map="comment-reference => ",
)
- for test_string in DOCX_COMMENT_TEST_STRINGS:
- text_content = result.text_content.replace("\\", "")
- assert test_string in text_content
+ validate_strings(result, DOCX_COMMENT_TEST_STRINGS)
# Test DOCX processing, with comments and setting style_map on init
markitdown_with_style_map = MarkItDown(style_map="comment-reference => ")
result = markitdown_with_style_map.convert(
os.path.join(TEST_FILES_DIR, "test_with_comment.docx")
)
- for test_string in DOCX_COMMENT_TEST_STRINGS:
- text_content = result.text_content.replace("\\", "")
- assert test_string in text_content
+ validate_strings(result, DOCX_COMMENT_TEST_STRINGS)
# Test PPTX processing
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.pptx"))
- for test_string in PPTX_TEST_STRINGS:
- text_content = result.text_content.replace("\\", "")
- assert test_string in text_content
+ validate_strings(result, PPTX_TEST_STRINGS)
# Test HTML processing
result = markitdown.convert(
os.path.join(TEST_FILES_DIR, "test_blog.html"), url=BLOG_TEST_URL
)
- for test_string in BLOG_TEST_STRINGS:
- text_content = result.text_content.replace("\\", "")
- assert test_string in text_content
+ validate_strings(result, BLOG_TEST_STRINGS)
# Test ZIP file processing
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_files.zip"))
- for test_string in DOCX_TEST_STRINGS:
- text_content = result.text_content.replace("\\", "")
- assert test_string in text_content
+ validate_strings(result, XLSX_TEST_STRINGS)
# Test Wikipedia processing
result = markitdown.convert(
os.path.join(TEST_FILES_DIR, "test_wikipedia.html"), url=WIKIPEDIA_TEST_URL
)
text_content = result.text_content.replace("\\", "")
- for test_string in WIKIPEDIA_TEST_EXCLUDES:
- assert test_string not in text_content
- for test_string in WIKIPEDIA_TEST_STRINGS:
- assert test_string in text_content
+ validate_strings(result, WIKIPEDIA_TEST_STRINGS, WIKIPEDIA_TEST_EXCLUDES)
# Test Bing processing
result = markitdown.convert(
os.path.join(TEST_FILES_DIR, "test_serp.html"), url=SERP_TEST_URL
)
text_content = result.text_content.replace("\\", "")
- for test_string in SERP_TEST_EXCLUDES:
- assert test_string not in text_content
- for test_string in SERP_TEST_STRINGS:
- assert test_string in text_content
+ validate_strings(result, SERP_TEST_STRINGS, SERP_TEST_EXCLUDES)
# Test RSS processing
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_rss.xml"))
@@ -239,9 +230,7 @@ def test_markitdown_local() -> None:
## Test non-UTF-8 encoding
result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_mskanji.csv"))
- text_content = result.text_content.replace("\\", "")
- for test_string in CSV_CP932_TEST_STRINGS:
- assert test_string in text_content
+ validate_strings(result, CSV_CP932_TEST_STRINGS)
@pytest.mark.skipif(