Merge pull request #94 from iamjackg/develop

Develop

.github/workflows/ci.yml

@@ -1,5 +1,5 @@
 name: Lint and test
-on: [ push ]
+on: [ push, pull_request ]
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -28,4 +28,4 @@ jobs:
           python -m pip install -r requirements-test.txt
       - name: Test with pytest
         run: |
-          pytest
+          python -m pytest

.pre-commit-config.yaml

@@ -17,9 +17,9 @@ repos:
     hooks:
       - id: isort
         args: ["--profile", "black", "--filter-files"]
-        language_version: python3.10
+        language_version: python3.11
   - repo: https://github.com/pycqa/flake8
     rev: '6.0.0'  # pick a git hash / tag to point to
     hooks:
       - id: flake8
-        language_version: python3.10
+        language_version: python3.11

CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+### Added
+- Relative links support section headers (@jimstein3d)
+- `.pages` files are now supported at the top level as well as in subdirectories (@galund)
+### Fixed
+- `--only-changed` now works for attachments (@pmarchini)
+- relative links with URL-encoded paths now work (@mschmieder)
+- folders in `.gitignore` are now actually ignored (@jmonsma)
+### Security
+- Updated `requests` to 2.31.0
+
 ## 2.1.0 - 2023-03-13
 ### Fixed
 - Moving page from top level to child page

README.md

@@ -165,6 +165,15 @@ To avoid re-uploading unchanged content and receiving update emails when there a
 
 By default, support for relative links is disabled. To enable it, pass the `--enable-relative-links` flag. The behavior of relative links is similar to [GitHub relative links](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-readmes#relative-links-and-image-paths-in-readme-files), with the exception that links starting with `/` are **not supported** and will be left unchanged.
 
+Reference to a section from another file is possible using Markdown fragment link navigation:
+` [link](./file.md#section-name) // note the dash!`
+
+In file.md:
+```
+## ...
+## section name
+```
+
 > :warning: Enabling this function requires two uploads for every page containing relative links. First, a page must be uploaded to Confluence with all internal links replaced by placeholders. Then, once the final Confluence link is known, the placeholders will be replaced with the appropriate links.
 
 By default, relative links that point to non-existent files (or files that are not being uploaded in the current batch) will result in an error. To ignore these errors and keep the links as they are, use the `--ignore-relative-link-errors` flag.

md2cf/__main__.py

@@ -584,7 +584,9 @@ def update_pages_with_relative_links(
             except KeyError:
                 if args.ignore_relative_link_errors:
                     page.body = page.body.replace(
-                        link_data.replacement, link_data.escaped_original
+                        link_data.replacement,
+                        link_data.escaped_original
+                        + (("#" + link_data.fragment) if link_data.fragment else ""),
                     )
                     continue
                 else:
@@ -598,7 +600,9 @@ def update_pages_with_relative_links(
             # anything
             if not args.dry_run:
                 page.body = page.body.replace(
-                    link_data.replacement, confluence.get_url(page_on_confluence)
+                    link_data.replacement,
+                    confluence.get_url(page_on_confluence)
+                    + (("#" + link_data.fragment) if link_data.fragment else ""),
                 )
             page_modified = True
 
@@ -706,19 +710,22 @@ def collect_pages_to_upload(args):
             if args.title:
                 only_page.title = args.title
 
-            # This is implicitly only here if relative link processing is active
+            # This is implicitly only truthy if relative link processing is active
             if only_page.relative_links:
                 # This covers the last edge case where directory processing leaves us
-                # with only one page, which we can't anticipate.
+                # with only one page, which we can't anticipate at startup time.
                 # In this case, we have to restore all the links to their original
                 # values.
                 error_console.log(
                     "Relative links are ignored when there's a single page\n"
                 )
                 for link_data in only_page.relative_links:
                     only_page.body.replace(
-                        link_data.replacement, link_data.escaped_original
+                        link_data.replacement,
+                        link_data.escaped_original
+                        + (("#" + link_data.fragment) if link_data.fragment else ""),
                     )
+                only_page.relative_links = []
 
     return pages_to_upload
 

md2cf/api.py

@@ -224,9 +224,8 @@ def update_attachment(self, confluence_page, fp, existing_attachment, message=""
         return self._post(
             f"content/{confluence_page.id}/child/attachment/{existing_attachment.id}/"
             f"data",
-            json={"comment": message} if message else None,
             headers={"X-Atlassian-Token": "nocheck"},
-            files={"file": fp},
+            files={"file": fp, "comment": message if message else None},
         )
 
     def create_attachment(self, confluence_page, fp, message=""):
@@ -235,7 +234,7 @@ def create_attachment(self, confluence_page, fp, message=""):
             json={"comment": message} if message else None,
             headers={"X-Atlassian-Token": "nocheck"},
             params={"allowDuplicated": "true"},
-            files={"file": fp},
+            files={"file": fp, "comment": message if message else None},
         )
 
     def add_labels(self, page, labels):

md2cf/confluence_renderer.py

@@ -1,13 +1,14 @@
-import urllib.parse as urlparse
 import uuid
 from pathlib import Path
 from typing import List, NamedTuple
+from urllib.parse import unquote, urlparse
 
 import mistune
 
 
 class RelativeLink(NamedTuple):
     path: str
+    fragment: str
     replacement: str
     original: str
     escaped_original: str
@@ -103,18 +104,19 @@ def plain_text_body(self, text):
         return body_tag
 
     def link(self, link, title, text):
-        parsed_link = urlparse.urlparse(link)
+        parsed_link = urlparse(link)
         if self.enable_relative_links and (
-            not parsed_link.scheme
-            and not parsed_link.netloc
-            and not parsed_link.fragment
+            not parsed_link.scheme and not parsed_link.netloc
         ):
             # relative link
             replacement_link = f"md2cf-internal-link-{uuid.uuid4()}"
             self.relative_links.append(
                 RelativeLink(
-                    path=parsed_link.path,
+                    # make sure to unquote the url as relative paths
+                    # might have escape sequences
+                    path=unquote(parsed_link.path),
                     replacement=replacement_link,
+                    fragment=parsed_link.fragment,
                     original=link,
                     escaped_original=mistune.escape_link(link),
                 )
@@ -143,7 +145,7 @@ def image(self, src, title, text):
             attributes["title"] = title
 
         root_element = ConfluenceTag(name="image", attrib=attributes)
-        parsed_source = urlparse.urlparse(src)
+        parsed_source = urlparse(src)
         if not parsed_source.netloc:
             # Local file, requires upload
             basename = Path(src).name

md2cf/document.py

@@ -111,13 +111,15 @@ def get_pages_from_directory(
     """
     processed_pages = list()
     base_path = file_path.resolve()
-    parent_page_title = None
     folder_data = dict()
     git_repo = GitRepository(file_path, use_gitignore=use_gitignore)
 
     for current_path, directories, file_names in os.walk(file_path):
         current_path = Path(current_path).resolve()
 
+        if git_repo.is_ignored(current_path):
+            continue
+
         markdown_files = [
             Path(current_path, file_name)
             for file_name in file_names
@@ -128,16 +130,16 @@ def get_pages_from_directory(
             path for path in markdown_files if not git_repo.is_ignored(path)
         ]
 
-        folder_data[current_path] = {
-            "n_files": len(markdown_files),
-            "title": current_path.name if current_path != base_path else None,
-        }
+        folder_data[current_path] = {"n_files": len(markdown_files)}
 
-        if not markdown_files and not directories:
-            continue
+        # we'll capture title and path of the parent folder for this folder:
+        folder_parent_title = None
+        folder_parent_path = None
 
-        if not markdown_files and (skip_empty or collapse_empty):
-            continue
+        # title for this folder's page (as parent of its children):
+        parent_page_title = None
+        # title for the folder (same as above except when collapsing):
+        folder_title = None
 
         if current_path != base_path:
             # TODO: add support for .pages file to read folder title
@@ -149,35 +151,41 @@ def get_pages_from_directory(
                 folder_parent_path = current_path.parent
 
             folder_parent_title = folder_data[folder_parent_path]["title"]
+            parent_page_title = current_path.name
             if len(markdown_files) == 1 and collapse_single_pages:
                 parent_page_title = folder_parent_title
+                folder_title = None
             else:
                 if collapse_empty:
-                    folder_data[current_path]["title"] = str(
+                    parent_page_title = str(
                         current_path.relative_to(folder_parent_path)
                     )
                 if beautify_folders:
-                    folder_data[current_path]["title"] = (
-                        folder_data[current_path]["title"]
-                        .replace("-", " ")
+                    parent_page_title = (
+                        current_path.name.replace("-", " ")
                         .replace("_", " ")
                         .capitalize()
                     )
-                elif use_pages_file and ".pages" in file_names:
-                    with open(current_path.joinpath(".pages")) as pages_fp:
-                        pages_file_contents = yaml.safe_load(pages_fp)
-                    if "title" in pages_file_contents:
-                        folder_data[current_path]["title"] = pages_file_contents[
-                            "title"
-                        ]
-                parent_page_title = folder_data[current_path]["title"]
-                processed_pages.append(
-                    Page(
-                        title=parent_page_title,
-                        parent_title=folder_parent_title,
-                        body="",
-                    )
+                folder_title = parent_page_title
+        if use_pages_file and ".pages" in file_names:
+            with open(current_path.joinpath(".pages")) as pages_fp:
+                pages_file_contents = yaml.safe_load(pages_fp)
+            if "title" in pages_file_contents:
+                parent_page_title = pages_file_contents["title"]
+                folder_title = parent_page_title
+
+        folder_data[current_path]["title"] = folder_title
+
+        if folder_title is not None and (
+            markdown_files or (directories and not skip_empty and not collapse_empty)
+        ):
+            processed_pages.append(
+                Page(
+                    title=folder_title,
+                    parent_title=folder_parent_title,
+                    body="",
                 )
+            )
 
         for markdown_file in markdown_files:
             processed_page = get_page_data_from_file_path(

pytest.ini

@@ -2,3 +2,4 @@
 filterwarnings =
 	# Ignore Deprecation Warning in gitignore parser until it can be fixed upstream.
 	ignore:Flags not at the start of the expression:DeprecationWarning
+pythonpath = .

setup.py

@@ -25,7 +25,7 @@
         "rich==13.0.1",
         "mistune==0.8.4",
         "chardet==5.1.0",
-        "requests==2.28.2",
+        "requests==2.31.0",
         "PyYAML==6.0",
         "gitignorefile==1.1.2",
     ],

tests/unit/test_document.py → test_package/unit/test_document.py

@@ -1,7 +1,10 @@
 from pathlib import Path
 
 import md2cf.document as doc
-from tests.utils import FakePage
+from test_package.utils import FakePage
+
+ROOT_GITIGNORE = """.git
+"""
 
 
 def test_page_get_content_hash():
@@ -31,6 +34,34 @@ def test_get_pages_from_directory(fs):
     ]
 
 
+def test_get_pages_from_directory_use_pages(fs):
+    fs.create_file("/root-folder/.gitignore", contents=ROOT_GITIGNORE)
+    fs.create_dir("/root-folder/.git")
+    fs.create_dir("/root-folder/.git/refs")
+    fs.create_file("/root-folder/.git/refs/test.md")
+    fs.create_file("/root-folder/root-folder-file.md")
+    fs.create_dir("/root-folder/empty-dir")
+    fs.create_file("/root-folder/parent/child/child-file.md")
+
+    result = doc.get_pages_from_directory(
+        Path("/root-folder"), use_pages_file=True, enable_relative_links=True
+    )
+    print(result)
+    assert result == [
+        FakePage(
+            title="root-folder-file",
+            file_path=Path("/root-folder/root-folder-file.md", parent_title=None),
+        ),
+        FakePage(title="parent", file_path=None, parent_title=None),
+        FakePage(title="child", file_path=None, parent_title="parent"),
+        FakePage(
+            title="child-file",
+            file_path=Path("/root-folder/parent/child/child-file.md"),
+            parent_title="child",
+        ),
+    ]
+
+
 def test_get_pages_from_directory_collapse_single_pages(fs):
     fs.create_file("/root-folder/root-folder-file.md")
     fs.create_file("/root-folder/parent/child/child-file.md")
@@ -173,6 +204,50 @@ def test_get_pages_from_directory_beautify_folders(fs):
     ]
 
 
+def test_get_pages_from_directory_with_pages_file_multi_level(fs):
+    fs.create_file("/root-folder/sub-folder-a/some-page.md")
+    fs.create_file("/root-folder/sub-folder-b/some-page.md")
+    fs.create_file("/root-folder/sub-folder-a/.pages", contents='title: "Folder A"')
+    fs.create_file("/root-folder/sub-folder-b/.pages", contents='title: "Folder B"')
+
+    result = doc.get_pages_from_directory(Path("/root-folder"), use_pages_file=True)
+    assert result == [
+        FakePage(
+            title="Folder A",
+        ),
+        FakePage(
+            title="some-page",
+            file_path=Path("/root-folder/sub-folder-a/some-page.md"),
+            parent_title="Folder A",
+        ),
+        FakePage(
+            title="Folder B",
+        ),
+        FakePage(
+            title="some-page",
+            file_path=Path("/root-folder/sub-folder-b/some-page.md"),
+            parent_title="Folder B",
+        ),
+    ]
+
+
+def test_get_pages_from_directory_with_pages_file_single_level(fs):
+    fs.create_file("/root-folder/some-page.md")
+    fs.create_file("/root-folder/.pages", contents='title: "Root folder"')
+
+    result = doc.get_pages_from_directory(Path("/root-folder"), use_pages_file=True)
+    assert result == [
+        FakePage(
+            title="Root folder",
+        ),
+        FakePage(
+            title="some-page",
+            file_path=Path("/root-folder/some-page.md"),
+            parent_title="Root folder",
+        ),
+    ]
+
+
 def test_get_document_frontmatter():
     source_markdown = """---
 title: This is a title