diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fd41c8e --- /dev/null +++ b/.gitignore @@ -0,0 +1,152 @@ +.idea/ + +# Created by https://www.gitignore.io/api/pycharm + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# End of https://www.gitignore.io/api/pycharm + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +.venv/ +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fa5e297 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +help: + cat Makefile + +init: + pip install -r requirements.txt + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade build + python3 -m pip install --upgrade twine + +test: + PYTHONPATH=. pytest tests + +build: + python3 -m build + +pypi: + twine upload dist/* + +testpypi: + python3 -m twine upload --repository testpypi dist/* + + diff --git a/changelog.txt b/changelog.txt new file mode 100644 index 0000000..930a348 --- /dev/null +++ b/changelog.txt @@ -0,0 +1,2 @@ +0.0.1 (2021-07-07) +- Initial release diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..374b58c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ec57ce2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +pytest==6.2.4 +httpx==0.18.2 +rdflib==5.0.0 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..36924b7 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,25 @@ +[metadata] +name = solid-file +version = 0.0.1 +author = hrchu +author_email = petertc.chu@gmail.com +description = Solid Python library (http://solidproject.org) +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/twonote/solid-file-python +project_urls = + Bug Tracker = https://github.com/twonote/solid-file-python/issues +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: MIT License + Operating System :: OS Independent + Topic :: System :: Systems Administration + Topic :: Utilities +[options] +package_dir = + = src +packages = find: +python_requires = >=3.8 + +[options.packages.find] +where = src \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/solid/__init__.py b/src/solid/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/solid/solid_api.py b/src/solid/solid_api.py new file mode 100644 index 0000000..6b05698 --- /dev/null +++ b/src/solid/solid_api.py @@ -0,0 +1,235 @@ +from enum import Enum +from typing import Optional, Union, Dict, Callable, Iterable, AsyncIterable, List + +import httpx +from httpx import Response, HTTPStatusError + +from solid.utils.api_util import get_root_url, LINK, get_parent_url, get_item_name +from solid.utils.folder_utils import parse_folder_response + + +class MERGE(Enum): + REPLACE = 'replace' + KEEP_SOURCE = 'keep_source' + KEEP_TARGET = 'keep_target' + + +class LINKS(Enum): + EXCLUDE = 'exclude' + INCLUDE = 'include' + INCLUDE_POSSIBLE = 'include_possible' + + +class AGENT(Enum): + NO_MODIFY = 'no_modify' + TO_TARGET = 'to_target' + TO_SOURCE = 'to_source' + + +class WriteOptions: + def __init__(self, create_path: bool = True, with_acl: bool = True, agent: AGENT = AGENT.NO_MODIFY, + with_meta: bool = True, merge: MERGE = MERGE.REPLACE): + self.create_path: bool = create_path + self.with_acl: bool = with_acl + self.agent: AGENT = agent + self.with_meta: bool = with_meta + self.merge: MERGE = merge + + +class ReadFolderOptions: + def __init__(self): + self.links: LINKS = LINKS.EXCLUDE.value + + +class SolidAPIOptions: + def __init__(self): + self.enable_logging: bool = False + + +class Links: + def __init__(self): + self.acl = None + self.meta = None + + +class Item: + def __init__(self): + self.url = None + self.name = None + self.parent = None + self.itemType = None # "Container" | "Resource" + self.links: Optional[Links] = None + + +class FolderData: + def __init__(self): + self.url = None + self.name = None + self.parent = None + self.links: Links = None + self.type = 'folder' + self.folders: List[Item] = None + self.files: List[Item] = None + + +RequestContent = Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] + + +class SolidAPI: + def __init__(self, fetch: Callable[[str, Dict], Response], options=None): + pass + + def fetch(self, method, url, options: Dict = None) -> Response: + if not options: + options = {} + options['verify'] = False + + r = httpx.request(method, url, **options) + r.raise_for_status() + return r + + def get(self, url, options: Dict = None) -> Response: + return self.fetch('GET', url, options) + + def delete(self, url, options: Dict = None) -> Response: + return self.fetch('DELETE', url, options) + + def post(self, url, options: Dict = None) -> Response: + return self.fetch('POST', url, options) + + def put(self, url, options: Dict = None) -> Response: + return self.fetch('PUT', url, options) + + def patch(self, url, options: Dict = None) -> Response: + return self.fetch('PATCH', url, options) + + def head(self, url, options: Dict = None) -> Response: + return self.fetch('HEAD', url, options) + + def option(self, url, options: Dict = None) -> Response: + return self.fetch('OPTION', url, options) + + def item_exists(self, url) -> bool: + try: + self.head(url) + return True + except HTTPStatusError as e: + if e.response.status_code == 404: + return False + else: + raise e + + def post_item(self, url, content: RequestContent, content_type, link: LINK, + options: WriteOptions = WriteOptions(create_path=True)) -> Response: + parent_url = get_parent_url(url) + + if options.create_path: + self.create_folder(parent_url) + + request_options = { + 'headers': { + 'Link': link.value, + 'Slug': get_item_name(url), + 'Content-Type': content_type, + }, + 'content': content + } + + return self.post(parent_url, request_options) + + def create_folder(self, url, options: WriteOptions = WriteOptions(merge=MERGE.KEEP_TARGET)) -> Response: + if url[-1] != '/': + raise Exception(f'Cannot use createFolder to create a file : ${url}') + + try: + res = self.head(url) + if options.merge != MERGE.REPLACE: + return res + self.delete_folder(url, recursive=True) + except HTTPStatusError as e: + if e.response.status_code == 404: + pass + else: + raise e + + return self.post_item(url, '', 'text/turtle', LINK.CONTAINER, options) + + def post_file(self, url, content: RequestContent, content_type, options: WriteOptions = None) -> Response: + if url[-1] == '/': + raise Exception(f'Cannot use postFile to create a folder : ${url}') + return self.post_item(url, content, content_type, LINK.RESOURCE, options) + + def create_file(self, url, content: RequestContent, content_type, options: WriteOptions = None) -> Response: + return self.post_file(url, content, content_type, options) + + """ + files + Support upload file, get/delete in higher level api + """ + + def put_file(self, url, content: RequestContent, content_type, options: WriteOptions = WriteOptions()) -> Response: + if url[-1] == '/': + raise Exception(f'Cannot use putFile to create a folder : {url}') + + if options.merge == MERGE.KEEP_TARGET and self.item_exists(url): + raise Exception(f'File already existed: {url}') + + request_options = { + 'headers': { + 'Link': LINK.RESOURCE.value, + 'Content-Type': content_type, + }, + 'content': content + } + + return self.put(url, request_options) + + def patch_file(self, url, patch_content, patch_content_type) -> Response: + raise Exception('Not implemented') + + def read_folder(self, url, options: ReadFolderOptions = ReadFolderOptions()) -> FolderData: + if url[-1] != '/': + url += '/' + + folder_res = self.get(url, {'headers': {'Accept': 'text/turtle'}}) + parsed_folder = parse_folder_response(folder_res, url) + + if options.links in (LINKS.INCLUDE_POSSIBLE, LINKS.INCLUDE): + raise Exception('Not implemented') + + return parsed_folder + + def get_item_links(self, url, options: Dict = None) -> Response: + raise Exception('Not implemented') + + def copy_file(self, _from, to, options: WriteOptions = None) -> Response: + raise Exception('Not implemented') + + def copy_meta_file_for_item(self, old_target_file, new_target_file, options: WriteOptions = None) -> Response: + raise Exception('Not implemented') + + def copy_acl_file_for_item(self, old_target_file, new_target_file, options: WriteOptions = None) -> Response: + raise Exception('Not implemented') + + def copy_links_for_item(self, old_target_file, new_target_file, options: WriteOptions = None) -> List[Response]: + raise Exception('Not implemented') + + def copy_folder(self, _from, to, options: WriteOptions = None) -> List[Response]: + raise Exception('Not implemented') + + def copy(self, _from, to, options: WriteOptions = None) -> List[Response]: + raise Exception('Not implemented') + + def delete_folder(self, url, recursive=False) -> List[Response]: + if recursive: + raise Exception('Not implemented') + + if url == get_root_url(url): + raise Exception('405 Pod cannot be deleted') + return [self.delete(url)] + + def move(self, _from, to, copy_options: WriteOptions = None) -> List[Response]: + raise Exception('Not implemented') + + def rename(self, url, new_name, move_options: WriteOptions = None) -> List[Response]: + raise Exception('Not implemented') diff --git a/src/solid/utils/__init__.py b/src/solid/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/solid/utils/api_util.py b/src/solid/utils/api_util.py new file mode 100644 index 0000000..caefd5b --- /dev/null +++ b/src/solid/utils/api_util.py @@ -0,0 +1,61 @@ +from enum import Enum +from typing import List + + +class LINK(Enum): + CONTAINER = '; rel="type"' + RESOURCE = '; rel="type"' + + +def append_slashes_at_end(url) -> str: + if url[-1] != '/': + url += '/' + return url + + +def remove_slashes_at_end(url) -> str: + if url[-1] == '/': + url = url[:-1] + return url + + +def get_root_url(url: str) -> str: + slash_count = 0 + for i in range(len(url)): + if url[i] == '/': + slash_count += 1 + if slash_count == 3: + break + + if slash_count == 3: + return url[:i + 1] + else: + return append_slashes_at_end(url) + + +def get_parent_url(url) -> str: + url = remove_slashes_at_end(url) + + if url.count('/') == 2: # is base url, no parent url, return it self + return append_slashes_at_end(url) + + i = url.rindex('/') + return url[:i + 1] + + +def get_item_name(url) -> str: + url = remove_slashes_at_end(url) + + if url.count('/') == 2: # is base url, no item name + return '' + + i = url.rindex('/') + return url[i + 1:] + + +def are_folders(urls: List) -> bool: + pass + + +def are_files(urls: List) -> bool: + pass diff --git a/src/solid/utils/folder_utils.py b/src/solid/utils/folder_utils.py new file mode 100644 index 0000000..be8f5e7 --- /dev/null +++ b/src/solid/utils/folder_utils.py @@ -0,0 +1,53 @@ +from httpx import Response + +from rdflib import Namespace, Graph, URIRef, RDF + +from solid.utils.api_util import get_item_name, get_parent_url + +LDP = Namespace("http://www.w3.org/ns/ldp#") +container_types = [URIRef('http://www.w3.org/ns/ldp#Container'), URIRef('http://www.w3.org/ns/ldp#BasicContainer')] + +# FIXME +# def parse_folder_response(folder_response: Response, url) -> FolderData: +def parse_folder_response(folder_response: Response, url): + from solid.solid_api import FolderData, Item + + g = Graph().parse(data=folder_response.text, publicID=url, format='turtle') + + def is_type(sub, type) -> bool: + return (sub, RDF.type, type) in g + + def is_container(sub) -> bool: + for ct in container_types: + if is_type(sub, ct): + return True + return False + + this = URIRef(url) + + if not is_container(this): + raise Exception('Not a container.') + + folders, files = [], [] + + for obj in g.objects(this, LDP.contains): + item_url = str(obj) + item = Item() + item.parent = get_parent_url(item_url) + item.links = None # TODO + item.name = get_item_name(item_url) + item.url = item_url + item.itemType = 'Container' if is_container(obj) else 'Resource' + + cat = folders if is_container(obj) else files + cat.append(item) + + ret = FolderData() + ret.url = url + ret.name = get_item_name(url) + ret.parent = get_parent_url(url) + ret.links = None # TODO + ret.folders = folders + ret.files = files + + return ret diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_api_util.py b/tests/test_api_util.py new file mode 100644 index 0000000..0a3cb2c --- /dev/null +++ b/tests/test_api_util.py @@ -0,0 +1,38 @@ +from solid.utils.api_util import remove_slashes_at_end, get_root_url, get_parent_url, get_item_name + + +def test_remove_slashes_at_end(): + assert remove_slashes_at_end('http://pod.com/xxx/xxx/') == 'http://pod.com/xxx/xxx' + assert remove_slashes_at_end('http://pod.com/xxx/xxx') == 'http://pod.com/xxx/xxx' + + +def test_get_root_url(): + assert get_root_url('http://pod.com/folder/file') == 'http://pod.com/' + assert get_root_url('http://pod.com/folder/') == 'http://pod.com/' + assert get_root_url('http://pod.com/folder') == 'http://pod.com/' + assert get_root_url('http://pod.com/') == 'http://pod.com/' + assert get_root_url('http://pod.com') == 'http://pod.com/' + + +def test_get_parent_url(): + assert get_parent_url('http://pod.com/folder/file') == 'http://pod.com/folder/' + assert get_parent_url('http://pod.com/folder/') == 'http://pod.com/' + assert get_parent_url('http://pod.com/folder') == 'http://pod.com/' + assert get_parent_url('http://pod.com/') == 'http://pod.com/' + assert get_parent_url('http://pod.com') == 'http://pod.com/' + + +def test_get_item_name(): + assert get_item_name('http://pod.com/folder/file') == 'file' + assert get_item_name('http://pod.com/folder/') == 'folder' + assert get_item_name('http://pod.com/folder') == 'folder' + assert get_item_name('http://pod.com/') == '' + assert get_item_name('http://pod.com') == '' + + +def test_are_folders(): + assert False + + +def test_are_files(): + assert False diff --git a/tests/test_folder_utils.py b/tests/test_folder_utils.py new file mode 100644 index 0000000..67d2174 --- /dev/null +++ b/tests/test_folder_utils.py @@ -0,0 +1,9 @@ +from solid.utils.folder_utils import parse_folder_response +import httpx + + +def test_parse_folder_response(): + url = 'https://dahanhsi.solidcommunity.net/test/' + resp = httpx.get(url) + fold_data = parse_folder_response(resp, url) + pass diff --git a/tests/test_solid_api.py b/tests/test_solid_api.py new file mode 100644 index 0000000..7dd71e5 --- /dev/null +++ b/tests/test_solid_api.py @@ -0,0 +1,116 @@ +import io +import uuid + +import pytest +from httpx import HTTPStatusError + +from solid.solid_api import SolidAPI +from solid.utils.api_util import append_slashes_at_end + + +def gen_random_str() -> str: + return uuid.uuid4().hex + + +def test_folder(): + base_url = 'https://dahanhsi.solidcommunity.net/' + folder_name = 'testfolder-' + gen_random_str() + url = base_url + folder_name + '/' + + api = SolidAPI(None) + api.create_folder(url) + assert api.item_exists(url) + api.delete_folder(url) + + with pytest.raises(Exception): + api.delete_folder(base_url) + + +def test_read_folder(): + base_url = 'https://dahanhsi.solidcommunity.net/' + folder_name = 'testfolder' + url = base_url + folder_name + '/' + + body = '#hello Solid!' + f = io.BytesIO(body.encode('UTF-8')) + file_name = 'test.md' + file_url = url + file_name + + sub_folder_name = 'subfolder' + sub_folder_url = url + sub_folder_name + '/' + + api = SolidAPI(None) + + try: + api.delete(file_url) + except HTTPStatusError as e: + if e.response.status_code != 404: + raise e + + try: + api.delete(sub_folder_url) + except HTTPStatusError as e: + if e.response.status_code != 404: + raise e + + try: + api.delete(url) + except HTTPStatusError as e: + if e.response.status_code != 404: + raise e + + api.create_folder(url) + + # empty folder + folder_data = api.read_folder(url) + assert folder_data.name == folder_name + assert len(folder_data.folders) == 0 + assert len(folder_data.files) == 0 + assert folder_data.url == url + assert folder_data.parent == append_slashes_at_end(base_url) + assert folder_data.type == 'folder' + # assert folder_data.links == None # TODO + + # folder with subdir and file + api.create_folder(sub_folder_url) + api.put_file(file_url, f, 'text/markdown') + + folder_data = api.read_folder(url) + assert len(folder_data.folders) == 1 + assert len(folder_data.files) == 1 + file_item, sub_folder_item = folder_data.files[0], folder_data.folders[0] + + assert sub_folder_item.itemType == 'Container' + assert sub_folder_item.url == sub_folder_url + assert sub_folder_item.name == sub_folder_name + # assert sub_folder_item.links = None # TODO + assert sub_folder_item.parent == url + + assert file_item.itemType == 'Resource' + assert file_item.url == file_url + assert file_item.name == file_name + # assert file_item.links == None # TODO + assert file_item.parent == url + + +def test_file(): + url = 'https://dahanhsi.solidcommunity.net/public/test.md.' + gen_random_str() + api = SolidAPI(None) + + assert not api.item_exists(url) + with pytest.raises(HTTPStatusError) as e: + api.get(url) + assert e.value.response.status_code == 404 + + # create + body = '#hello Solid!' + f = io.BytesIO(body.encode('UTF-8')) + api.put_file(url, f, 'text/markdown') + + # retrieve + assert api.item_exists(url) + resp = api.get(url) + assert resp.text == body + + # delete + api.delete(url)