From b9483c9f3038fc583776d69705279372c66404ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nuno=20Gon=C3=A7alves?= Date: Mon, 25 Nov 2024 15:41:11 +0000 Subject: [PATCH] feat(fusion-sql): add support for Files API (personal and shared spaces) --- singlestoredb/fusion/handler.py | 3 + singlestoredb/fusion/handlers/files.py | 690 +++++++++++++++++++++++++ singlestoredb/fusion/handlers/utils.py | 38 ++ singlestoredb/management/files.py | 8 - 4 files changed, 731 insertions(+), 8 deletions(-) create mode 100644 singlestoredb/fusion/handlers/files.py diff --git a/singlestoredb/fusion/handler.py b/singlestoredb/fusion/handler.py index 1fcde40d..5eac9bcc 100644 --- a/singlestoredb/fusion/handler.py +++ b/singlestoredb/fusion/handler.py @@ -74,6 +74,9 @@ '': '', '': '', '': '', + '': r''' + file_type = { FILE | FOLDER } + ''', } BUILTIN_DEFAULTS = { # type: ignore diff --git a/singlestoredb/fusion/handlers/files.py b/singlestoredb/fusion/handlers/files.py new file mode 100644 index 00000000..727eb8dd --- /dev/null +++ b/singlestoredb/fusion/handlers/files.py @@ -0,0 +1,690 @@ +#!/usr/bin/env python3 +from typing import Any +from typing import Dict +from typing import Optional + +from .. import result +from ..handler import SQLHandler +from ..result import FusionSQLResult +from .utils import dt_isoformat +from .utils import get_file_space + + +class ShowFilesHandler(SQLHandler): + """ + Generic handler for listing files in a personal/shared space. + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + file_space = get_file_space(params) + + res = FusionSQLResult() + res.add_field('Name', result.STRING) + + if params['extended']: + res.add_field('Type', result.STRING) + res.add_field('Size', result.INTEGER) + res.add_field('Writable', result.STRING) + res.add_field('CreatedAt', result.DATETIME) + res.add_field('LastModifiedAt', result.DATETIME) + + files = [] + for x in file_space.listdir( + params['at_path'] or '/', + recursive=params['recursive'], + ): + info = file_space.info(x) + files.append( + tuple([ + x, info.type, info.size or 0, info.writable, + dt_isoformat(info.created_at), + dt_isoformat(info.last_modified_at), + ]), + ) + res.set_rows(files) + + else: + res.set_rows([(x,) for x in file_space.listdir( + params['at_path'] or '/', + recursive=params['recursive'], + )]) + + if params['like']: + res = res.like(Name=params['like']) + + return res.order_by(**params['order_by']).limit(params['limit']) + + +class ShowPersonalFilesHandler(ShowFilesHandler): + """ + SHOW PERSONAL FILES + [ at_path ] [ ] + [ ] + [ ] [ recursive ] [ extended ]; + + # File path to list + at_path = AT '' + + # Should the listing be recursive? + recursive = RECURSIVE + + # Should extended attributes be shown? + extended = EXTENDED + + Description + ----------- + Displays a list of files in a personal/shared space. + + Arguments + --------- + * ````: A path in the personal/shared space. + * ````: A pattern similar to SQL LIKE clause. + Uses ``%`` as the wildcard character. + + Remarks + ------- + * Use the ``LIKE`` clause to specify a pattern and return only the + files that match the specified pattern. + * The ``LIMIT`` clause limits the number of results to the + specified number. + * Use the ``ORDER BY`` clause to sort the results by the specified + key. By default, the results are sorted in the ascending order. + * The ``AT PATH`` clause specifies the path in the personal/shared + space to list the files from. + * Use the ``RECURSIVE`` clause to list the files recursively. + * To return more information about the files, use the ``EXTENDED`` + clause. + + Examples + -------- + The following command lists the files at a specific path:: + + SHOW PERSONAL FILES AT PATH "/data/"; + + The following command lists the files recursively with + additional information:: + + SHOW PERSONAL FILES RECURSIVE EXTENDED; + + See Also + -------- + * ``SHOW SHARED FILES`` + * ``UPLOAD PERSONAL FILE`` + * ``UPLOAD SHARED FILE`` + * ``DOWNLOAD PERSONAL FILE`` + * ``DOWNLOAD SHARED FILE`` + + """ # noqa: E501 + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'PERSONAL' + return super().run(params) + + +class ShowSharedFilesHandler(ShowFilesHandler): + """ + SHOW SHARED FILES + [ at_path ] [ ] + [ ] + [ ] [ recursive ] [ extended ]; + + # File path to list + at_path = AT '' + + # Should the listing be recursive? + recursive = RECURSIVE + + # Should extended attributes be shown? + extended = EXTENDED + + Description + ----------- + Displays a list of files in a personal/shared space. + + Arguments + --------- + * ````: A path in the personal/shared space. + * ````: A pattern similar to SQL LIKE clause. + Uses ``%`` as the wildcard character. + + Remarks + ------- + * Use the ``LIKE`` clause to specify a pattern and return only the + files that match the specified pattern. + * The ``LIMIT`` clause limits the number of results to the + specified number. + * Use the ``ORDER BY`` clause to sort the results by the specified + key. By default, the results are sorted in the ascending order. + * The ``AT PATH`` clause specifies the path in the personal/shared + space to list the files from. + * Use the ``RECURSIVE`` clause to list the files recursively. + * To return more information about the files, use the ``EXTENDED`` + clause. + + Examples + -------- + The following command lists the files at a specific path:: + + SHOW SHARED FILES AT PATH "/data/"; + + The following command lists the files recursively with + additional information:: + + SHOW SHARED FILES RECURSIVE EXTENDED; + + See Also + -------- + * ``SHOW PERSONAL FILES`` + * ``UPLOAD PERSONAL FILE`` + * ``UPLOAD SHARED FILE`` + * ``DOWNLOAD PERSONAL FILE`` + * ``DOWNLOAD SHARED FILE`` + + """ # noqa: E501 + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'SHARED' + return super().run(params) + + +ShowPersonalFilesHandler.register(overwrite=True) +ShowSharedFilesHandler.register(overwrite=True) + + +class UploadFileHandler(SQLHandler): + """ + Generic handler for uploading files to a personal/shared space. + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + file_space = get_file_space(params) + file_space.upload_file( + params['local_path'], params['path'], + overwrite=params['overwrite'], + ) + return None + + +class UploadPersonalFileHandler(UploadFileHandler): + """ + UPLOAD PERSONAL FILE TO path + FROM local_path [ overwrite ]; + + # Path to file + path = '' + + # Path to local file + local_path = '' + + # Should an existing file be overwritten? + overwrite = OVERWRITE + + Description + ----------- + Uploads a file to a personal/shared space. + + Arguments + --------- + * ````: The path in the personal/shared space where the file is uploaded. + * ````: The path to the file to upload in the local + directory. + + Remarks + ------- + * If the ``OVERWRITE`` clause is specified, any existing file at the + specified path in the personal/shared space is overwritten. + + Examples + -------- + The following command uploads a file to a personal/shared space and overwrite any + existing files at the specified path:: + + UPLOAD PERSONAL FILE TO '/data/stats.csv' + FROM '/tmp/user/stats.csv' OVERWRITE; + + See Also + -------- + * ``UPLOAD SHARED FILE`` + * ``DOWNLOAD PERSONAL FILE`` + * ``DOWNLOAD SHARED FILE`` + + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'PERSONAL' + return super().run(params) + + +class UploadSharedFileHandler(UploadFileHandler): + """ + UPLOAD SHARED FILE TO path + FROM local_path [ overwrite ]; + + # Path to file + path = '' + + # Path to local file + local_path = '' + + # Should an existing file be overwritten? + overwrite = OVERWRITE + + Description + ----------- + Uploads a file to a personal/shared space. + + Arguments + --------- + * ````: The path in the personal/shared space where the file is uploaded. + * ````: The path to the file to upload in the local + directory. + + Remarks + ------- + * If the ``OVERWRITE`` clause is specified, any existing file at the + specified path in the personal/shared space is overwritten. + + Examples + -------- + The following command uploads a file to a personal/shared space and overwrite any + existing files at the specified path:: + + UPLOAD SHARED FILE TO '/data/stats.csv' + FROM '/tmp/user/stats.csv' OVERWRITE; + + See Also + -------- + * ``UPLOAD PERSONAL FILE`` + * ``DOWNLOAD PERSONAL FILE`` + * ``DOWNLOAD SHARED FILE`` + + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'SHARED' + return super().run(params) + + +UploadPersonalFileHandler.register(overwrite=True) +UploadSharedFileHandler.register(overwrite=True) + + +class DownloadFileHandler(SQLHandler): + """ + Generic handler for downloading files from a personal/shared space. + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + file_space = get_file_space(params) + + out = file_space.download_file( + params['path'], + local_path=params['local_path'] or None, + overwrite=params['overwrite'], + encoding=params['encoding'] or None, + ) + + if not params['local_path']: + res = FusionSQLResult() + if params['encoding']: + res.add_field('Data', result.STRING) + else: + res.add_field('Data', result.BLOB) + res.set_rows([(out,)]) + return res + + return None + + +class DownloadPersonalFileHandler(DownloadFileHandler): + """ + DOWNLOAD PERSONAL FILE path + [ local_path ] + [ overwrite ] + [ encoding ]; + + # Path to file + path = '' + + # Path to local file + local_path = TO '' + + # Should an existing file be overwritten? + overwrite = OVERWRITE + + # File encoding + encoding = ENCODING '' + + Description + ----------- + Download a file from a personal/shared space. + + Arguments + --------- + * ````: The path to the file to download in a personal/shared space. + * ````: The encoding to apply to the downloaded file. + * ````: Specifies the path in the local directory + where the file is downloaded. + + Remarks + ------- + * If the ``OVERWRITE`` clause is specified, any existing file at + the download location is overwritten. + * By default, files are downloaded in binary encoding. To view + the contents of the file on the standard output, use the + ``ENCODING`` clause and specify an encoding. + * If ```` is not specified, the file is displayed + on the standard output. + + Examples + -------- + The following command displays the contents of the file on the + standard output:: + + DOWNLOAD PERSONAL FILE '/data/stats.csv' ENCODING 'utf8'; + + The following command downloads a file to a specific location and + overwrites any existing file with the name ``stats.csv`` on the local storage:: + + DOWNLOAD PERSONAL FILE '/data/stats.csv' + TO '/tmp/data.csv' OVERWRITE; + + See Also + -------- + * ``DOWNLOAD SHARED FILE`` + * ``UPLOAD PERSONAL FILE`` + * ``UPLOAD SHARED FILE`` + + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'PERSONAL' + return super().run(params) + + +class DownloadSharedFileHandler(DownloadFileHandler): + """ + DOWNLOAD SHARED FILE path + [ local_path ] + [ overwrite ] + [ encoding ]; + + # Path to file + path = '' + + # Path to local file + local_path = TO '' + + # Should an existing file be overwritten? + overwrite = OVERWRITE + + # File encoding + encoding = ENCODING '' + + Description + ----------- + Download a file from a personal/shared space. + + Arguments + --------- + * ````: The path to the file to download in a personal/shared space. + * ````: The encoding to apply to the downloaded file. + * ````: Specifies the path in the local directory + where the file is downloaded. + + Remarks + ------- + * If the ``OVERWRITE`` clause is specified, any existing file at + the download location is overwritten. + * By default, files are downloaded in binary encoding. To view + the contents of the file on the standard output, use the + ``ENCODING`` clause and specify an encoding. + * If ```` is not specified, the file is displayed + on the standard output. + + Examples + -------- + The following command displays the contents of the file on the + standard output:: + + DOWNLOAD SHARED FILE '/data/stats.csv' ENCODING 'utf8'; + + The following command downloads a file to a specific location and + overwrites any existing file with the name ``stats.csv`` on the local storage:: + + DOWNLOAD SHARED FILE '/data/stats.csv' + TO '/tmp/data.csv' OVERWRITE; + + See Also + -------- + * ``DOWNLOAD PERSONAL FILE`` + * ``UPLOAD PERSONAL FILE`` + * ``UPLOAD SHARED FILE`` + + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'SHARED' + return super().run(params) + + +DownloadPersonalFileHandler.register(overwrite=True) +DownloadSharedFileHandler.register(overwrite=True) + + +class DropHandler(SQLHandler): + """ + Generic handler for deleting files/folders from a personal/shared space. + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + file_space = get_file_space(params) + + file_type = params['file_type'] + if not file_type: + raise KeyError('file type was not specified') + + file_type = file_type.lower() + if file_type not in ['file', 'folder']: + raise ValueError('file type must be either FILE or FOLDER') + + if file_type == 'file': + file_space.remove(params['path']) + elif file_type == 'folder': + if params['recursive']: + file_space.removedirs(params['path']) + else: + file_space.rmdir(params['path']) + + return None + + +class DropPersonalHandler(DropHandler): + """ + DROP PERSONAL path + [ recursive ]; + + # Path to file + path = '' + + # Should folders be deleted recursively? + recursive = RECURSIVE + + Description + ----------- + Deletes a file/folder from a personal/shared space. + + Arguments + --------- + * ````: The type of the file, it can + be either 'FILE' or 'FOLDER'. + * ````: The path to the file to delete in a personal/shared space. + + Remarks + ------- + * The ``RECURSIVE`` clause indicates that the specified folder + is deleted recursively. + + Example + -------- + The following commands delete a file/folder from a personal/shared space:: + + DROP PERSONAL FILE '/data/stats.csv'; + DROP PERSONAL FOLDER '/data/' RECURSIVE; + + See Also + -------- + * ``DROP SHARED FILE`` + * ``DROP SHARED FOLDER`` + + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'PERSONAL' + return super().run(params) + + +class DropSharedHandler(DropHandler): + """ + DROP SHARED path + [ recursive ]; + + # Path to file + path = '' + + # Should folders be deleted recursively? + recursive = RECURSIVE + + Description + ----------- + Deletes a file/folder from a personal/shared space. + + Arguments + --------- + * ````: The type of the file, it can + be either 'FILE' or 'FOLDER'. + * ````: The path to the file to delete in a personal/shared space. + + Remarks + ------- + * The ``RECURSIVE`` clause indicates that the specified folder + is deleted recursively. + + Example + -------- + The following commands delete a file/folder from a personal/shared space:: + + DROP SHARED FILE '/data/stats.csv'; + DROP SHARED FOLDER '/data/' RECURSIVE; + + See Also + -------- + * ``DROP PERSONAL FILE`` + * ``DROP PERSONAL FOLDER`` + + """ # noqa: E501 + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'SHARED' + return super().run(params) + + +DropPersonalHandler.register(overwrite=True) +DropSharedHandler.register(overwrite=True) + + +class CreateFolderHandler(SQLHandler): + """ + Generic handler for creating folders in a personal/shared space. + """ + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + file_space = get_file_space(params) + file_space.mkdir(params['path'], overwrite=params['overwrite']) + return None + + +class CreatePersonalFolderHandler(CreateFolderHandler): + """ + CREATE PERSONAL FOLDER path + [ overwrite ]; + + # Path to folder + path = '' + + # Should an existing folder be overwritten? + overwrite = OVERWRITE + + Description + ----------- + Creates a new folder at the specified path in a personal/shared space. + + Arguments + --------- + * ````: The path in a personal/shared space where the folder + is created. The path must end with a trailing slash (/). + + Remarks + ------- + * If the ``OVERWRITE`` clause is specified, any existing + folder at the specified path is overwritten. + + Example + ------- + The following command creates a folder in a personal/shared space:: + + CREATE PERSONAL FOLDER `/data/csv/`; + + See Also + -------- + * ``CREATE SHARED FOLDER`` + + """ + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'PERSONAL' + return super().run(params) + + +class CreateSharedFolderHandler(CreateFolderHandler): + """ + CREATE SHARED FOLDER path + [ overwrite ]; + + # Path to folder + path = '' + + # Should an existing folder be overwritten? + overwrite = OVERWRITE + + Description + ----------- + Creates a new folder at the specified path in a personal/shared space. + + Arguments + --------- + * ````: The path in a personal/shared space where the folder + is created. The path must end with a trailing slash (/). + + Remarks + ------- + * If the ``OVERWRITE`` clause is specified, any existing + folder at the specified path is overwritten. + + Example + ------- + The following command creates a folder in a personal/shared space:: + + CREATE SHARED FOLDER `/data/csv/`; + + See Also + -------- + * ``CREATE PERSONAL FOLDER`` + + """ + + def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]: + params['file_location'] = 'SHARED' + return super().run(params) + + +CreatePersonalFolderHandler.register(overwrite=True) +CreateSharedFolderHandler.register(overwrite=True) diff --git a/singlestoredb/fusion/handlers/utils.py b/singlestoredb/fusion/handlers/utils.py index 27ffc177..a3df6a74 100644 --- a/singlestoredb/fusion/handlers/utils.py +++ b/singlestoredb/fusion/handlers/utils.py @@ -8,6 +8,11 @@ from ...exceptions import ManagementError from ...management import manage_workspaces +from ...management.files import FilesManager +from ...management.files import FileSpace +from ...management.files import manage_files +from ...management.files import PERSONAL_SPACE +from ...management.files import SHARED_SPACE from ...management.workspace import StarterWorkspace from ...management.workspace import Workspace from ...management.workspace import WorkspaceGroup @@ -19,6 +24,11 @@ def get_workspace_manager() -> WorkspaceManager: return manage_workspaces() +def get_files_manager() -> FilesManager: + """Return a new files manager.""" + return manage_files() + + def dt_isoformat(dt: Optional[datetime.datetime]) -> Optional[str]: """Convert datetime to string.""" if dt is None: @@ -270,3 +280,31 @@ def get_deployment( raise raise KeyError('no deployment was specified') + + +def get_file_space(params: Dict[str, Any]) -> FileSpace: + """ + Retrieve the specified file space. + + This function will get a file space from the + following parameters: + + * params['file_location'] + """ + manager = get_files_manager() + + file_location = params.get('file_location') + if file_location: + file_location_lower_case = file_location.lower() + if ( + file_location_lower_case != PERSONAL_SPACE and + file_location_lower_case != SHARED_SPACE + ): + raise ValueError(f'invalid file location: {file_location}') + + if file_location_lower_case == PERSONAL_SPACE: + return manager.personal_space + elif file_location_lower_case == SHARED_SPACE: + return manager.shared_space + + raise KeyError('no file space was specified') diff --git a/singlestoredb/management/files.py b/singlestoredb/management/files.py index 48ea7e5a..682e1e81 100644 --- a/singlestoredb/management/files.py +++ b/singlestoredb/management/files.py @@ -653,7 +653,6 @@ def upload_file( return self._upload(local_path, path, overwrite=overwrite) return self._upload(open(local_path, 'rb'), path, overwrite=overwrite) - # TODO: remove from FileLocation? def upload_folder( self, local_path: PathLike, @@ -725,7 +724,6 @@ def _upload( return self.info(path) - # TODO: remove from FileLocation? def mkdir(self, path: PathLike, overwrite: bool = False) -> FilesObject: """ Make a directory in the file space. @@ -834,7 +832,6 @@ def exists(self, path: PathLike) -> bool: return False raise - # TODO: remove from FileLocation? def is_dir(self, path: PathLike) -> bool: """ Is the given file path a directory? @@ -856,7 +853,6 @@ def is_dir(self, path: PathLike) -> bool: return False raise - # TODO: remove from FileLocation? def is_file(self, path: PathLike) -> bool: """ Is the given file path a file? @@ -889,7 +885,6 @@ def _list_root_dir(self) -> List[str]: ).json() return [x['path'] for x in res['content'] or []] - # TODO: remove from FileLocation? def listdir( self, path: PathLike = '/', @@ -964,7 +959,6 @@ def download_file( return out - # TODO: remove from FileLocation? def download_folder( self, path: PathLike, @@ -1005,7 +999,6 @@ def remove(self, path: PathLike) -> None: self._manager._delete(f'files/fs/{self._location}/{path}') - # TODO: remove from FileLocation? def removedirs(self, path: PathLike) -> None: """ Delete a folder recursively. @@ -1021,7 +1014,6 @@ def removedirs(self, path: PathLike) -> None: 'in Files API', ) - # TODO: remove from FileLocation? def rmdir(self, path: PathLike) -> None: """ Delete a folder.