diff --git a/filetracker/scripts/migrate.py b/filetracker/scripts/migrate.py index 19b20d3..4de8657 100644 --- a/filetracker/scripts/migrate.py +++ b/filetracker/scripts/migrate.py @@ -6,8 +6,9 @@ import argparse import os +import sys -from filetracker.client import Client +from filetracker.client import Client, FiletrackerError from filetracker.scripts import progress_bar # Value used for aligning printed action names @@ -15,7 +16,9 @@ _DESCRIPTION = """ -Uploads all files to a remote filetracker server. +Uploads files to a remote filetracker server. + +Use --root parameter to upload only parts of the storage. The intention for this script is to support migration to new filetracker servers that change the format of disk storage. @@ -25,15 +28,24 @@ """ -def main(): +def main(args=None): parser = argparse.ArgumentParser(description=_DESCRIPTION) - parser.add_argument('files', help='root of the file tree to be uploaded') + parser.add_argument('files', help='file tree to be uploaded') parser.add_argument('url', help='URL of the filetracker server') + parser.add_argument('--root', + help='the directory that corresponds to the storage root') parser.add_argument('-s', '--silent', action='store_true', help='if set, progress bar is not printed') - args = parser.parse_args() - root, url, silent = args.files, args.url, args.silent + args = parser.parse_args(args) + + upload_root = args.files + url = args.url + storage_root = args.root + silent = args.silent + + if storage_root is None: + storage_root = upload_root # Create a client without local cache. client = Client(local_store=None, remote_url=url) @@ -50,7 +62,7 @@ def main(): with progress_bar.conditional(show=not silent, widgets=size_widgets) as bar: - for cur_dir, _, files in os.walk(root): + for cur_dir, _, files in os.walk(upload_root): for file_name in files: total_size += os.path.getsize(os.path.join(cur_dir, file_name)) bar.update(total_size) @@ -69,10 +81,10 @@ def main(): with progress_bar.conditional(show=not silent, max_value=total_size, widgets=upload_widgets) as bar: - for cur_dir, _, files in os.walk(root): + for cur_dir, _, files in os.walk(upload_root): for file_name in files: file_path = os.path.join(cur_dir, file_name) - remote_path = '/' + os.path.relpath(file_path, root) + remote_path = '/' + os.path.relpath(file_path, storage_root) file_stat = os.stat(file_path) file_size = file_stat.st_size @@ -80,7 +92,11 @@ def main(): remote_name = '{}@{}'.format(remote_path, file_version) - client.put_file(remote_name, file_path, to_local_store=False) + try: + client.put_file(remote_name, file_path, to_local_store=False) + except FiletrackerError as e: + print('ERROR when uploading {}:\n{}'.format(file_path, e), + file=sys.stderr) processed_size += file_size bar.update(processed_size) diff --git a/filetracker/scripts/migrate_test.py b/filetracker/scripts/migrate_test.py new file mode 100644 index 0000000..3bff017 --- /dev/null +++ b/filetracker/scripts/migrate_test.py @@ -0,0 +1,69 @@ +"""Tests for migrate script.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from multiprocessing import Process +import os +import shutil +import tempfile +import time +import unittest + +from filetracker.client import Client, FiletrackerError +from filetracker.scripts import migrate +from filetracker.servers.run import main as server_main + +_TEST_PORT_NUMBER = 45785 + + +class MigrateScriptTest(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + os.makedirs(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar')) + os.makedirs(os.path.join(self.temp_dir, 'new_root')) + + self.server_process = Process( + target=_start_server, + args=(os.path.join(self.temp_dir, 'new_root'),)) + self.server_process.start() + time.sleep(2) + + self.server_url = 'http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER) + self.client = Client(local_store=None, remote_url=self.server_url) + + def tearDown(self): + self.server_process.terminate() + shutil.rmtree(self.temp_dir) + + def test_should_upload_files_with_correct_relative_root(self): + _touch(os.path.join(self.temp_dir, 'old_root', 'foo', 'a.txt')) + _touch(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar', 'b.txt')) + _touch(os.path.join(self.temp_dir, 'old_root', 'c.txt')) + _touch(os.path.join(self.temp_dir, 'old_root', 'd.txt')) + + migrate.main([ + os.path.join(self.temp_dir, 'old_root', 'foo'), + self.server_url, + '--root', + os.path.join(self.temp_dir, 'old_root'), + '-s']) + + self.assertEqual(self.client.get_stream('/foo/a.txt')[0].read(), b'') + self.assertEqual(self.client.get_stream('/foo/bar/b.txt')[0].read(), b'') + + with self.assertRaises(FiletrackerError): + self.client.get_stream('/c.txt') + + with self.assertRaises(FiletrackerError): + self.client.get_stream('/d.txt') + + +def _start_server(server_dir): + server_main(['-p', str(_TEST_PORT_NUMBER), '-d', server_dir, '-D', + '--workers', '4']) + +def _touch(path): + with open(path, 'w') as f: + pass diff --git a/filetracker/servers/base.py b/filetracker/servers/base.py index f004cee..a65d6d9 100644 --- a/filetracker/servers/base.py +++ b/filetracker/servers/base.py @@ -67,11 +67,11 @@ def get_endpoint_and_path(environ): of them are without leading slashes. """ path = environ['PATH_INFO'] - if '..' in path: - raise HttpError('400 Bad Request', 'Path cannot contain "..".') - components = path.split('/') + if '..' in components: + raise HttpError('400 Bad Request', 'Path cannot contain "..".') + # Strip closing slash if components and components[-1] == '': components.pop() diff --git a/setup.py b/setup.py index 1151a38..c4d8177 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup( name = 'filetracker', - version = '2.1.1', + version = '2.1.2', author = 'SIO2 Project Team', author_email = 'sio2@sio2project.mimuw.edu.pl', description = 'Filetracker caching file storage',