From 12708a5d1515b9e7f5669e1c23b1817ab9a7b7e2 Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Tue, 27 Aug 2019 10:20:04 +0100 Subject: [PATCH 1/8] Implement .rsync-ignore --- README.rst | 6 ++++++ rshell/main.py | 22 +++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index db114dc..07b1716 100644 --- a/README.rst +++ b/README.rst @@ -535,6 +535,12 @@ Synchronisation is performed by comparing the date and time of source and destination files. Files are copied if the source is newer than the destination. +Synchronisation can be configured to ignore files such as documents, +usually to conserve space on the destination. This is done by means +of a file named .rshell-ignore. This should comprise a list of files +and/or subdirectories with each item on a separate line. If such a +file is found in a source directory, items found in the file's +directory that match its contents will not be synchronised. shell ----- diff --git a/rshell/main.py b/rshell/main.py index 5bb5b7a..3a3a80c 100755 --- a/rshell/main.py +++ b/rshell/main.py @@ -147,6 +147,8 @@ RTS = '' DTR = '' +IGFILE_NAME = '.rshell-ignore' + # It turns out that just because pyudev is installed doesn't mean that # it can actually be used. So we only bother to try if we're running # under linux. @@ -875,6 +877,21 @@ def rsync(src_dir, dst_dir, mirror, dry_run, print_func, recursed, sync_hidden): for name, stat in src_files: d_src[name] = stat + # Check source for an ignore file + all_src = auto(listdir_stat, src_dir, show_hidden=True) + igfiles = [x for x in all_src if x[0] == IGFILE_NAME] + set_ignore = set() + if len(igfiles): + igfile, mode = igfiles[0] + if mode_isfile(stat_mode(mode)): + with open(src_dir + '/' + igfile, 'r') as f: + for line in f.readlines(): + line = line.strip() + if line: + set_ignore.add(line) + else: + print_err('Ignore file "{:s}" is not a file'.format(IGFILE_NAME)) + d_dst = {} dst_files = auto(listdir_stat, dst_dir, show_hidden=sync_hidden) if dst_files is None: # Directory does not exist @@ -885,7 +902,7 @@ def rsync(src_dir, dst_dir, mirror, dry_run, print_func, recursed, sync_hidden): d_dst[name] = stat set_dst = set(d_dst.keys()) - set_src = set(d_src.keys()) + set_src = set(d_src.keys()) - set_ignore to_add = set_src - set_dst # Files to copy to dest to_del = set_dst - set_src # To delete from dest to_upd = set_dst.intersection(set_src) # In both: may need updating @@ -2647,6 +2664,9 @@ def do_shell(self, line): ), ) + def complete_rsync(self, text, line, begidx, endidx): + return self.filename_complete(text, line, begidx, endidx) + def do_rsync(self, line): """rsync [-m|--mirror] [-n|--dry-run] [-q|--quiet] SRC_DIR DEST_DIR From 8490560479a226e2237babb646538c99a15003dc Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Mon, 2 Sep 2019 16:56:47 +0100 Subject: [PATCH 2/8] 1st pass at macro functionality. --- README.rst | 114 ++++++++++++++++++++++++++++++++++++++++--------- rshell/main.py | 110 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 19 deletions(-) diff --git a/README.rst b/README.rst index 07b1716..2d7c2e9 100644 --- a/README.rst +++ b/README.rst @@ -95,34 +95,39 @@ following displayed: :: - usage: rshell [options] [command] +usage: rshell [options] [command] Remote Shell for a MicroPython board. positional arguments: - cmd Optional command to execute + cmd Optional command to execute optional arguments: - -h, --help show this help message and exit - -b BAUD, --baud BAUD Set the baudrate used (default = 115200) - --buffer-size BUFFER_SIZE - Set the buffer size used for transfers (default = 512) - -p PORT, --port PORT Set the serial port to use (default '/dev/ttyACM0') - --rts RTS Set the RTS state (default '') - --dtr DTR Set the DTR state (default '') - -u USER, --user USER Set username to use (default 'micro') - -w PASSWORD, --password PASSWORD + -h, --help show this help message and exit + -b BAUD, --baud BAUD Set the baudrate used (default = 115200) + --buffer-size BUFFER_SIZE + Set the buffer size used for transfers (default = 512 + for USB, 32 for UART) + -p PORT, --port PORT Set the serial port to use (default 'None') + --rts RTS Set the RTS state (default '') + --dtr DTR Set the DTR state (default '') + -u USER, --user USER Set username to use (default 'micro') + -w PASSWORD, --password PASSWORD Set password to use (default 'python') - -e EDITOR, --editor EDITOR + -e EDITOR, --editor EDITOR Set the editor to use (default 'vi') - -f FILENAME, --file FILENAME + -f FILENAME, --file FILENAME Specifies a file of commands to process. - -d, --debug Enable debug features - -n, --nocolor Turn off colorized output - --wait How long to wait for serial port - --binary Enable binary file transfer - --timing Print timing information about each command - --quiet Turns off some output (useful for testing) + -m MACRO_MODULE, --macros MACRO_MODULE + Specify a macro module. + -d, --debug Enable debug features + -n, --nocolor Turn off colorized output + -l, --list Display serial ports + -a, --ascii ASCII encode binary files for transfer + --wait WAIT Seconds to wait for serial port + --timing Print timing information about each command + -V, --version Reports the version and exits. + --quiet Turns off some output (useful for testing) You can specify the default serial port using the RSHELL_PORT environment variable. @@ -163,6 +168,12 @@ be used. Specifies a file of rshell commands to process. This allows you to create a script which executes any valid rshell commands. +-m MACRO_MODULE, --macros MACRO_MODULE +-------------------------------------- + +Specifies a Python module containing macros which may be expanded at +the rshell prompt. See below for the file format and its usage. + -n, --nocolor ------------- @@ -560,6 +571,71 @@ This will invoke a command, and return back to rshell. Example: will flash the pyboard. +lm +-- + +:: + + usage lm [macro_name] + + If issued without an arg lists available macros, otheriwse lists the + specified macro. + +m +- + +:: + + usage m macro_name [arg0 [arg1 [args...]]] + + Expands the named macro, passing it any supplied positional args, + and executes it. + +Macros +====== + +Macros enable short strings to be expanded into longer ones and enable +common names to be used to similar or different effect across multiple +projects. + +If rshell is invoked with -m MACRO_MODULE argument the specified Python +module will be imported (assuming it is on the Python path). + +The module should contain a dict named macros. Each key should be a string +specifying the name; the value may be a string (being the expansion) or a +2-tuple. In the case of a tuple the expansion will element[0], with +element[1] being an arbitrary help string. + +The macro name and expansion string may not contain whitespace. The +expansion string may contain argument specifiers compatible with the +Python string format operator. Consider this module: + +:: + + from global_rshell_macros import macros # Common across projects + # Macros specific to the foo project + macros['sync'] = 'rsync foo/ /flash/foo/', 'Sync foo project' + macros['run'] = 'repl ~ import foo.demos.{}', 'Run foo demo e.g. > m run hst' + macros['proj'] = 'ls -l /flash/foo/{}', 'List directory in foo project.' + macros['cpf'] = 'cp foo/py/{} /flash/foo/py/; repl ~ import foo.demos.{}', 'Copy a py file, run a demo' + macros['cpd'] = 'cp foo/demos/{0}.py /flash/foo/demos/; repl ~ import foo.demos.{0}', 'Copy a demo file and run it' + +If at the rshell prompt we issue + +:: + + > m cpd hst + +this will expand to + +:: + + > cp foo/demos/hst.py /flash/foo/demos/; repl ~ import foo.demos.hst + +If no args are passed, rshell will expand the macro with argument specifiers +removed. This enables macros such as 'proj' above to run with zero or one +argument. + Pattern Matching ================ diff --git a/rshell/main.py b/rshell/main.py index 3a3a80c..af6eb88 100755 --- a/rshell/main.py +++ b/rshell/main.py @@ -50,9 +50,13 @@ import shlex import itertools from serial.tools import list_ports +import importlib import traceback +# Macros: values are strings or 2-lists +macros = {} + if sys.platform == 'win32': EXIT_STR = 'Use the exit command to exit rshell.' else: @@ -2036,6 +2040,72 @@ def do_boards(self, _): else: print('No boards connected') + def do_m(self, line): + """m macro_name [[arg0] arg1]... + + Expand a macro with args and run. + """ + msg = '''usage m MACRO [[[arg0] arg1] ...] + Run macro MACRO with any required arguments.''' + tokens = [x for x in line.split(' ') if x] + cmd = tokens[0] + if cmd in macros: + data = macros[cmd] + if isinstance(data, str): + go = data + else: # List or tuple: discard help + go = data[0] + if len(tokens) > 1: # Args to process + try: + to_run = go.format(*tokens[1:]) + except: + print_err('Macro {} is incompatible with args {}'.format(cmd, tokens[1:])) + return + else: + to_run = go.format('') + self.print(to_run) + self.onecmd(to_run) + elif cmd == '-h' or l == '--help': + self.print(msg) + else: + print_err('Unknown macro', cmd) + + def do_lm(self, line): + """lm + + Lists available macros. + """ + msg = '''usage lm [MACRO] + list loaded macros. + Positional argument + MACRO the name of a single macro to list.''' + if not macros: + print_err('No macros loaded.') + return + def add_col(l): + d = macros[l] + sp = '' + if isinstance(d, str): + cols.append((l, sp, d, '', '')) + else: + cols.append((l, sp, d[0], sp, d[1])) + + l = line.strip() + cols = [] + if l: + if l in macros: + add_col(l) + elif l == '-h' or l == '--help': + self.print(msg) + return + else: + print_err('Unknown macro {}'.format(l)) + return + else: + for l in macros: + add_col(l) + column_print('<<<<<', cols, self.print) + def complete_cat(self, text, line, begidx, endidx): return self.filename_complete(text, line, begidx, endidx) @@ -2680,6 +2750,37 @@ def do_rsync(self, line): rsync(src_dir, dst_dir, mirror=args.mirror, dry_run=args.dry_run, print_func=pf, recursed=False, sync_hidden=args.all) +def load_macros(mod_name): + """Update the global macros dict. + Validate on import to avoid runtime errors as far as possible. + """ + try: + mmod = importlib.import_module(mod_name) + except ImportError: + print("Can't import macro module", mod_name) + return False + except: + print("Macro module {} is invalid".format(mod_name)) + return False + + if hasattr(mmod, 'macros') and isinstance(mmod.macros, dict): + md = mmod.macros + else: + print('Macro module {} has missing or invalid dict.'.format(mod_name)) + return False + for k, v in md.items(): + if isinstance(v, str): + s = v + elif isinstance(v, tuple) or isinstance(v, list): + s = v[0] + else: + print('Macro {} is invalid.'.format(k)) + return False + if '\n' in s: + print('Invalid multi-line macro {} {}'.format(k, s)) + return False + macros.update(md) + return True def real_main(): """The main program.""" @@ -2767,6 +2868,11 @@ def real_main(): dest="filename", help="Specifies a file of commands to process." ) + parser.add_argument( + "-m", "--macros", + dest="macro_module", + help="Specify a macro module." + ) parser.add_argument( "-d", "--debug", dest="debug", @@ -2874,6 +2980,10 @@ def real_main(): global FAKE_INPUT_PROMPT FAKE_INPUT_PROMPT = True + if args.macro_module: # Attempt to load a macro module + if load_macros(args.macro_module): + print('Macro file {} loaded OK.'.format(args.macro_module)) + global ASCII_XFER ASCII_XFER = args.ascii_xfer RTS = args.rts From b15eb0e0a6bf89ed51c5c9703d3112f4072334b7 Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Tue, 3 Sep 2019 07:13:37 +0100 Subject: [PATCH 3/8] Improve doc for macros. --- README.rst | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 2d7c2e9..e1a3d88 100644 --- a/README.rst +++ b/README.rst @@ -596,7 +596,8 @@ Macros Macros enable short strings to be expanded into longer ones and enable common names to be used to similar or different effect across multiple -projects. +projects. They also enable rshell functionality to be enhanced, e.g. +adding an mv command to move files. If rshell is invoked with -m MACRO_MODULE argument the specified Python module will be imported (assuming it is on the Python path). @@ -606,14 +607,32 @@ specifying the name; the value may be a string (being the expansion) or a 2-tuple. In the case of a tuple the expansion will element[0], with element[1] being an arbitrary help string. -The macro name and expansion string may not contain whitespace. The -expansion string may contain argument specifiers compatible with the -Python string format operator. Consider this module: +The macro name and expansion string may not contain whitespace. Multi-line +expansions are supported by virtue of rshell's ; operator: see the mv +macro below. + +The expansion string may contain argument specifiers compatible with the +Python string format operator. This enables arguments passed to the macro +to be expanded in ways which are highly flexible. + +Consider a global macro module: + +:: + + macros = {} + macros['..'] = 'cd ..' + macros['...'] = 'cd ../..' + macros['ll'] = 'ls -al {}', 'List a directory (default current one)' + macros['lf'] = 'ls -al /flash/{}', 'List contents of target flash' + macros['lsd'] = 'ls -al /sd/{}' + macros['lpb'] = 'ls -al /pyboard/{}' + macros['mv'] = 'cp {0} {1}; rm {0}', 'File move command' + +A module specific to the foo project: :: - from global_rshell_macros import macros # Common across projects - # Macros specific to the foo project + from global_rshell_macros import macros macros['sync'] = 'rsync foo/ /flash/foo/', 'Sync foo project' macros['run'] = 'repl ~ import foo.demos.{}', 'Run foo demo e.g. > m run hst' macros['proj'] = 'ls -l /flash/foo/{}', 'List directory in foo project.' From 421959374ce380569b945ec133a974d984b08a3a Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Tue, 3 Sep 2019 07:16:21 +0100 Subject: [PATCH 4/8] Improve doc for macros. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e1a3d88..994c072 100644 --- a/README.rst +++ b/README.rst @@ -95,7 +95,7 @@ following displayed: :: -usage: rshell [options] [command] + usage: rshell [options] [command] Remote Shell for a MicroPython board. From 6f05f246af4f350c20e142b1f49ee664757fa7f1 Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Tue, 3 Sep 2019 15:40:19 +0100 Subject: [PATCH 5/8] Filename completion for macro expansion. --- rshell/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rshell/main.py b/rshell/main.py index af6eb88..ab000d9 100755 --- a/rshell/main.py +++ b/rshell/main.py @@ -2040,6 +2040,9 @@ def do_boards(self, _): else: print('No boards connected') + def complete_m(self, text, line, begidx, endidx): # Assume macro works on filenames for completion. + return self.filename_complete(text, line, begidx, endidx) + def do_m(self, line): """m macro_name [[arg0] arg1]... From f883d0ace51f6801d213b46de6b09331d13d090e Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Tue, 3 Sep 2019 16:26:46 +0100 Subject: [PATCH 6/8] Fix bug with m --help. Improve README. --- README.rst | 14 ++++++++------ rshell/main.py | 7 +++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 994c072..a0551e9 100644 --- a/README.rst +++ b/README.rst @@ -607,9 +607,9 @@ specifying the name; the value may be a string (being the expansion) or a 2-tuple. In the case of a tuple the expansion will element[0], with element[1] being an arbitrary help string. -The macro name and expansion string may not contain whitespace. Multi-line -expansions are supported by virtue of rshell's ; operator: see the mv -macro below. +The macro name must conform to Python rules for dict keys. The expansion +string may not contain newline characters. Multi-line expansions are +supported by virtue of rshell's ; operator: see the mv macro below. The expansion string may contain argument specifiers compatible with the Python string format operator. This enables arguments passed to the macro @@ -651,9 +651,11 @@ this will expand to > cp foo/demos/hst.py /flash/foo/demos/; repl ~ import foo.demos.hst -If no args are passed, rshell will expand the macro with argument specifiers -removed. This enables macros such as 'proj' above to run with zero or one -argument. +In general args should be regarded as mandatory. Any excess args supplied +will be ignored. In the case where no args are passed to a macro that +expects some, the macro will be expanded and run with each placeholder +replaced with an empty string. This enables directory listing macros such as +'proj' above to run with zero or one argument. Pattern Matching ================ diff --git a/rshell/main.py b/rshell/main.py index ab000d9..367e480 100755 --- a/rshell/main.py +++ b/rshell/main.py @@ -2049,7 +2049,10 @@ def do_m(self, line): Expand a macro with args and run. """ msg = '''usage m MACRO [[[arg0] arg1] ...] - Run macro MACRO with any required arguments.''' + Run macro MACRO with any required arguments. + In general args should be regarded as mandatory. In the case where + no args are passed to a macro expecting some the macro will be run + with each placeholder replaced with an empty string.''' tokens = [x for x in line.split(' ') if x] cmd = tokens[0] if cmd in macros: @@ -2068,7 +2071,7 @@ def do_m(self, line): to_run = go.format('') self.print(to_run) self.onecmd(to_run) - elif cmd == '-h' or l == '--help': + elif cmd == '-h' or cmd == '--help': self.print(msg) else: print_err('Unknown macro', cmd) From f2e18060f47cc8aafa2398e7b3d1ab159792f76a Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Thu, 5 Sep 2019 09:46:59 +0100 Subject: [PATCH 7/8] Implement macro feature. --- README.rst | 7 ------- rshell/main.py | 22 +--------------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/README.rst b/README.rst index a0551e9..5797d39 100644 --- a/README.rst +++ b/README.rst @@ -546,13 +546,6 @@ Synchronisation is performed by comparing the date and time of source and destination files. Files are copied if the source is newer than the destination. -Synchronisation can be configured to ignore files such as documents, -usually to conserve space on the destination. This is done by means -of a file named .rshell-ignore. This should comprise a list of files -and/or subdirectories with each item on a separate line. If such a -file is found in a source directory, items found in the file's -directory that match its contents will not be synchronised. - shell ----- diff --git a/rshell/main.py b/rshell/main.py index 367e480..b40cc91 100755 --- a/rshell/main.py +++ b/rshell/main.py @@ -151,8 +151,6 @@ RTS = '' DTR = '' -IGFILE_NAME = '.rshell-ignore' - # It turns out that just because pyudev is installed doesn't mean that # it can actually be used. So we only bother to try if we're running # under linux. @@ -881,21 +879,6 @@ def rsync(src_dir, dst_dir, mirror, dry_run, print_func, recursed, sync_hidden): for name, stat in src_files: d_src[name] = stat - # Check source for an ignore file - all_src = auto(listdir_stat, src_dir, show_hidden=True) - igfiles = [x for x in all_src if x[0] == IGFILE_NAME] - set_ignore = set() - if len(igfiles): - igfile, mode = igfiles[0] - if mode_isfile(stat_mode(mode)): - with open(src_dir + '/' + igfile, 'r') as f: - for line in f.readlines(): - line = line.strip() - if line: - set_ignore.add(line) - else: - print_err('Ignore file "{:s}" is not a file'.format(IGFILE_NAME)) - d_dst = {} dst_files = auto(listdir_stat, dst_dir, show_hidden=sync_hidden) if dst_files is None: # Directory does not exist @@ -906,7 +889,7 @@ def rsync(src_dir, dst_dir, mirror, dry_run, print_func, recursed, sync_hidden): d_dst[name] = stat set_dst = set(d_dst.keys()) - set_src = set(d_src.keys()) - set_ignore + set_src = set(d_src.keys()) to_add = set_src - set_dst # Files to copy to dest to_del = set_dst - set_src # To delete from dest to_upd = set_dst.intersection(set_src) # In both: may need updating @@ -2740,9 +2723,6 @@ def do_shell(self, line): ), ) - def complete_rsync(self, text, line, begidx, endidx): - return self.filename_complete(text, line, begidx, endidx) - def do_rsync(self, line): """rsync [-m|--mirror] [-n|--dry-run] [-q|--quiet] SRC_DIR DEST_DIR From 2ff57ae108d9e4d32f2e5935ce43bdf9bbd366ad Mon Sep 17 00:00:00 2001 From: Peter Hinch Date: Thu, 26 Sep 2019 08:33:54 +0100 Subject: [PATCH 8/8] Implement default macro file rshell_macros.py --- README.rst | 26 +++++++++++++++++++------- rshell/main.py | 12 ++++++++++-- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 5797d39..fbd48ec 100644 --- a/README.rst +++ b/README.rst @@ -590,14 +590,21 @@ Macros Macros enable short strings to be expanded into longer ones and enable common names to be used to similar or different effect across multiple projects. They also enable rshell functionality to be enhanced, e.g. -adding an mv command to move files. +by adding an mv command to move files. -If rshell is invoked with -m MACRO_MODULE argument the specified Python -module will be imported (assuming it is on the Python path). +Macros are defined by macro modules: these comprise Python code. Their +filenames must conform to Python rules and they should be located on the +Python path. -The module should contain a dict named macros. Each key should be a string +If a module named rshell_macros.py is found, this will be imported. + +If rshell is invoked with -m MACRO_MODULE argument, the specified Python +module will (if found) be imported and its macros appended to any in +rshell_macros.py. + +Macro modules should contain a dict named macros. Each key should be a string specifying the name; the value may be a string (being the expansion) or a -2-tuple. In the case of a tuple the expansion will element[0], with +2-tuple. In the case of a tuple, element[0] is the expansion with element[1] being an arbitrary help string. The macro name must conform to Python rules for dict keys. The expansion @@ -608,7 +615,13 @@ The expansion string may contain argument specifiers compatible with the Python string format operator. This enables arguments passed to the macro to be expanded in ways which are highly flexible. -Consider a global macro module: +Because macro modules contain Python code there are a variety of ways to +configure them: for example macro modules can impport other macro modules. +One approach is to use rshell_macros.py to define global macros applicable +to all projects with project-specific macros being appended with the -m +command line argument. + +rshell_macros.py: :: @@ -625,7 +638,6 @@ A module specific to the foo project: :: - from global_rshell_macros import macros macros['sync'] = 'rsync foo/ /flash/foo/', 'Sync foo project' macros['run'] = 'repl ~ import foo.demos.{}', 'Run foo demo e.g. > m run hst' macros['proj'] = 'ls -l /flash/foo/{}', 'List directory in foo project.' diff --git a/rshell/main.py b/rshell/main.py index b40cc91..928b065 100755 --- a/rshell/main.py +++ b/rshell/main.py @@ -151,6 +151,8 @@ RTS = '' DTR = '' +MACFILE_NAME = 'rshell_macros' + # It turns out that just because pyudev is installed doesn't mean that # it can actually be used. So we only bother to try if we're running # under linux. @@ -2736,14 +2738,18 @@ def do_rsync(self, line): rsync(src_dir, dst_dir, mirror=args.mirror, dry_run=args.dry_run, print_func=pf, recursed=False, sync_hidden=args.all) -def load_macros(mod_name): +def load_macros(mod_name=None): """Update the global macros dict. Validate on import to avoid runtime errors as far as possible. """ + default = mod_name is None + if default: + mod_name = MACFILE_NAME try: mmod = importlib.import_module(mod_name) except ImportError: - print("Can't import macro module", mod_name) + if not default: + print("Can't import macro module", mod_name) return False except: print("Macro module {} is invalid".format(mod_name)) @@ -2966,6 +2972,8 @@ def real_main(): global FAKE_INPUT_PROMPT FAKE_INPUT_PROMPT = True + if load_macros(): # Attempt to load default macro module + print('Default macro file {} loaded OK.'.format(MACFILE_NAME)) if args.macro_module: # Attempt to load a macro module if load_macros(args.macro_module): print('Macro file {} loaded OK.'.format(args.macro_module))